Compare commits

..
Author SHA1 Message Date
pedrooot 77089eba57 docs(api): use mintlify for API specs 2025-10-30 18:58:05 +01:00
1019 changed files with 39973 additions and 53977 deletions
+2 -18
View File
@@ -10,16 +10,13 @@ NEXT_PUBLIC_API_BASE_URL=${API_BASE_URL}
NEXT_PUBLIC_API_DOCS_URL=http://prowler-api:8080/api/v1/docs
AUTH_TRUST_HOST=true
UI_PORT=3000
# Temp URL for feeds need to use actual
RSS_FEED_URL=https://prowler.com/blog/rss
# openssl rand -base64 32
AUTH_SECRET="N/c6mnaS5+SWq81+819OrzQZlmx1Vxtp/orjttJSmw8="
# Google Tag Manager ID
NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID=""
#### Code Review Configuration ####
# Enable Claude Code standards validation on pre-push hook
# Set to 'true' to validate changes against AGENTS.md standards via Claude Code
# Set to 'false' to skip validation
CODE_REVIEW_ENABLED=true
#### Prowler API Configuration ####
PROWLER_API_VERSION="stable"
@@ -38,8 +35,6 @@ POSTGRES_DB=prowler_db
# POSTGRES_REPLICA_USER=prowler
# POSTGRES_REPLICA_PASSWORD=postgres
# POSTGRES_REPLICA_DB=prowler_db
# POSTGRES_REPLICA_MAX_ATTEMPTS=3
# POSTGRES_REPLICA_RETRY_BASE_DELAY=0.5
# Celery-Prowler task settings
TASK_RETRY_DELAY_SECONDS=0.1
@@ -108,8 +103,6 @@ DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute
# Sentry settings
SENTRY_ENVIRONMENT=local
SENTRY_RELEASE=local
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
#### Prowler release version ####
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.12.2
@@ -131,12 +124,3 @@ LANGSMITH_TRACING=false
LANGSMITH_ENDPOINT="https://api.smith.langchain.com"
LANGSMITH_API_KEY=""
LANGCHAIN_PROJECT=""
# RSS Feed Configuration
# Multiple feed sources can be configured as a JSON array (must be valid JSON, no trailing commas)
# Each source requires: id, name, type (github_releases|blog|custom), url, and enabled flag
# IMPORTANT: Must be a single line with valid JSON (no newlines, no trailing commas)
# Example with one source:
RSS_FEED_SOURCES='[{"id":"prowler-releases","name":"Prowler Releases","type":"github_releases","url":"https://github.com/prowler-cloud/prowler/releases.atom","enabled":true}]'
# Example with multiple sources (no trailing comma after last item):
# RSS_FEED_SOURCES='[{"id":"prowler-releases","name":"Prowler Releases","type":"github_releases","url":"https://github.com/prowler-cloud/prowler/releases.atom","enabled":true},{"id":"prowler-blog","name":"Prowler Blog","type":"blog","url":"https://prowler.com/blog/rss","enabled":false}]'
+8 -14
View File
@@ -12,7 +12,7 @@ inputs:
required: false
default: ''
step-outcome:
description: 'Outcome of a step to determine status (success/failure) - automatically sets STATUS_TEXT and STATUS_COLOR env vars'
description: 'Outcome of a step to determine status (success/failure) - automatically sets STATUS_EMOJI, STATUS_TEXT, and STATUS_COLOR env vars'
required: false
default: ''
outputs:
@@ -27,41 +27,35 @@ runs:
shell: bash
run: |
if [[ "${{ inputs.step-outcome }}" == "success" ]]; then
echo "STATUS_TEXT=Completed" >> $GITHUB_ENV
echo "STATUS_COLOR=#6aa84f" >> $GITHUB_ENV
echo "STATUS_EMOJI=[✓]" >> $GITHUB_ENV
echo "STATUS_TEXT=succeeded" >> $GITHUB_ENV
echo "STATUS_COLOR=6aa84f" >> $GITHUB_ENV
elif [[ "${{ inputs.step-outcome }}" == "failure" ]]; then
echo "STATUS_TEXT=Failed" >> $GITHUB_ENV
echo "STATUS_COLOR=#fc3434" >> $GITHUB_ENV
echo "STATUS_EMOJI=[✗]" >> $GITHUB_ENV
echo "STATUS_TEXT=failed" >> $GITHUB_ENV
echo "STATUS_COLOR=fc3434" >> $GITHUB_ENV
else
# No outcome provided - pending/in progress state
echo "STATUS_COLOR=#dbab09" >> $GITHUB_ENV
echo "STATUS_COLOR=dbab09" >> $GITHUB_ENV
fi
- name: Send Slack notification (new message)
if: inputs.update-ts == ''
id: slack-notification-post
uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
env:
SLACK_PAYLOAD_FILE_PATH: ${{ inputs.payload-file-path }}
with:
method: chat.postMessage
token: ${{ inputs.slack-bot-token }}
payload-file-path: ${{ inputs.payload-file-path }}
payload-templated: true
errors: true
- name: Update Slack notification
if: inputs.update-ts != ''
id: slack-notification-update
uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
env:
SLACK_PAYLOAD_FILE_PATH: ${{ inputs.payload-file-path }}
with:
method: chat.update
token: ${{ inputs.slack-bot-token }}
payload-file-path: ${{ inputs.payload-file-path }}
payload-templated: true
errors: true
- name: Set output
id: slack-notification
-7
View File
@@ -22,13 +22,6 @@ Please add a detailed description of how to review this PR.
- [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable.
#### UI
- [ ] All issue/task requirements work as expected on the UI
- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px)
- [ ] Screenshots/Video of the functionality flow (if applicable) - Table (640px > X < 1024px)
- [ ] Screenshots/Video of the functionality flow (if applicable) - Desktop (X > 1024px)
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable.
#### API
- [ ] Verify if API specs need to be regenerated.
- [ ] Check if version updates are required (e.g., specs, Poetry, etc.).
@@ -1,17 +1,4 @@
{
"channel": "${{ env.SLACK_CHANNEL_ID }}",
"attachments": [
{
"color": "${{ env.STATUS_COLOR }}",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Status:*\n${{ env.STATUS_TEXT }}\n\n${{ env.COMPONENT }} container release ${{ env.RELEASE_TAG }} push ${{ env.STATUS_TEXT }}\n\n<${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/actions/runs/${{ env.GITHUB_RUN_ID }}|View run>"
}
}
]
}
]
}
"channel": "$SLACK_CHANNEL_ID",
"text": "$STATUS_EMOJI $COMPONENT container release $RELEASE_TAG push $STATUS_TEXT <$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID|View run>"
}
@@ -1,17 +1,4 @@
{
"channel": "${{ env.SLACK_CHANNEL_ID }}",
"attachments": [
{
"color": "${{ env.STATUS_COLOR }}",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Status:*\nStarted\n\n${{ env.COMPONENT }} container release ${{ env.RELEASE_TAG }} push started...\n\n<${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/actions/runs/${{ env.GITHUB_RUN_ID }}|View run>"
}
}
]
}
]
"channel": "$SLACK_CHANNEL_ID",
"text": "$COMPONENT container release $RELEASE_TAG push started... <$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID|View run>"
}
+6 -3
View File
@@ -5,10 +5,16 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'api/**'
- '.github/workflows/api-code-quality.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'api/**'
- '.github/workflows/api-code-quality.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -39,9 +45,6 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
api/**
.github/workflows/api-code-quality.yml
files_ignore: |
api/docs/**
api/README.md
+3 -3
View File
@@ -25,7 +25,7 @@ concurrency:
cancel-in-progress: true
jobs:
api-analyze:
analyze:
name: CodeQL Security Analysis
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -45,12 +45,12 @@ jobs:
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Initialize CodeQL
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
with:
languages: ${{ matrix.language }}
config-file: ./.github/codeql/api-codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
with:
category: '/language:${{ matrix.language }}'
+22 -71
View File
@@ -11,12 +11,6 @@ on:
release:
types:
- 'published'
workflow_dispatch:
inputs:
release_tag:
description: 'Release tag (e.g., 5.14.0)'
required: true
type: string
permissions:
contents: read
@@ -28,7 +22,7 @@ concurrency:
env:
# Tags
LATEST_TAG: latest
RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }}
RELEASE_TAG: ${{ github.event.release.tag_name }}
STABLE_TAG: stable
WORKING_DIRECTORY: ./api
@@ -50,16 +44,7 @@ jobs:
container-build-push:
needs: setup
runs-on: ${{ matrix.runner }}
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
arch: amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
arch: arm64
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
@@ -78,8 +63,20 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build and push API container (latest)
if: github.event_name == 'push'
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: ${{ env.WORKING_DIRECTORY }}
push: true
tags: |
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Notify container push started
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
if: github.event_name == 'release'
uses: ./.github/actions/slack-notification
env:
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
@@ -92,21 +89,21 @@ jobs:
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
- name: Build and push API container for ${{ matrix.arch }}
- name: Build and push API container (release)
if: github.event_name == 'release'
id: container-push
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: ${{ env.WORKING_DIRECTORY }}
push: true
platforms: ${{ matrix.platform }}
tags: |
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }}
cache-from: type=gha,scope=${{ matrix.arch }}
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Notify container push completed
if: (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && always()
if: github.event_name == 'release' && always()
uses: ./.github/actions/slack-notification
env:
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
@@ -120,52 +117,6 @@ jobs:
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
step-outcome: ${{ steps.container-push.outcome }}
# Create and push multi-architecture manifest
create-manifest:
needs: [setup, container-build-push]
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Create and push manifests for push event
if: github.event_name == 'push'
run: |
docker buildx imagetools create \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
- name: Create and push manifests for release event
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
run: |
docker buildx imagetools create \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
- name: Install regctl
if: always()
uses: regclient/actions/regctl-installer@f61d18f46c86af724a9c804cb9ff2a6fec741c7c # main
- name: Cleanup intermediate architecture tags
if: always()
run: |
echo "Cleaning up intermediate tags..."
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64" || true
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true
echo "Cleanup completed"
trigger-deployment:
if: github.event_name == 'push'
needs: [setup, container-build-push]
+6 -1
View File
@@ -5,10 +5,16 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'api/**'
- '.github/workflows/api-container-checks.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'api/**'
- '.github/workflows/api-container-checks.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -58,7 +64,6 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: api/**
files_ignore: |
api/docs/**
api/README.md
+6 -3
View File
@@ -5,10 +5,16 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'api/**'
- '.github/workflows/api-security.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'api/**'
- '.github/workflows/api-security.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -39,9 +45,6 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
api/**
.github/workflows/api-security.yml
files_ignore: |
api/docs/**
api/README.md
+6 -3
View File
@@ -5,10 +5,16 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'api/**'
- '.github/workflows/api-tests.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'api/**'
- '.github/workflows/api-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -79,9 +85,6 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
api/**
.github/workflows/api-tests.yml
files_ignore: |
api/docs/**
api/README.md
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
- name: Backport PR
if: steps.label_check.outputs.label_check == 'success'
uses: sorenlouv/backport-github-action@516854e7c9f962b9939085c9a92ea28411d1ae90 # v10.2.0
uses: sorenlouv/backport-github-action@ad888e978060bc1b2798690dd9d03c4036560947 # v9.5.1
with:
github_token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
auto_backport_label_prefix: ${{ env.BACKPORT_LABEL_PREFIX }}
@@ -1,39 +0,0 @@
name: 'Tools: Comment Label Update'
on:
issue_comment:
types:
- 'created'
concurrency:
group: ${{ github.workflow }}-${{ github.event.issue.number }}
cancel-in-progress: false
jobs:
update-labels:
if: contains(github.event.issue.labels.*.name, 'status/awaiting-response')
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
issues: write
pull-requests: write
steps:
- name: Remove 'status/awaiting-response' label
env:
GH_TOKEN: ${{ github.token }}
ISSUE_NUMBER: ${{ github.event.issue.number }}
run: |
echo "Removing 'status/awaiting-response' label from #$ISSUE_NUMBER"
gh api /repos/${{ github.repository }}/issues/$ISSUE_NUMBER/labels/status%2Fawaiting-response \
-X DELETE
- name: Add 'status/waiting-for-revision' label
env:
GH_TOKEN: ${{ github.token }}
ISSUE_NUMBER: ${{ github.event.issue.number }}
run: |
echo "Adding 'status/waiting-for-revision' label to #$ISSUE_NUMBER"
gh api /repos/${{ github.repository }}/issues/$ISSUE_NUMBER/labels \
-X POST \
-f labels[]='status/waiting-for-revision'
+1 -1
View File
@@ -26,6 +26,6 @@ jobs:
steps:
- name: Check PR title format
uses: agenthunt/conventional-commit-checker-action@f1823f632e95a64547566dcd2c7da920e67117ad # v2.0.1
uses: agenthunt/conventional-commit-checker-action@9e552d650d0e205553ec7792d447929fc78e012b # v2.0.0
with:
pr-title-regex: '^(feat|fix|docs|style|refactor|perf|test|chore|build|ci|revert)(\([^)]+\))?!?: .+'
+1 -1
View File
@@ -28,6 +28,6 @@ jobs:
fetch-depth: 0
- name: Scan for secrets with TruffleHog
uses: trufflesecurity/trufflehog@b84c3d14d189e16da175e2c27fa8136603783ffc # v3.90.12
uses: trufflesecurity/trufflehog@ad6fc8fb446b8fafbf7ea8193d2d6bfd42f45690 # v3.90.11
with:
extra_args: '--results=verified,unknown'
+43
View File
@@ -0,0 +1,43 @@
name: Community PR labelling
on:
# We need "write" permissions on the PR to be able to add a label.
pull_request_target: # We need this to have labelling permissions. There are no user inputs here, so we should be fine.
types:
- opened
permissions: {}
jobs:
label-if-community:
name: Add 'community' label if the PR is from a community contributor
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Check if author is org member
id: check_membership
env:
GH_TOKEN: ${{ github.token }}
AUTHOR: ${{ github.event.pull_request.user.login }}
ORG: ${{ github.repository_owner }}
run: |
echo "Checking if $AUTHOR is a member of $ORG"
if gh api --method GET "orgs/$ORG/members/$AUTHOR" >/dev/null 2>&1; then
echo "is_member=true" >> $GITHUB_OUTPUT
echo "$AUTHOR is an organization member"
else
echo "is_member=false" >> $GITHUB_OUTPUT
echo "$AUTHOR is not an organization member"
fi
- name: Add community label
if: steps.check_membership.outputs.is_member == 'false'
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
GH_TOKEN: ${{ github.token }}
run: |
echo "Adding 'community' label to PR #$PR_NUMBER"
gh pr edit "$PR_NUMBER" --add-label community
-63
View File
@@ -27,66 +27,3 @@ jobs:
uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1
with:
sync-labels: true
label-community:
name: Add 'community' label if the PR is from a community contributor
needs: labeler
if: github.repository == 'prowler-cloud/prowler' && github.event.action == 'opened'
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Check if author is org member
id: check_membership
env:
AUTHOR: ${{ github.event.pull_request.user.login }}
run: |
# Hardcoded list of prowler-cloud organization members
# This list includes members who have set their organization membership as private
ORG_MEMBERS=(
"AdriiiPRodri"
"Alan-TheGentleman"
"alejandrobailo"
"amitsharm"
"andoniaf"
"cesararroba"
"Chan9390"
"danibarranqueroo"
"HugoPBrito"
"jfagoagas"
"josemazo"
"lydiavilchez"
"mmuller88"
"MrCloudSec"
"pedrooot"
"prowler-bot"
"puchy22"
"rakan-pro"
"RosaRivasProwler"
"StylusFrost"
"toniblyx"
"vicferpoy"
)
echo "Checking if $AUTHOR is a member of prowler-cloud organization"
# Check if author is in the org members list
if printf '%s\n' "${ORG_MEMBERS[@]}" | grep -q "^${AUTHOR}$"; then
echo "is_member=true" >> $GITHUB_OUTPUT
echo "$AUTHOR is an organization member"
else
echo "is_member=false" >> $GITHUB_OUTPUT
echo "$AUTHOR is not an organization member"
fi
- name: Add community label
if: steps.check_membership.outputs.is_member == 'false'
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
GH_TOKEN: ${{ github.token }}
run: |
echo "Adding 'community' label to PR #$PR_NUMBER"
gh api /repos/${{ github.repository }}/issues/${{ github.event.number }}/labels \
-X POST \
-f labels[]='community'
+31 -73
View File
@@ -10,12 +10,6 @@ on:
release:
types:
- 'published'
workflow_dispatch:
inputs:
release_tag:
description: 'Release tag (e.g., 5.14.0)'
required: true
type: string
permissions:
contents: read
@@ -27,7 +21,7 @@ concurrency:
env:
# Tags
LATEST_TAG: latest
RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }}
RELEASE_TAG: ${{ github.event.release.tag_name }}
STABLE_TAG: stable
WORKING_DIRECTORY: ./mcp_server
@@ -49,16 +43,7 @@ jobs:
container-build-push:
needs: setup
runs-on: ${{ matrix.runner }}
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
arch: amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
arch: arm64
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
@@ -76,8 +61,27 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build and push MCP container (latest)
if: github.event_name == 'push'
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: ${{ env.WORKING_DIRECTORY }}
push: true
tags: |
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}
labels: |
org.opencontainers.image.title=Prowler MCP Server
org.opencontainers.image.description=Model Context Protocol server for Prowler
org.opencontainers.image.vendor=ProwlerPro, Inc.
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.created=${{ github.event.head_commit.timestamp }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Notify container push started
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
if: github.event_name == 'release'
uses: ./.github/actions/slack-notification
env:
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
@@ -90,29 +94,29 @@ jobs:
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
- name: Build and push MCP container for ${{ matrix.arch }}
- name: Build and push MCP container (release)
if: github.event_name == 'release'
id: container-push
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: ${{ env.WORKING_DIRECTORY }}
push: true
platforms: ${{ matrix.platform }}
tags: |
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }}
labels: |
org.opencontainers.image.title=Prowler MCP Server
org.opencontainers.image.description=Model Context Protocol server for Prowler
org.opencontainers.image.vendor=ProwlerPro, Inc.
org.opencontainers.image.version=${{ env.RELEASE_TAG }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.created=${{ github.event_name == 'release' && github.event.release.published_at || github.event.head_commit.timestamp }}
${{ github.event_name == 'release' && format('org.opencontainers.image.version={0}', env.RELEASE_TAG) || '' }}
cache-from: type=gha,scope=${{ matrix.arch }}
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
org.opencontainers.image.created=${{ github.event.release.published_at }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Notify container push completed
if: (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && always()
if: github.event_name == 'release' && always()
uses: ./.github/actions/slack-notification
env:
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
@@ -126,52 +130,6 @@ jobs:
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
step-outcome: ${{ steps.container-push.outcome }}
# Create and push multi-architecture manifest
create-manifest:
needs: [setup, container-build-push]
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Create and push manifests for push event
if: github.event_name == 'push'
run: |
docker buildx imagetools create \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
- name: Create and push manifests for release event
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
run: |
docker buildx imagetools create \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
- name: Install regctl
if: always()
uses: regclient/actions/regctl-installer@main
- name: Cleanup intermediate architecture tags
if: always()
run: |
echo "Cleaning up intermediate tags..."
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64" || true
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true
echo "Cleanup completed"
trigger-deployment:
if: github.event_name == 'push'
needs: [setup, container-build-push]
+6 -1
View File
@@ -5,10 +5,16 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'mcp_server/**'
- '.github/workflows/mcp-container-checks.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'mcp_server/**'
- '.github/workflows/mcp-container-checks.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -57,7 +63,6 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: mcp_server/**
files_ignore: |
mcp_server/README.md
mcp_server/CHANGELOG.md
+1 -1
View File
@@ -83,7 +83,7 @@ jobs:
- name: Update PR comment with changelog status
if: github.event.pull_request.head.repo.full_name == github.repository
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
with:
issue-number: ${{ github.event.pull_request.number }}
comment-id: ${{ steps.find-comment.outputs.comment-id }}
+1 -1
View File
@@ -97,7 +97,7 @@ jobs:
body-includes: '<!-- conflict-checker-comment -->'
- name: Create or update comment
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
with:
comment-id: ${{ steps.find-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
+81 -71
View File
@@ -47,7 +47,7 @@ jobs:
git config --global user.name 'prowler-bot'
git config --global user.email '179230569+prowler-bot@users.noreply.github.com'
- name: Parse version and determine branch
- name: Parse version and read changelogs
run: |
# Validate version format (reusing pattern from sdk-bump-version.yml)
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
@@ -64,83 +64,66 @@ jobs:
BRANCH_NAME="v${MAJOR_VERSION}.${MINOR_VERSION}"
echo "BRANCH_NAME=${BRANCH_NAME}" >> "${GITHUB_ENV}"
# Function to extract the latest version from changelog
extract_latest_version() {
local changelog_file="$1"
if [ -f "$changelog_file" ]; then
# Extract the first version entry (most recent) from changelog
# Format: ## [version] (1.2.3) or ## [vversion] (v1.2.3)
local version=$(grep -m 1 '^## \[' "$changelog_file" | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]')
echo "$version"
else
echo ""
fi
}
# Read actual versions from changelogs (source of truth)
UI_VERSION=$(extract_latest_version "ui/CHANGELOG.md")
API_VERSION=$(extract_latest_version "api/CHANGELOG.md")
SDK_VERSION=$(extract_latest_version "prowler/CHANGELOG.md")
MCP_VERSION=$(extract_latest_version "mcp_server/CHANGELOG.md")
echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}"
echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}"
echo "SDK_VERSION=${SDK_VERSION}" >> "${GITHUB_ENV}"
echo "MCP_VERSION=${MCP_VERSION}" >> "${GITHUB_ENV}"
if [ -n "$UI_VERSION" ]; then
echo "Read UI version from changelog: $UI_VERSION"
else
echo "Warning: No UI version found in ui/CHANGELOG.md"
fi
if [ -n "$API_VERSION" ]; then
echo "Read API version from changelog: $API_VERSION"
else
echo "Warning: No API version found in api/CHANGELOG.md"
fi
if [ -n "$SDK_VERSION" ]; then
echo "Read SDK version from changelog: $SDK_VERSION"
else
echo "Warning: No SDK version found in prowler/CHANGELOG.md"
fi
if [ -n "$MCP_VERSION" ]; then
echo "Read MCP version from changelog: $MCP_VERSION"
else
echo "Warning: No MCP version found in mcp_server/CHANGELOG.md"
fi
echo "Prowler version: $PROWLER_VERSION"
echo "Branch name: $BRANCH_NAME"
echo "UI version: $UI_VERSION"
echo "API version: $API_VERSION"
echo "SDK version: $SDK_VERSION"
echo "MCP version: $MCP_VERSION"
echo "Is minor release: $([ $PATCH_VERSION -eq 0 ] && echo 'true' || echo 'false')"
else
echo "Invalid version syntax: '$PROWLER_VERSION' (must be N.N.N)" >&2
exit 1
fi
- name: Checkout release branch
run: |
echo "Checking out branch $BRANCH_NAME for release $PROWLER_VERSION..."
if git show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then
echo "Branch $BRANCH_NAME exists locally, checking out..."
git checkout "$BRANCH_NAME"
elif git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then
echo "Branch $BRANCH_NAME exists remotely, checking out..."
git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME"
else
echo "ERROR: Branch $BRANCH_NAME does not exist. For minor releases (X.Y.0), create it manually first. For patch releases (X.Y.Z), the branch should already exist."
exit 1
fi
- name: Read changelog versions from release branch
run: |
# Function to extract the latest version from changelog
extract_latest_version() {
local changelog_file="$1"
if [ -f "$changelog_file" ]; then
# Extract the first version entry (most recent) from changelog
# Format: ## [version] (1.2.3) or ## [vversion] (v1.2.3)
local version=$(grep -m 1 '^## \[' "$changelog_file" | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]')
echo "$version"
else
echo ""
fi
}
# Read actual versions from changelogs (source of truth)
UI_VERSION=$(extract_latest_version "ui/CHANGELOG.md")
API_VERSION=$(extract_latest_version "api/CHANGELOG.md")
SDK_VERSION=$(extract_latest_version "prowler/CHANGELOG.md")
MCP_VERSION=$(extract_latest_version "mcp_server/CHANGELOG.md")
echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}"
echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}"
echo "SDK_VERSION=${SDK_VERSION}" >> "${GITHUB_ENV}"
echo "MCP_VERSION=${MCP_VERSION}" >> "${GITHUB_ENV}"
if [ -n "$UI_VERSION" ]; then
echo "Read UI version from changelog: $UI_VERSION"
else
echo "Warning: No UI version found in ui/CHANGELOG.md"
fi
if [ -n "$API_VERSION" ]; then
echo "Read API version from changelog: $API_VERSION"
else
echo "Warning: No API version found in api/CHANGELOG.md"
fi
if [ -n "$SDK_VERSION" ]; then
echo "Read SDK version from changelog: $SDK_VERSION"
else
echo "Warning: No SDK version found in prowler/CHANGELOG.md"
fi
if [ -n "$MCP_VERSION" ]; then
echo "Read MCP version from changelog: $MCP_VERSION"
else
echo "Warning: No MCP version found in mcp_server/CHANGELOG.md"
fi
echo "UI version: $UI_VERSION"
echo "API version: $API_VERSION"
echo "SDK version: $SDK_VERSION"
echo "MCP version: $MCP_VERSION"
- name: Extract and combine changelog entries
run: |
set -e
@@ -272,6 +255,21 @@ jobs:
echo "Combined changelog preview:"
cat combined_changelog.md
- name: Checkout release branch for patch release
if: ${{ env.PATCH_VERSION != '0' }}
run: |
echo "Patch release detected, checking out existing branch $BRANCH_NAME..."
if git show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then
echo "Branch $BRANCH_NAME exists locally, checking out..."
git checkout "$BRANCH_NAME"
elif git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then
echo "Branch $BRANCH_NAME exists remotely, checking out..."
git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME"
else
echo "ERROR: Branch $BRANCH_NAME should exist for patch release $PROWLER_VERSION"
exit 1
fi
- name: Verify SDK version in pyproject.toml
run: |
CURRENT_VERSION=$(grep '^version = ' pyproject.toml | sed -E 's/version = "([^"]+)"/\1/' | tr -d '[:space:]')
@@ -325,6 +323,18 @@ jobs:
fi
echo "✓ api/src/backend/api/v1/views.py version: $CURRENT_API_VERSION"
- name: Checkout release branch for minor release
if: ${{ env.PATCH_VERSION == '0' }}
run: |
echo "Minor release detected (patch = 0), checking out existing branch $BRANCH_NAME..."
if git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then
echo "Branch $BRANCH_NAME exists remotely, checking out..."
git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME"
else
echo "ERROR: Branch $BRANCH_NAME should exist for minor release $PROWLER_VERSION. Please create it manually first."
exit 1
fi
- name: Update API prowler dependency for minor release
if: ${{ env.PATCH_VERSION == '0' }}
run: |
@@ -382,7 +392,7 @@ jobs:
no-changelog
- name: Create draft release
uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2.4.1
uses: softprops/action-gh-release@6cbd405e2c4e67a21c47fa9e383d020e4e28b836 # v2.3.3
with:
tag_name: ${{ env.PROWLER_VERSION }}
name: Prowler ${{ env.PROWLER_VERSION }}
+12 -3
View File
@@ -5,10 +5,22 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'prowler/**'
- 'tests/**'
- 'pyproject.toml'
- 'poetry.lock'
- '.github/workflows/sdk-code-quality.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'prowler/**'
- 'tests/**'
- 'pyproject.toml'
- 'poetry.lock'
- '.github/workflows/sdk-code-quality.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -16,7 +28,6 @@ concurrency:
jobs:
sdk-code-quality:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
@@ -37,9 +48,7 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ./**
files_ignore: |
.github/**
prowler/CHANGELOG.md
docs/**
permissions/**
+3 -4
View File
@@ -31,8 +31,7 @@ concurrency:
cancel-in-progress: true
jobs:
sdk-analyze:
if: github.repository == 'prowler-cloud/prowler'
analyze:
name: CodeQL Security Analysis
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -52,12 +51,12 @@ jobs:
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Initialize CodeQL
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
with:
languages: ${{ matrix.language }}
config-file: ./.github/codeql/sdk-codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
with:
category: '/language:${{ matrix.language }}'
+27 -92
View File
@@ -16,12 +16,6 @@ on:
release:
types:
- 'published'
workflow_dispatch:
inputs:
release_tag:
description: 'Release tag (e.g., 5.14.0)'
required: true
type: string
permissions:
contents: read
@@ -52,16 +46,7 @@ env:
jobs:
container-build-push:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ${{ matrix.runner }}
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
arch: amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
arch: arm64
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
contents: read
@@ -69,8 +54,6 @@ jobs:
outputs:
prowler_version: ${{ steps.get-prowler-version.outputs.prowler_version }}
prowler_version_major: ${{ steps.get-prowler-version.outputs.prowler_version_major }}
latest_tag: ${{ steps.get-prowler-version.outputs.latest_tag }}
stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }}
env:
POETRY_VIRTUALENVS_CREATE: 'false'
@@ -105,22 +88,16 @@ jobs:
3)
echo "LATEST_TAG=v3-latest" >> "${GITHUB_ENV}"
echo "STABLE_TAG=v3-stable" >> "${GITHUB_ENV}"
echo "latest_tag=v3-latest" >> "${GITHUB_OUTPUT}"
echo "stable_tag=v3-stable" >> "${GITHUB_OUTPUT}"
echo "✓ Prowler v3 detected - tags: v3-latest, v3-stable"
;;
4)
echo "LATEST_TAG=v4-latest" >> "${GITHUB_ENV}"
echo "STABLE_TAG=v4-stable" >> "${GITHUB_ENV}"
echo "latest_tag=v4-latest" >> "${GITHUB_OUTPUT}"
echo "stable_tag=v4-stable" >> "${GITHUB_OUTPUT}"
echo "✓ Prowler v4 detected - tags: v4-latest, v4-stable"
;;
5)
echo "LATEST_TAG=latest" >> "${GITHUB_ENV}"
echo "STABLE_TAG=stable" >> "${GITHUB_ENV}"
echo "latest_tag=latest" >> "${GITHUB_OUTPUT}"
echo "stable_tag=stable" >> "${GITHUB_OUTPUT}"
echo "✓ Prowler v5 detected - tags: latest, stable"
;;
*)
@@ -147,8 +124,22 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build and push SDK container (latest)
if: github.event_name == 'push'
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: .
file: ${{ env.DOCKERFILE_PATH }}
push: true
tags: |
${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }}
${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Notify container push started
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
if: github.event_name == 'release'
uses: ./.github/actions/slack-notification
env:
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
@@ -161,22 +152,26 @@ jobs:
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
- name: Build and push SDK container for ${{ matrix.arch }}
- name: Build and push SDK container (release)
if: github.event_name == 'release'
id: container-push
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: .
file: ${{ env.DOCKERFILE_PATH }}
push: true
platforms: ${{ matrix.platform }}
tags: |
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }}-${{ matrix.arch }}
cache-from: type=gha,scope=${{ matrix.arch }}
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.PROWLER_VERSION }}
${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.STABLE_TAG }}
${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.PROWLER_VERSION }}
${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.STABLE_TAG }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.PROWLER_VERSION }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Notify container push completed
if: (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && always()
if: github.event_name == 'release' && always()
uses: ./.github/actions/slack-notification
env:
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
@@ -190,66 +185,6 @@ jobs:
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
step-outcome: ${{ steps.container-push.outcome }}
# Create and push multi-architecture manifest
create-manifest:
needs: [container-build-push]
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Public ECR
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
registry: public.ecr.aws
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
env:
AWS_REGION: ${{ env.AWS_REGION }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Create and push manifests for push event
if: github.event_name == 'push'
run: |
docker buildx imagetools create \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \
-t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \
-t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64 \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64
- name: Create and push manifests for release event
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
run: |
docker buildx imagetools create \
-t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.prowler_version }} \
-t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.stable_tag }} \
-t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.prowler_version }} \
-t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.stable_tag }} \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.prowler_version }} \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.stable_tag }} \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64 \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64
- name: Install regctl
if: always()
uses: regclient/actions/regctl-installer@main
- name: Cleanup intermediate architecture tags
if: always()
run: |
echo "Cleaning up intermediate tags..."
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64" || true
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64" || true
echo "Cleanup completed"
dispatch-v3-deployment:
if: needs.container-build-push.outputs.prowler_version_major == '3'
needs: container-build-push
+11 -5
View File
@@ -5,10 +5,20 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'prowler/**'
- 'tests/**'
- 'Dockerfile'
- '.github/workflows/sdk-container-checks.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'prowler/**'
- 'tests/**'
- 'Dockerfile'
- '.github/workflows/sdk-container-checks.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -19,7 +29,6 @@ env:
jobs:
sdk-dockerfile-lint:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
@@ -43,7 +52,6 @@ jobs:
ignore: DL3013
sdk-container-build-and-scan:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
@@ -59,9 +67,7 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ./**
files_ignore: |
.github/**
prowler/CHANGELOG.md
docs/**
permissions/**
@@ -94,7 +100,7 @@ jobs:
cache-to: type=gha,mode=max
- name: Scan SDK container with Trivy
if: steps.check-changes.outputs.any_changed == 'true'
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
uses: ./.github/actions/trivy-scan
with:
image-name: ${{ env.IMAGE_NAME }}
@@ -39,7 +39,7 @@ jobs:
run: pip install boto3
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0
with:
aws-region: ${{ env.AWS_REGION }}
role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }}
+12 -3
View File
@@ -5,10 +5,22 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'prowler/**'
- 'tests/**'
- 'pyproject.toml'
- 'poetry.lock'
- '.github/workflows/sdk-security.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'prowler/**'
- 'tests/**'
- 'pyproject.toml'
- 'poetry.lock'
- '.github/workflows/sdk-security.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -16,7 +28,6 @@ concurrency:
jobs:
sdk-security-scans:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
@@ -30,9 +41,7 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ./**
files_ignore: |
.github/**
prowler/CHANGELOG.md
docs/**
permissions/**
+12 -3
View File
@@ -5,10 +5,22 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'prowler/**'
- 'tests/**'
- 'pyproject.toml'
- 'poetry.lock'
- '.github/workflows/sdk-tests.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'prowler/**'
- 'tests/**'
- 'pyproject.toml'
- 'poetry.lock'
- '.github/workflows/sdk-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -16,7 +28,6 @@ concurrency:
jobs:
sdk-tests:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 120
permissions:
@@ -37,9 +48,7 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ./**
files_ignore: |
.github/**
prowler/CHANGELOG.md
docs/**
permissions/**
+3 -4
View File
@@ -27,8 +27,7 @@ concurrency:
cancel-in-progress: true
jobs:
ui-analyze:
if: github.repository == 'prowler-cloud/prowler'
analyze:
name: CodeQL Security Analysis
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -48,12 +47,12 @@ jobs:
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Initialize CodeQL
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
with:
languages: ${{ matrix.language }}
config-file: ./.github/codeql/ui-codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
with:
category: '/language:${{ matrix.language }}'
+26 -72
View File
@@ -10,12 +10,6 @@ on:
release:
types:
- 'published'
workflow_dispatch:
inputs:
release_tag:
description: 'Release tag (e.g., 5.14.0)'
required: true
type: string
permissions:
contents: read
@@ -27,7 +21,7 @@ concurrency:
env:
# Tags
LATEST_TAG: latest
RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }}
RELEASE_TAG: ${{ github.event.release.tag_name }}
STABLE_TAG: stable
WORKING_DIRECTORY: ./ui
@@ -52,16 +46,7 @@ jobs:
container-build-push:
needs: setup
runs-on: ${{ matrix.runner }}
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
arch: amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
arch: arm64
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
@@ -80,8 +65,23 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build and push UI container (latest)
if: github.event_name == 'push'
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: ${{ env.WORKING_DIRECTORY }}
build-args: |
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ needs.setup.outputs.short-sha }}
NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }}
push: true
tags: |
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Notify container push started
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
if: github.event_name == 'release'
uses: ./.github/actions/slack-notification
env:
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
@@ -94,24 +94,24 @@ jobs:
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
- name: Build and push UI container for ${{ matrix.arch }}
- name: Build and push UI container (release)
if: github.event_name == 'release'
id: container-push
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: ${{ env.WORKING_DIRECTORY }}
build-args: |
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && format('v{0}', env.RELEASE_TAG) || needs.setup.outputs.short-sha }}
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${{ env.RELEASE_TAG }}
NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }}
push: true
platforms: ${{ matrix.platform }}
tags: |
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }}
cache-from: type=gha,scope=${{ matrix.arch }}
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }}
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Notify container push completed
if: (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && always()
if: github.event_name == 'release' && always()
uses: ./.github/actions/slack-notification
env:
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
@@ -125,52 +125,6 @@ jobs:
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
step-outcome: ${{ steps.container-push.outcome }}
# Create and push multi-architecture manifest
create-manifest:
needs: [setup, container-build-push]
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Create and push manifests for push event
if: github.event_name == 'push'
run: |
docker buildx imagetools create \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
- name: Create and push manifests for release event
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
run: |
docker buildx imagetools create \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} \
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
- name: Install regctl
if: always()
uses: regclient/actions/regctl-installer@main
- name: Cleanup intermediate architecture tags
if: always()
run: |
echo "Cleaning up intermediate tags..."
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64" || true
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true
echo "Cleanup completed"
trigger-deployment:
if: github.event_name == 'push'
needs: [setup, container-build-push]
+6 -1
View File
@@ -5,10 +5,16 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'ui/**'
- '.github/workflows/ui-container-checks.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'ui/**'
- '.github/workflows/ui-container-checks.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -58,7 +64,6 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ui/**
files_ignore: |
ui/CHANGELOG.md
ui/README.md
+1 -10
View File
@@ -24,15 +24,6 @@ jobs:
E2E_AWS_PROVIDER_ACCESS_KEY: ${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}
E2E_AWS_PROVIDER_SECRET_KEY: ${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}
E2E_AWS_PROVIDER_ROLE_ARN: ${{ secrets.E2E_AWS_PROVIDER_ROLE_ARN }}
E2E_AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_AZURE_SUBSCRIPTION_ID }}
E2E_AZURE_CLIENT_ID: ${{ secrets.E2E_AZURE_CLIENT_ID }}
E2E_AZURE_SECRET_ID: ${{ secrets.E2E_AZURE_SECRET_ID }}
E2E_AZURE_TENANT_ID: ${{ secrets.E2E_AZURE_TENANT_ID }}
E2E_M365_DOMAIN_ID: ${{ secrets.E2E_M365_DOMAIN_ID }}
E2E_M365_CLIENT_ID: ${{ secrets.E2E_M365_CLIENT_ID }}
E2E_M365_SECRET_ID: ${{ secrets.E2E_M365_SECRET_ID }}
E2E_M365_TENANT_ID: ${{ secrets.E2E_M365_TENANT_ID }}
E2E_M365_CERTIFICATE_CONTENT: ${{ secrets.E2E_M365_CERTIFICATE_CONTENT }}
E2E_NEW_PASSWORD: ${{ secrets.E2E_NEW_PASSWORD }}
steps:
- name: Checkout repository
@@ -75,7 +66,7 @@ jobs:
echo "All database fixtures loaded successfully!"
'
- name: Setup Node.js environment
uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version: '20.x'
cache: 'npm'
+6 -3
View File
@@ -5,10 +5,16 @@ on:
branches:
- 'master'
- 'v5.*'
paths:
- 'ui/**'
- '.github/workflows/ui-tests.yml'
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'ui/**'
- '.github/workflows/ui-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -36,9 +42,6 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
ui/**
.github/workflows/ui-tests.yml
files_ignore: |
ui/CHANGELOG.md
ui/README.md
+5 -69
View File
@@ -45,86 +45,21 @@ pytest_*.xml
.coverage
htmlcov/
# VSCode files and settings
# VSCode files
.vscode/
*.code-workspace
.vscode-test/
# VSCode extension settings and workspaces
.history/
.ionide/
# MCP Server Settings (various locations)
**/cline_mcp_settings.json
**/mcp_settings.json
**/mcp-config.json
**/mcpServers.json
.mcp/
# AI Coding Assistants - Cursor
# Cursor files
.cursorignore
.cursor/
.cursorrules
# AI Coding Assistants - RooCode
# RooCode files
.roo/
.rooignore
.roomodes
# AI Coding Assistants - Cline (formerly Claude Dev)
# Cline files
.cline/
.clineignore
.clinerules
# AI Coding Assistants - Continue
.continue/
continue.json
.continuerc
.continuerc.json
# AI Coding Assistants - GitHub Copilot
.copilot/
.github/copilot/
# AI Coding Assistants - Amazon Q Developer (formerly CodeWhisperer)
.aws/
.codewhisperer/
.amazonq/
.aws-toolkit/
# AI Coding Assistants - Tabnine
.tabnine/
tabnine_config.json
# AI Coding Assistants - Kiro
.kiro/
.kiroignore
kiro.config.json
# AI Coding Assistants - Aider
.aider/
.aider.chat.history.md
.aider.input.history
.aider.tags.cache.v3/
# AI Coding Assistants - Windsurf
.windsurf/
.windsurfignore
# AI Coding Assistants - Replit Agent
.replit
.replitignore
# AI Coding Assistants - Supermaven
.supermaven/
# AI Coding Assistants - Sourcegraph Cody
.cody/
# AI Coding Assistants - General
.ai/
.aiconfig
ai-config.json
# Terraform
.terraform*
@@ -135,6 +70,7 @@ ai-config.json
ui/.env*
api/.env*
mcp_server/.env*
.env.local
# Coverage
.coverage*
-9
View File
@@ -126,12 +126,3 @@ repos:
entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/" --min-confidence 100 .'
language: system
files: '.*\.py'
- id: ui-checks
name: UI - Husky Pre-commit
description: "Run UI pre-commit checks (Claude Code validation + healthcheck)"
entry: bash -c 'cd ui && .husky/pre-commit'
language: system
files: '^ui/.*\.(ts|tsx|js|jsx|json|css)$'
pass_filenames: false
verbose: true
-22
View File
@@ -4,10 +4,6 @@ LABEL maintainer="https://github.com/prowler-cloud/prowler"
LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
ARG POWERSHELL_VERSION=7.5.0
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
ARG TRIVY_VERSION=0.66.0
ENV TRIVY_VERSION=${TRIVY_VERSION}
# hadolint ignore=DL3008
RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -29,24 +25,6 @@ RUN ARCH=$(uname -m) && \
ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh && \
rm /tmp/powershell.tar.gz
# Install Trivy for IaC scanning
RUN ARCH=$(uname -m) && \
if [ "$ARCH" = "x86_64" ]; then \
TRIVY_ARCH="Linux-64bit" ; \
elif [ "$ARCH" = "aarch64" ]; then \
TRIVY_ARCH="Linux-ARM64" ; \
else \
echo "Unsupported architecture for Trivy: $ARCH" && exit 1 ; \
fi && \
wget --progress=dot:giga "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_${TRIVY_ARCH}.tar.gz" -O /tmp/trivy.tar.gz && \
tar zxf /tmp/trivy.tar.gz -C /tmp && \
mv /tmp/trivy /usr/local/bin/trivy && \
chmod +x /usr/local/bin/trivy && \
rm /tmp/trivy.tar.gz && \
# Create trivy cache directory with proper permissions
mkdir -p /tmp/.cache/trivy && \
chmod 777 /tmp/.cache/trivy
# Add prowler user
RUN addgroup --gid 1000 prowler && \
adduser --uid 1000 --gid 1000 --disabled-password --gecos "" prowler
+14 -14
View File
@@ -56,7 +56,7 @@ Prowler includes hundreds of built-in controls to ensure compliance with standar
Prowler App is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments.
![Prowler App](docs/images/products/overview.png)
![Prowler App](docs/products/img/overview.png)
>For more details, refer to the [Prowler App Documentation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-app-installation)
@@ -73,26 +73,26 @@ prowler <provider>
```console
prowler dashboard
```
![Prowler Dashboard](docs/images/products/dashboard.png)
![Prowler Dashboard](docs/products/img/dashboard.png)
# Prowler at a Glance
> [!Tip]
> For the most accurate and up-to-date information about checks, services, frameworks, and categories, visit [**Prowler Hub**](https://hub.prowler.com).
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|---|---|---|---|---|---|---|
| AWS | 576 | 82 | 39 | 10 | Official | UI, API, CLI |
| GCP | 79 | 13 | 13 | 3 | Official | UI, API, CLI |
| Azure | 162 | 19 | 13 | 4 | Official | UI, API, CLI |
| Kubernetes | 83 | 7 | 5 | 7 | Official | UI, API, CLI |
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Stage | Interface |
|---|---|---|---|---|---|---|---|
| AWS | 576 | 82 | 38 | 10 | Official | Stable | UI, API, CLI |
| GCP | 79 | 13 | 11 | 3 | Official | Stable | UI, API, CLI |
| Azure | 162 | 19 | 12 | 4 | Official | Stable | UI, API, CLI |
| Kubernetes | 83 | 7 | 5 | 7 | Official | Stable | UI, API, CLI |
| GitHub | 17 | 2 | 1 | 0 | Official | Stable | UI, API, CLI |
| M365 | 70 | 7 | 3 | 2 | Official | UI, API, CLI |
| OCI | 51 | 13 | 1 | 10 | Official | UI, API, CLI |
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI |
| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | CLI, API |
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI |
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
| M365 | 70 | 7 | 3 | 2 | Official | Stable | UI, API, CLI |
| OCI | 51 | 13 | 1 | 10 | Official | Stable | UI, API, CLI |
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | Beta | CLI |
| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | Beta | CLI |
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | Beta | CLI |
| NHN | 6 | 2 | 1 | 0 | Unofficial | Beta | CLI |
> [!Note]
> The numbers in the table are updated periodically.
+8 -48
View File
@@ -2,63 +2,23 @@
All notable changes to the **Prowler API** are documented in this file.
## [1.15.1] (Prowler v5.14.1)
### Fixed
- Typo in PDF reporting [(#9322)](https://github.com/prowler-cloud/prowler/pull/9322)
- IaC provider initialization failure when mutelist processor is configured [(#9331)](https://github.com/prowler-cloud/prowler/pull/9331)
- Match logic for ThreatScore when counting findings [(#9348)](https://github.com/prowler-cloud/prowler/pull/9348)
---
## [1.15.0] (Prowler v5.14.0)
## [1.15.0] (Prowler UNRELEASED)
### Added
- IaC (Infrastructure as Code) provider support for remote repositories [(#8751)](https://github.com/prowler-cloud/prowler/pull/8751)
- Extend `GET /api/v1/providers` with provider-type filters and optional pagination disable to support the new Overview filters [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975)
- New endpoint to retrieve the number of providers grouped by provider type [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975)
- Support for configuring multiple LLM providers [(#8772)](https://github.com/prowler-cloud/prowler/pull/8772)
- Support C5 compliance framework for Azure provider [(#9081)](https://github.com/prowler-cloud/prowler/pull/9081)
- Support for Oracle Cloud Infrastructure (OCI) provider [(#8927)](https://github.com/prowler-cloud/prowler/pull/8927)
- Support muting findings based on simple rules with custom reason [(#9051)](https://github.com/prowler-cloud/prowler/pull/9051)
- Support C5 compliance framework for the GCP provider [(#9097)](https://github.com/prowler-cloud/prowler/pull/9097)
- Support for Amazon Bedrock and OpenAI compatible providers in Lighthouse AI [(#8957)](https://github.com/prowler-cloud/prowler/pull/8957)
- Support PDF reporting for ENS compliance framework [(#9158)](https://github.com/prowler-cloud/prowler/pull/9158)
- Support PDF reporting for NIS2 compliance framework [(#9170)](https://github.com/prowler-cloud/prowler/pull/9170)
- Tenant-wide ThreatScore overview aggregation and snapshot persistence with backfill support [(#9148)](https://github.com/prowler-cloud/prowler/pull/9148)
- Added `metadata`, `details`, and `partition` attributes to `/resources` endpoint & `details`, and `partition` to `/findings` endpoint [(#9098)](https://github.com/prowler-cloud/prowler/pull/9098)
- Support for MongoDB Atlas provider [(#9167)](https://github.com/prowler-cloud/prowler/pull/9167)
- Support Prowler ThreatScore for the K8S provider [(#9235)](https://github.com/prowler-cloud/prowler/pull/9235)
- Enhanced compliance overview endpoint with provider filtering and latest scan aggregation [(#9244)](https://github.com/prowler-cloud/prowler/pull/9244)
- New endpoint `GET /api/v1/overview/regions` to retrieve aggregated findings data by region [(#9273)](https://github.com/prowler-cloud/prowler/pull/9273)
### Changed
- Optimized database write queries for scan related tasks [(#9190)](https://github.com/prowler-cloud/prowler/pull/9190)
- Date filters are now optional for `GET /api/v1/overviews/services` endpoint; returns latest scan data by default [(#9248)](https://github.com/prowler-cloud/prowler/pull/9248)
### Fixed
- Scans no longer fail when findings have UIDs exceeding 300 characters; such findings are now skipped with detailed logging [(#9246)](https://github.com/prowler-cloud/prowler/pull/9246)
- Updated unique constraint for `Provider` model to exclude soft-deleted entries, resolving duplicate errors when re-deleting providers [(#9054)](https://github.com/prowler-cloud/prowler/pull/9054)
- Removed compliance generation for providers without compliance frameworks [(#9208)](https://github.com/prowler-cloud/prowler/pull/9208)
- Refresh output report timestamps for each scan [(#9272)](https://github.com/prowler-cloud/prowler/pull/9272)
- Severity overview endpoint now ignores muted findings as expected [(#9283)](https://github.com/prowler-cloud/prowler/pull/9283)
- Fixed discrepancy between ThreatScore PDF report values and database calculations [(#9296)](https://github.com/prowler-cloud/prowler/pull/9296)
### Security
- Django updated to the latest 5.1 security release, 5.1.14, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/113) and [denial-of-service vulnerability](https://github.com/prowler-cloud/prowler/security/dependabot/114) [(#9176)](https://github.com/prowler-cloud/prowler/pull/9176)
---
## [1.14.1] (Prowler v5.13.1)
## [1.14.1] (Prowler 5.13.1)
### Fixed
- `/api/v1/overviews/providers` collapses data by provider type so the UI receives a single aggregated record per cloud family even when multiple accounts exist [(#9053)](https://github.com/prowler-cloud/prowler/pull/9053)
- Added retry logic to database transactions to handle Aurora read replica connection failures during scale-down events [(#9064)](https://github.com/prowler-cloud/prowler/pull/9064)
- Security Hub integrations stop failing when they read relationships via the replica by allowing replica relations and saving updates through the primary [(#9080)](https://github.com/prowler-cloud/prowler/pull/9080)
---
## [1.14.0] (Prowler v5.13.0)
## [1.14.0] (Prowler 5.13.0)
### Added
- Default JWT keys are generated and stored if they are missing from configuration [(#8655)](https://github.com/prowler-cloud/prowler/pull/8655)
@@ -82,14 +42,14 @@ All notable changes to the **Prowler API** are documented in this file.
---
## [1.13.2] (Prowler v5.12.3)
## [1.13.2] (Prowler 5.12.3)
### Fixed
- 500 error when deleting user [(#8731)](https://github.com/prowler-cloud/prowler/pull/8731)
---
## [1.13.1] (Prowler v5.12.2)
## [1.13.1] (Prowler 5.12.2)
### Changed
- Renamed compliance overview task queue to `compliance` [(#8755)](https://github.com/prowler-cloud/prowler/pull/8755)
@@ -99,7 +59,7 @@ All notable changes to the **Prowler API** are documented in this file.
---
## [1.13.0] (Prowler v5.12.0)
## [1.13.0] (Prowler 5.12.0)
### Added
- Integration with JIRA, enabling sending findings to a JIRA project [(#8622)](https://github.com/prowler-cloud/prowler/pull/8622), [(#8637)](https://github.com/prowler-cloud/prowler/pull/8637)
@@ -108,7 +68,7 @@ All notable changes to the **Prowler API** are documented in this file.
---
## [1.12.0] (Prowler v5.11.0)
## [1.12.0] (Prowler 5.11.0)
### Added
- Lighthouse support for OpenAI GPT-5 [(#8527)](https://github.com/prowler-cloud/prowler/pull/8527)
@@ -120,7 +80,7 @@ All notable changes to the **Prowler API** are documented in this file.
---
## [1.11.0] (Prowler v5.10.0)
## [1.11.0] (Prowler 5.10.0)
### Added
- Github provider support [(#8271)](https://github.com/prowler-cloud/prowler/pull/8271)
-21
View File
@@ -5,9 +5,6 @@ LABEL maintainer="https://github.com/prowler-cloud/api"
ARG POWERSHELL_VERSION=7.5.0
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
ARG TRIVY_VERSION=0.66.0
ENV TRIVY_VERSION=${TRIVY_VERSION}
# hadolint ignore=DL3008
RUN apt-get update && apt-get install -y --no-install-recommends \
wget \
@@ -39,24 +36,6 @@ RUN ARCH=$(uname -m) && \
ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh && \
rm /tmp/powershell.tar.gz
# Install Trivy for IaC scanning
RUN ARCH=$(uname -m) && \
if [ "$ARCH" = "x86_64" ]; then \
TRIVY_ARCH="Linux-64bit" ; \
elif [ "$ARCH" = "aarch64" ]; then \
TRIVY_ARCH="Linux-ARM64" ; \
else \
echo "Unsupported architecture for Trivy: $ARCH" && exit 1 ; \
fi && \
wget --progress=dot:giga "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_${TRIVY_ARCH}.tar.gz" -O /tmp/trivy.tar.gz && \
tar zxf /tmp/trivy.tar.gz -C /tmp && \
mv /tmp/trivy /usr/local/bin/trivy && \
chmod +x /usr/local/bin/trivy && \
rm /tmp/trivy.tar.gz && \
# Create trivy cache directory with proper permissions
mkdir -p /tmp/.cache/trivy && \
chmod 777 /tmp/.cache/trivy
# Add prowler user
RUN addgroup --gid 1000 prowler && \
adduser --uid 1000 --gid 1000 --disabled-password --gecos "" prowler
+8 -291
View File
@@ -1,4 +1,4 @@
# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand.
# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand.
[[package]]
name = "about-time"
@@ -610,24 +610,6 @@ azure-mgmt-core = ">=1.3.2"
isodate = ">=0.6.1"
typing-extensions = ">=4.6.0"
[[package]]
name = "azure-mgmt-postgresqlflexibleservers"
version = "1.1.0"
description = "Microsoft Azure Postgresqlflexibleservers Management Client Library for Python"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "azure_mgmt_postgresqlflexibleservers-1.1.0-py3-none-any.whl", hash = "sha256:87ddb5a5e6d12c45769485d234cfe0322140e3a0a7636d0e61fb00ac544b5d20"},
{file = "azure_mgmt_postgresqlflexibleservers-1.1.0.tar.gz", hash = "sha256:9ede9d8ba63e9d2879cb74adc903c649af3bc5460a02787287b0cd18d754af14"},
]
[package.dependencies]
azure-common = ">=1.1"
azure-mgmt-core = ">=1.3.2"
isodate = ">=0.6.1"
typing-extensions = ">=4.6.0"
[[package]]
name = "azure-mgmt-rdbms"
version = "10.1.0"
@@ -1182,18 +1164,6 @@ files = [
{file = "charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14"},
]
[[package]]
name = "circuitbreaker"
version = "2.1.3"
description = "Python Circuit Breaker pattern implementation"
optional = false
python-versions = "*"
groups = ["main"]
files = [
{file = "circuitbreaker-2.1.3-py3-none-any.whl", hash = "sha256:87ba6a3ed03fdc7032bc175561c2b04d52ade9d5faf94ca2b035fbdc5e6b1dd1"},
{file = "circuitbreaker-2.1.3.tar.gz", hash = "sha256:1a4baee510f7bea3c91b194dcce7c07805fe96c4423ed5594b75af438531d084"},
]
[[package]]
name = "click"
version = "8.2.1"
@@ -1701,14 +1671,14 @@ with-social = ["django-allauth[socialaccount] (>=64.0.0)"]
[[package]]
name = "django"
version = "5.1.14"
version = "5.1.13"
description = "A high-level Python web framework that encourages rapid development and clean, pragmatic design."
optional = false
python-versions = ">=3.10"
groups = ["main", "dev"]
files = [
{file = "django-5.1.14-py3-none-any.whl", hash = "sha256:2a4b9c20404fd1bf50aaaa5542a19d860594cba1354f688f642feb271b91df27"},
{file = "django-5.1.14.tar.gz", hash = "sha256:b98409fb31fdd6e8c3a6ba2eef3415cc5c0020057b43b21ba7af6eff5f014831"},
{file = "django-5.1.13-py3-none-any.whl", hash = "sha256:06f257f79dc4c17f3f9e23b106a4c5ed1335abecbe731e83c598c941d14fbeed"},
{file = "django-5.1.13.tar.gz", hash = "sha256:543ff21679f15e80edfc01fe7ea35f8291b6d4ea589433882913626a7c1cf929"},
]
[package.dependencies]
@@ -2468,72 +2438,6 @@ files = [
{file = "frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f"},
]
[[package]]
name = "gevent"
version = "25.9.1"
description = "Coroutine-based network library"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "gevent-25.9.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:856b990be5590e44c3a3dc6c8d48a40eaccbb42e99d2b791d11d1e7711a4297e"},
{file = "gevent-25.9.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fe1599d0b30e6093eb3213551751b24feeb43db79f07e89d98dd2f3330c9063e"},
{file = "gevent-25.9.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:f0d8b64057b4bf1529b9ef9bd2259495747fba93d1f836c77bfeaacfec373fd0"},
{file = "gevent-25.9.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b56cbc820e3136ba52cd690bdf77e47a4c239964d5f80dc657c1068e0fe9521c"},
{file = "gevent-25.9.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c5fa9ce5122c085983e33e0dc058f81f5264cebe746de5c401654ab96dddfca8"},
{file = "gevent-25.9.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:03c74fec58eda4b4edc043311fca8ba4f8744ad1632eb0a41d5ec25413581975"},
{file = "gevent-25.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:a8ae9f895e8651d10b0a8328a61c9c53da11ea51b666388aa99b0ce90f9fdc27"},
{file = "gevent-25.9.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:18e5aff9e8342dc954adb9c9c524db56c2f3557999463445ba3d9cbe3dada7b7"},
{file = "gevent-25.9.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1cdf6db28f050ee103441caa8b0448ace545364f775059d5e2de089da975c457"},
{file = "gevent-25.9.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:812debe235a8295be3b2a63b136c2474241fa5c58af55e6a0f8cfc29d4936235"},
{file = "gevent-25.9.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b28b61ff9216a3d73fe8f35669eefcafa957f143ac534faf77e8a19eb9e6883a"},
{file = "gevent-25.9.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5e4b6278b37373306fc6b1e5f0f1cf56339a1377f67c35972775143d8d7776ff"},
{file = "gevent-25.9.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d99f0cb2ce43c2e8305bf75bee61a8bde06619d21b9d0316ea190fc7a0620a56"},
{file = "gevent-25.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:72152517ecf548e2f838c61b4be76637d99279dbaa7e01b3924df040aa996586"},
{file = "gevent-25.9.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:46b188248c84ffdec18a686fcac5dbb32365d76912e14fda350db5dc0bfd4f86"},
{file = "gevent-25.9.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f2b54ea3ca6f0c763281cd3f96010ac7e98c2e267feb1221b5a26e2ca0b9a692"},
{file = "gevent-25.9.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7a834804ac00ed8a92a69d3826342c677be651b1c3cd66cc35df8bc711057aa2"},
{file = "gevent-25.9.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:323a27192ec4da6b22a9e51c3d9d896ff20bc53fdc9e45e56eaab76d1c39dd74"},
{file = "gevent-25.9.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6ea78b39a2c51d47ff0f130f4c755a9a4bbb2dd9721149420ad4712743911a51"},
{file = "gevent-25.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:dc45cd3e1cc07514a419960af932a62eb8515552ed004e56755e4bf20bad30c5"},
{file = "gevent-25.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34e01e50c71eaf67e92c186ee0196a039d6e4f4b35670396baed4a2d8f1b347f"},
{file = "gevent-25.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:4acd6bcd5feabf22c7c5174bd3b9535ee9f088d2bbce789f740ad8d6554b18f3"},
{file = "gevent-25.9.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:4f84591d13845ee31c13f44bdf6bd6c3dbf385b5af98b2f25ec328213775f2ed"},
{file = "gevent-25.9.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9cdbb24c276a2d0110ad5c978e49daf620b153719ac8a548ce1250a7eb1b9245"},
{file = "gevent-25.9.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:88b6c07169468af631dcf0fdd3658f9246d6822cc51461d43f7c44f28b0abb82"},
{file = "gevent-25.9.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b7bb0e29a7b3e6ca9bed2394aa820244069982c36dc30b70eb1004dd67851a48"},
{file = "gevent-25.9.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2951bb070c0ee37b632ac9134e4fdaad70d2e660c931bb792983a0837fe5b7d7"},
{file = "gevent-25.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4e17c2d57e9a42e25f2a73d297b22b60b2470a74be5a515b36c984e1a246d47"},
{file = "gevent-25.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8d94936f8f8b23d9de2251798fcb603b84f083fdf0d7f427183c1828fb64f117"},
{file = "gevent-25.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:eb51c5f9537b07da673258b4832f6635014fee31690c3f0944d34741b69f92fa"},
{file = "gevent-25.9.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:1a3fe4ea1c312dbf6b375b416925036fe79a40054e6bf6248ee46526ea628be1"},
{file = "gevent-25.9.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0adb937f13e5fb90cca2edf66d8d7e99d62a299687400ce2edee3f3504009356"},
{file = "gevent-25.9.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:427f869a2050a4202d93cf7fd6ab5cffb06d3e9113c10c967b6e2a0d45237cb8"},
{file = "gevent-25.9.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c049880175e8c93124188f9d926af0a62826a3b81aa6d3074928345f8238279e"},
{file = "gevent-25.9.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b5a67a0974ad9f24721034d1e008856111e0535f1541499f72a733a73d658d1c"},
{file = "gevent-25.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1d0f5d8d73f97e24ea8d24d8be0f51e0cf7c54b8021c1fddb580bf239474690f"},
{file = "gevent-25.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddd3ff26e5c4240d3fbf5516c2d9d5f2a998ef87cfb73e1429cfaeaaec860fa6"},
{file = "gevent-25.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:bb63c0d6cb9950cc94036a4995b9cc4667b8915366613449236970f4394f94d7"},
{file = "gevent-25.9.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f18f80aef6b1f6907219affe15b36677904f7cfeed1f6a6bc198616e507ae2d7"},
{file = "gevent-25.9.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b274a53e818124a281540ebb4e7a2c524778f745b7a99b01bdecf0ca3ac0ddb0"},
{file = "gevent-25.9.1-cp39-cp39-win32.whl", hash = "sha256:c6c91f7e33c7f01237755884316110ee7ea076f5bdb9aa0982b6dc63243c0a38"},
{file = "gevent-25.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:012a44b0121f3d7c800740ff80351c897e85e76a7e4764690f35c5ad9ec17de5"},
{file = "gevent-25.9.1.tar.gz", hash = "sha256:adf9cd552de44a4e6754c51ff2e78d9193b7fa6eab123db9578a210e657235dd"},
]
[package.dependencies]
cffi = {version = ">=1.17.1", markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\""}
greenlet = {version = ">=3.2.2", markers = "platform_python_implementation == \"CPython\""}
"zope.event" = "*"
"zope.interface" = "*"
[package.extras]
dnspython = ["dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\""]
docs = ["furo", "repoze.sphinx.autointerface", "sphinx", "sphinxcontrib-programoutput", "zope.schema"]
monitor = ["psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""]
recommended = ["cffi (>=1.17.1) ; platform_python_implementation == \"CPython\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""]
test = ["cffi (>=1.17.1) ; platform_python_implementation == \"CPython\"", "coverage (>=5.0) ; sys_platform != \"win32\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "objgraph", "psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\"", "requests"]
[[package]]
name = "google-api-core"
version = "2.25.1"
@@ -2667,87 +2571,6 @@ files = [
dev = ["pytest"]
docs = ["sphinx", "sphinx-autobuild"]
[[package]]
name = "greenlet"
version = "3.2.4"
description = "Lightweight in-process concurrent programming"
optional = false
python-versions = ">=3.9"
groups = ["main"]
markers = "platform_python_implementation == \"CPython\""
files = [
{file = "greenlet-3.2.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:8c68325b0d0acf8d91dde4e6f930967dd52a5302cd4062932a6b2e7c2969f47c"},
{file = "greenlet-3.2.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:94385f101946790ae13da500603491f04a76b6e4c059dab271b3ce2e283b2590"},
{file = "greenlet-3.2.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f10fd42b5ee276335863712fa3da6608e93f70629c631bf77145021600abc23c"},
{file = "greenlet-3.2.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c8c9e331e58180d0d83c5b7999255721b725913ff6bc6cf39fa2a45841a4fd4b"},
{file = "greenlet-3.2.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58b97143c9cc7b86fc458f215bd0932f1757ce649e05b640fea2e79b54cedb31"},
{file = "greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d"},
{file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5"},
{file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8854167e06950ca75b898b104b63cc646573aa5fef1353d4508ecdd1ee76254f"},
{file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f47617f698838ba98f4ff4189aef02e7343952df3a615f847bb575c3feb177a7"},
{file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af41be48a4f60429d5cad9d22175217805098a9ef7c40bfef44f7669fb9d74d8"},
{file = "greenlet-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:73f49b5368b5359d04e18d15828eecc1806033db5233397748f4ca813ff1056c"},
{file = "greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2"},
{file = "greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246"},
{file = "greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3"},
{file = "greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633"},
{file = "greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079"},
{file = "greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8"},
{file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52"},
{file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa"},
{file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c"},
{file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5"},
{file = "greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9"},
{file = "greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd"},
{file = "greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb"},
{file = "greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968"},
{file = "greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9"},
{file = "greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6"},
{file = "greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0"},
{file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0"},
{file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f"},
{file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0"},
{file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d"},
{file = "greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02"},
{file = "greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31"},
{file = "greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945"},
{file = "greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc"},
{file = "greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a"},
{file = "greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504"},
{file = "greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671"},
{file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b"},
{file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae"},
{file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b"},
{file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929"},
{file = "greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b"},
{file = "greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0"},
{file = "greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f"},
{file = "greenlet-3.2.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c17b6b34111ea72fc5a4e4beec9711d2226285f0386ea83477cbb97c30a3f3a5"},
{file = "greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1"},
{file = "greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735"},
{file = "greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337"},
{file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269"},
{file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:015d48959d4add5d6c9f6c5210ee3803a830dce46356e3bc326d6776bde54681"},
{file = "greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01"},
{file = "greenlet-3.2.4-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:b6a7c19cf0d2742d0809a4c05975db036fdff50cd294a93632d6a310bf9ac02c"},
{file = "greenlet-3.2.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:27890167f55d2387576d1f41d9487ef171849ea0359ce1510ca6e06c8bece11d"},
{file = "greenlet-3.2.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:18d9260df2b5fbf41ae5139e1be4e796d99655f023a636cd0e11e6406cca7d58"},
{file = "greenlet-3.2.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:671df96c1f23c4a0d4077a325483c1503c96a1b7d9db26592ae770daa41233d4"},
{file = "greenlet-3.2.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:16458c245a38991aa19676900d48bd1a6f2ce3e16595051a4db9d012154e8433"},
{file = "greenlet-3.2.4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9913f1a30e4526f432991f89ae263459b1c64d1608c0d22a5c79c287b3c70df"},
{file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b90654e092f928f110e0007f572007c9727b5265f7632c2fa7415b4689351594"},
{file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81701fd84f26330f0d5f4944d4e92e61afe6319dcd9775e39396e39d7c3e5f98"},
{file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:28a3c6b7cd72a96f61b0e4b2a36f681025b60ae4779cc73c1535eb5f29560b10"},
{file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:52206cd642670b0b320a1fd1cbfd95bca0e043179c1d8a045f2c6109dfe973be"},
{file = "greenlet-3.2.4-cp39-cp39-win32.whl", hash = "sha256:65458b409c1ed459ea899e939f0e1cdb14f58dbc803f2f93c5eab5694d32671b"},
{file = "greenlet-3.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:d2e685ade4dafd447ede19c31277a224a239a0a1a4eca4e6390efedf20260cfb"},
{file = "greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d"},
]
[package.extras]
docs = ["Sphinx", "furo"]
test = ["objgraph", "psutil", "setuptools"]
[[package]]
name = "gunicorn"
version = "23.0.0"
@@ -4223,29 +4046,6 @@ rsa = ["cryptography (>=3.0.0)"]
signals = ["blinker (>=1.4.0)"]
signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
[[package]]
name = "oci"
version = "2.160.3"
description = "Oracle Cloud Infrastructure Python SDK"
optional = false
python-versions = "*"
groups = ["main"]
files = [
{file = "oci-2.160.3-py3-none-any.whl", hash = "sha256:858bff3e697098bdda44833d2476bfb4632126f0182178e7dbde4dbd156d71f0"},
{file = "oci-2.160.3.tar.gz", hash = "sha256:57514889be3b713a8385d86e3ba8a33cf46e3563c2a7e29a93027fb30b8a2537"},
]
[package.dependencies]
certifi = "*"
circuitbreaker = {version = ">=1.3.1,<3.0.0", markers = "python_version >= \"3.7\""}
cryptography = ">=3.2.1,<46.0.0"
pyOpenSSL = ">=17.5.0,<25.0.0"
python-dateutil = ">=2.5.3,<3.0.0"
pytz = ">=2016.10"
[package.extras]
adk = ["docstring-parser (>=0.16) ; python_version >= \"3.10\" and python_version < \"4\"", "mcp (>=1.6.0) ; python_version >= \"3.10\" and python_version < \"4\"", "pydantic (>=2.10.6) ; python_version >= \"3.10\" and python_version < \"4\"", "rich (>=13.9.4) ; python_version >= \"3.10\" and python_version < \"4\""]
[[package]]
name = "openai"
version = "1.101.0"
@@ -4780,7 +4580,7 @@ files = [
[[package]]
name = "prowler"
version = "5.14.0"
version = "5.13.0"
description = "Prowler is an Open Source security tool to perform AWS, GCP and Azure security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMP, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme) and your custom security frameworks."
optional = false
python-versions = ">3.9.1,<3.13"
@@ -4805,7 +4605,6 @@ azure-mgmt-keyvault = "10.3.1"
azure-mgmt-loganalytics = "12.0.0"
azure-mgmt-monitor = "6.0.2"
azure-mgmt-network = "28.1.0"
azure-mgmt-postgresqlflexibleservers = "1.1.0"
azure-mgmt-rdbms = "10.1.0"
azure-mgmt-recoveryservices = "3.1.0"
azure-mgmt-recoveryservicesbackup = "9.2.0"
@@ -4835,7 +4634,6 @@ markdown = "3.9.0"
microsoft-kiota-abstractions = "1.9.2"
msgraph-sdk = "1.23.0"
numpy = "2.0.2"
oci = "2.160.3"
pandas = "2.2.3"
py-iam-expand = "0.1.0"
py-ocsf-models = "0.5.0"
@@ -4852,8 +4650,8 @@ tzlocal = "5.3.1"
[package.source]
type = "git"
url = "https://github.com/prowler-cloud/prowler.git"
reference = "v5.14"
resolved_reference = "3b05a1430e016cee92b60973705cba400255d9e5"
reference = "master"
resolved_reference = "a52697bfdfee83d14a49c11dcbe96888b5cd767e"
[[package]]
name = "psutil"
@@ -5338,25 +5136,6 @@ cffi = ">=1.4.1"
docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"]
tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"]
[[package]]
name = "pyopenssl"
version = "24.3.0"
description = "Python wrapper module around the OpenSSL library"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "pyOpenSSL-24.3.0-py3-none-any.whl", hash = "sha256:e474f5a473cd7f92221cc04976e48f4d11502804657a08a989fb3be5514c904a"},
{file = "pyopenssl-24.3.0.tar.gz", hash = "sha256:49f7a019577d834746bc55c5fce6ecbcec0f2b4ec5ce1cf43a9a173b8138bb36"},
]
[package.dependencies]
cryptography = ">=41.0.5,<45"
[package.extras]
docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"]
test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"]
[[package]]
name = "pyparsing"
version = "3.2.3"
@@ -7004,69 +6783,7 @@ enabler = ["pytest-enabler (>=2.2)"]
test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"]
type = ["pytest-mypy"]
[[package]]
name = "zope-event"
version = "6.1"
description = "Very basic event publishing system"
optional = false
python-versions = ">=3.10"
groups = ["main"]
files = [
{file = "zope_event-6.1-py3-none-any.whl", hash = "sha256:0ca78b6391b694272b23ec1335c0294cc471065ed10f7f606858fc54566c25a0"},
{file = "zope_event-6.1.tar.gz", hash = "sha256:6052a3e0cb8565d3d4ef1a3a7809336ac519bc4fe38398cb8d466db09adef4f0"},
]
[package.extras]
docs = ["Sphinx"]
test = ["zope.testrunner (>=6.4)"]
[[package]]
name = "zope-interface"
version = "8.1.1"
description = "Interfaces for Python"
optional = false
python-versions = ">=3.10"
groups = ["main"]
files = [
{file = "zope_interface-8.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5c6b12b656c7d7e3d79cad8e2afc4a37eae6b6076e2c209a33345143148e435e"},
{file = "zope_interface-8.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:557c0f1363c300db406e9eeaae8ab6d1ba429d4fed60d8ab7dadab5ca66ccd35"},
{file = "zope_interface-8.1.1-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:127b0e4c873752b777721543cf8525b3db5e76b88bd33bab807f03c568e9003f"},
{file = "zope_interface-8.1.1-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e0892c9d2dd47b45f62d1861bcae8b427fcc49b4a04fff67f12c5c55e56654d7"},
{file = "zope_interface-8.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff8a92dc8c8a2c605074e464984e25b9b5a8ac9b2a0238dd73a0f374df59a77e"},
{file = "zope_interface-8.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:54627ddf6034aab1f506ba750dd093f67d353be6249467d720e9f278a578efe5"},
{file = "zope_interface-8.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e8a0fdd5048c1bb733e4693eae9bc4145a19419ea6a1c95299318a93fe9f3d72"},
{file = "zope_interface-8.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a4cb0ea75a26b606f5bc8524fbce7b7d8628161b6da002c80e6417ce5ec757c0"},
{file = "zope_interface-8.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c267b00b5a49a12743f5e1d3b4beef45479d696dab090f11fe3faded078a5133"},
{file = "zope_interface-8.1.1-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e25d3e2b9299e7ec54b626573673bdf0d740cf628c22aef0a3afef85b438aa54"},
{file = "zope_interface-8.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:63db1241804417aff95ac229c13376c8c12752b83cc06964d62581b493e6551b"},
{file = "zope_interface-8.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:9639bf4ed07b5277fb231e54109117c30d608254685e48a7104a34618bcbfc83"},
{file = "zope_interface-8.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a16715808408db7252b8c1597ed9008bdad7bf378ed48eb9b0595fad4170e49d"},
{file = "zope_interface-8.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce6b58752acc3352c4aa0b55bbeae2a941d61537e6afdad2467a624219025aae"},
{file = "zope_interface-8.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:807778883d07177713136479de7fd566f9056a13aef63b686f0ab4807c6be259"},
{file = "zope_interface-8.1.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50e5eb3b504a7d63dc25211b9298071d5b10a3eb754d6bf2f8ef06cb49f807ab"},
{file = "zope_interface-8.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eee6f93b2512ec9466cf30c37548fd3ed7bc4436ab29cd5943d7a0b561f14f0f"},
{file = "zope_interface-8.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:80edee6116d569883c58ff8efcecac3b737733d646802036dc337aa839a5f06b"},
{file = "zope_interface-8.1.1-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:84f9be6d959640de9da5d14ac1f6a89148b16da766e88db37ed17e936160b0b1"},
{file = "zope_interface-8.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:531fba91dcb97538f70cf4642a19d6574269460274e3f6004bba6fe684449c51"},
{file = "zope_interface-8.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:fc65f5633d5a9583ee8d88d1f5de6b46cd42c62e47757cfe86be36fb7c8c4c9b"},
{file = "zope_interface-8.1.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:efef80ddec4d7d99618ef71bc93b88859248075ca2e1ae1c78636654d3d55533"},
{file = "zope_interface-8.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49aad83525eca3b4747ef51117d302e891f0042b06f32aa1c7023c62642f962b"},
{file = "zope_interface-8.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:71cf329a21f98cb2bd9077340a589e316ac8a415cac900575a32544b3dffcb98"},
{file = "zope_interface-8.1.1-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:da311e9d253991ca327601f47c4644d72359bac6950fbb22f971b24cd7850f8c"},
{file = "zope_interface-8.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3fb25fca0442c7fb93c4ee40b42e3e033fef2f648730c4b7ae6d43222a3e8946"},
{file = "zope_interface-8.1.1-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:bac588d0742b4e35efb7c7df1dacc0397b51ed37a17d4169a38019a1cebacf0a"},
{file = "zope_interface-8.1.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d1f053d2d5e2b393e619bce1e55954885c2e63969159aa521839e719442db49"},
{file = "zope_interface-8.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:64a1ad7f4cb17d948c6bdc525a1d60c0e567b2526feb4fa38b38f249961306b8"},
{file = "zope_interface-8.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:169214da1b82b7695d1a36f92d70b11166d66b6b09d03df35d150cc62ac52276"},
{file = "zope_interface-8.1.1.tar.gz", hash = "sha256:51b10e6e8e238d719636a401f44f1e366146912407b58453936b781a19be19ec"},
]
[package.extras]
docs = ["Sphinx", "furo", "repoze.sphinx.autointerface"]
test = ["coverage[toml]", "zope.event", "zope.testing"]
testing = ["coverage[toml]", "zope.event", "zope.testing"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<3.13"
content-hash = "6dcdbbed2a46ab0111f4e32979fb7e5c7e3f6a80c4d293ac21b8c1f73c555204"
content-hash = "3c9164d668d37d6373eb5200bbe768232ead934d9312b9c68046b1df922789f3"
+4 -5
View File
@@ -7,7 +7,7 @@ authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}]
dependencies = [
"celery[pytest] (>=5.4.0,<6.0.0)",
"dj-rest-auth[with_social,jwt] (==7.0.1)",
"django (==5.1.14)",
"django (==5.1.13)",
"django-allauth[saml] (>=65.8.0,<66.0.0)",
"django-celery-beat (>=2.7.0,<3.0.0)",
"django-celery-results (>=2.5.1,<3.0.0)",
@@ -24,7 +24,7 @@ dependencies = [
"drf-spectacular-jsonapi==0.5.1",
"gunicorn==23.0.0",
"lxml==5.3.2",
"prowler @ git+https://github.com/prowler-cloud/prowler.git@v5.14",
"prowler @ git+https://github.com/prowler-cloud/prowler.git@master",
"psycopg2-binary==2.9.9",
"pytest-celery[redis] (>=1.0.1,<2.0.0)",
"sentry-sdk[django] (>=2.20.0,<3.0.0)",
@@ -35,8 +35,7 @@ dependencies = [
"markdown (>=3.9,<4.0)",
"drf-simple-apikey (==2.2.1)",
"matplotlib (>=3.10.6,<4.0.0)",
"reportlab (>=4.4.4,<5.0.0)",
"gevent (>=25.9.1,<26.0.0)"
"reportlab (>=4.4.4,<5.0.0)"
]
description = "Prowler's API (Django/DRF)"
license = "Apache-2.0"
@@ -44,7 +43,7 @@ name = "prowler-api"
package-mode = false
# Needed for the SDK compatibility
requires-python = ">=3.11,<3.13"
version = "1.15.1"
version = "1.15.0"
[project.scripts]
celery = "src.backend.config.settings.celery"
+25 -4
View File
@@ -9,6 +9,25 @@ PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = {}
PROWLER_CHECKS = {}
AVAILABLE_COMPLIANCE_FRAMEWORKS = {}
# Map API provider names to Prowler directory names
# This is needed because the OCI provider directory is 'oraclecloud' but the provider type is 'oci'
PROVIDER_NAME_MAPPING = {
"oci": "oraclecloud",
}
def get_prowler_provider_name(provider_type: str) -> str:
"""
Map API provider type to Prowler provider directory name.
Args:
provider_type: The provider type from the API (e.g., 'oci', 'aws', 'azure')
Returns:
The provider name used in Prowler's directory structure (e.g., 'oraclecloud', 'aws', 'azure')
"""
return PROVIDER_NAME_MAPPING.get(provider_type, provider_type)
def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]:
"""
@@ -28,8 +47,9 @@ def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[s
"""
global AVAILABLE_COMPLIANCE_FRAMEWORKS
if provider_type not in AVAILABLE_COMPLIANCE_FRAMEWORKS:
prowler_provider_name = get_prowler_provider_name(provider_type)
AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] = (
get_available_compliance_frameworks(provider_type)
get_available_compliance_frameworks(prowler_provider_name)
)
return AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type]
@@ -49,7 +69,8 @@ def get_prowler_provider_checks(provider_type: Provider.ProviderChoices):
Returns:
Iterable[str]: An iterable of check IDs associated with the specified provider type.
"""
return CheckMetadata.get_bulk(provider_type).keys()
prowler_provider_name = get_prowler_provider_name(provider_type)
return CheckMetadata.get_bulk(prowler_provider_name).keys()
def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) -> dict:
@@ -67,7 +88,8 @@ def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) ->
dict: A dictionary mapping compliance framework names to their respective
Compliance objects for the specified provider.
"""
return Compliance.get_bulk(provider_type)
prowler_provider_name = get_prowler_provider_name(provider_type)
return Compliance.get_bulk(prowler_provider_name)
def load_prowler_compliance():
@@ -144,7 +166,6 @@ def generate_scan_compliance(
Returns:
None: This function modifies the compliance_overview in place.
"""
for compliance_id in PROWLER_CHECKS[provider_type][check_id]:
for requirement in compliance_overview[compliance_id]["requirements"].values():
if check_id in requirement["checks"]:
+19 -66
View File
@@ -1,35 +1,18 @@
import re
import secrets
import time
import uuid
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from celery.utils.log import get_task_logger
from config.env import env
from django.conf import settings
from django.contrib.auth.models import BaseUserManager
from django.db import (
DEFAULT_DB_ALIAS,
OperationalError,
connection,
connections,
models,
transaction,
)
from django.db import DEFAULT_DB_ALIAS, connection, connections, models, transaction
from django_celery_beat.models import PeriodicTask
from psycopg2 import connect as psycopg2_connect
from psycopg2.extensions import AsIs, new_type, register_adapter, register_type
from rest_framework_json_api.serializers import ValidationError
from api.db_router import (
READ_REPLICA_ALIAS,
get_read_db_alias,
reset_read_db_alias,
set_read_db_alias,
)
logger = get_task_logger(__name__)
from api.db_router import get_read_db_alias, reset_read_db_alias, set_read_db_alias
DB_USER = settings.DATABASES["default"]["USER"] if not settings.TESTING else "test"
DB_PASSWORD = (
@@ -45,9 +28,6 @@ TASK_RUNNER_DB_TABLE = "django_celery_results_taskresult"
POSTGRES_TENANT_VAR = "api.tenant_id"
POSTGRES_USER_VAR = "api.user_id"
REPLICA_MAX_ATTEMPTS = env.int("POSTGRES_REPLICA_MAX_ATTEMPTS", default=3)
REPLICA_RETRY_BASE_DELAY = env.float("POSTGRES_REPLICA_RETRY_BASE_DELAY", default=0.5)
SET_CONFIG_QUERY = "SELECT set_config(%s, %s::text, TRUE);"
@@ -91,51 +71,24 @@ def rls_transaction(
if db_alias not in connections:
db_alias = DEFAULT_DB_ALIAS
alias = db_alias
is_replica = READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS
max_attempts = REPLICA_MAX_ATTEMPTS if is_replica else 1
router_token = None
try:
if db_alias != DEFAULT_DB_ALIAS:
router_token = set_read_db_alias(db_alias)
for attempt in range(1, max_attempts + 1):
router_token = None
# On final attempt, fallback to primary
if attempt == max_attempts and is_replica:
logger.warning(
f"RLS transaction failed after {attempt - 1} attempts on replica, "
f"falling back to primary DB"
)
alias = DEFAULT_DB_ALIAS
conn = connections[alias]
try:
if alias != DEFAULT_DB_ALIAS:
router_token = set_read_db_alias(alias)
with transaction.atomic(using=alias):
with conn.cursor() as cursor:
try:
# just in case the value is a UUID object
uuid.UUID(str(value))
except ValueError:
raise ValidationError("Must be a valid UUID")
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
yield cursor
return
except OperationalError as e:
# If on primary or max attempts reached, raise
if not is_replica or attempt == max_attempts:
raise
# Retry with exponential backoff
delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1))
logger.info(
f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), "
f"retrying in {delay}s. Error: {e}"
)
time.sleep(delay)
finally:
if router_token is not None:
reset_read_db_alias(router_token)
with transaction.atomic(using=db_alias):
conn = connections[db_alias]
with conn.cursor() as cursor:
try:
# just in case the value is a UUID object
uuid.UUID(str(value))
except ValueError:
raise ValidationError("Must be a valid UUID")
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
yield cursor
finally:
if router_token is not None:
reset_read_db_alias(router_token)
class CustomUserManager(BaseUserManager):
+24 -65
View File
@@ -30,7 +30,6 @@ from api.models import (
LighthouseProviderConfiguration,
LighthouseProviderModels,
Membership,
MuteRule,
OverviewStatusChoices,
PermissionChoices,
Processor,
@@ -47,7 +46,6 @@ from api.models import (
StatusChoices,
Task,
TenantAPIKey,
ThreatScoreSnapshot,
User,
)
from api.rls import Tenant
@@ -761,14 +759,6 @@ class RoleFilter(FilterSet):
class ComplianceOverviewFilter(FilterSet):
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
scan_id = UUIDFilter(field_name="scan_id")
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
provider_type = ChoiceFilter(
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
)
provider_type__in = ChoiceInFilter(
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
)
region = CharFilter(field_name="region")
class Meta:
@@ -820,8 +810,7 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
elif value == OverviewStatusChoices.PASS:
return queryset.annotate(status_count=F("_pass"))
else:
# Exclude muted findings by default
return queryset.annotate(status_count=F("_pass") + F("fail"))
return queryset.annotate(status_count=F("total"))
def filter_status_in(self, queryset, name, value):
# Validate the status values
@@ -830,7 +819,7 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
if status_val not in valid_statuses:
raise ValidationError(f"Invalid status value: {status_val}")
# If all statuses or no valid statuses, exclude muted findings (pass + fail)
# If all statuses or no valid statuses, use total
if (
set(value)
>= {
@@ -839,7 +828,7 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
}
or not value
):
return queryset.annotate(status_count=F("_pass") + F("fail"))
return queryset.annotate(status_count=F("total"))
# Build the sum expression based on status values
sum_expression = None
@@ -857,7 +846,7 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
sum_expression = sum_expression + field_expr
if sum_expression is None:
return queryset.annotate(status_count=F("_pass") + F("fail"))
return queryset.annotate(status_count=F("total"))
return queryset.annotate(status_count=sum_expression)
@@ -869,6 +858,26 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
}
class ServiceOverviewFilter(ScanSummaryFilter):
def is_valid(self):
# Check if at least one of the inserted_at filters is present
inserted_at_filters = [
self.data.get("inserted_at"),
self.data.get("inserted_at__gte"),
self.data.get("inserted_at__lte"),
]
if not any(inserted_at_filters):
raise ValidationError(
{
"inserted_at": [
"At least one of filter[inserted_at], filter[inserted_at__gte], or "
"filter[inserted_at__lte] is required."
]
}
)
return super().is_valid()
class IntegrationFilter(FilterSet):
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
integration_type = ChoiceFilter(choices=Integration.IntegrationChoices.choices)
@@ -971,53 +980,3 @@ class LighthouseProviderModelsFilter(FilterSet):
fields = {
"model_id": ["exact", "icontains", "in"],
}
class MuteRuleFilter(FilterSet):
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
created_by = UUIDFilter(field_name="created_by__id", lookup_expr="exact")
class Meta:
model = MuteRule
fields = {
"id": ["exact", "in"],
"name": ["exact", "icontains"],
"reason": ["icontains"],
"enabled": ["exact"],
"inserted_at": ["gte", "lte"],
"updated_at": ["gte", "lte"],
}
class ThreatScoreSnapshotFilter(FilterSet):
"""
Filter for ThreatScore snapshots.
Allows filtering by scan, provider, compliance_id, and date ranges.
"""
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
scan_id = UUIDFilter(field_name="scan__id", lookup_expr="exact")
scan_id__in = UUIDInFilter(field_name="scan__id", lookup_expr="in")
provider_id = UUIDFilter(field_name="provider__id", lookup_expr="exact")
provider_id__in = UUIDInFilter(field_name="provider__id", lookup_expr="in")
provider_type = ChoiceFilter(
field_name="provider__provider", choices=Provider.ProviderChoices.choices
)
provider_type__in = ChoiceInFilter(
field_name="provider__provider",
choices=Provider.ProviderChoices.choices,
lookup_expr="in",
)
compliance_id = CharFilter(field_name="compliance_id", lookup_expr="exact")
compliance_id__in = CharInFilter(field_name="compliance_id", lookup_expr="in")
class Meta:
model = ThreatScoreSnapshot
fields = {
"scan": ["exact", "in"],
"provider": ["exact", "in"],
"compliance_id": ["exact", "in"],
"inserted_at": ["date", "gte", "lte"],
"overall_score": ["exact", "gte", "lte"],
}
@@ -22,13 +22,13 @@ class Migration(migrations.Migration):
("kubernetes", "Kubernetes"),
("m365", "M365"),
("github", "GitHub"),
("oraclecloud", "Oracle Cloud Infrastructure"),
("oci", "Oracle Cloud Infrastructure"),
],
default="aws",
),
),
migrations.RunSQL(
"ALTER TYPE provider ADD VALUE IF NOT EXISTS 'oraclecloud';",
"ALTER TYPE provider ADD VALUE IF NOT EXISTS 'oci';",
reverse_sql=migrations.RunSQL.noop,
),
]
@@ -1,117 +0,0 @@
# Generated by Django 5.1.13 on 2025-10-22 11:56
import uuid
import django.contrib.postgres.fields
import django.core.validators
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
import api.rls
class Migration(migrations.Migration):
dependencies = [
("api", "0051_oraclecloud_provider"),
]
operations = [
migrations.CreateModel(
name="MuteRule",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("inserted_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"name",
models.CharField(
help_text="Human-readable name for this rule",
max_length=100,
validators=[django.core.validators.MinLengthValidator(3)],
),
),
(
"reason",
models.TextField(
help_text="Reason for muting",
max_length=500,
validators=[django.core.validators.MinLengthValidator(3)],
),
),
(
"enabled",
models.BooleanField(
default=True, help_text="Whether this rule is currently enabled"
),
),
(
"finding_uids",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=255),
help_text="List of finding UIDs to mute",
size=None,
),
),
],
options={
"db_table": "mute_rules",
"abstract": False,
},
),
migrations.AddField(
model_name="finding",
name="muted_at",
field=models.DateTimeField(
blank=True, help_text="Timestamp when this finding was muted", null=True
),
),
migrations.AlterField(
model_name="tenantapikey",
name="name",
field=models.CharField(
max_length=100,
validators=[django.core.validators.MinLengthValidator(3)],
),
),
migrations.AddField(
model_name="muterule",
name="created_by",
field=models.ForeignKey(
help_text="User who created this rule",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="created_mute_rules",
to=settings.AUTH_USER_MODEL,
),
),
migrations.AddField(
model_name="muterule",
name="tenant",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
),
),
migrations.AddConstraint(
model_name="muterule",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_muterule",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
migrations.AddConstraint(
model_name="muterule",
constraint=models.UniqueConstraint(
fields=("tenant_id", "name"), name="unique_mute_rule_name_per_tenant"
),
),
]
@@ -1,25 +0,0 @@
# Generated by Django 5.1.12 on 2025-10-14 11:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("api", "0052_mute_rules"),
]
operations = [
migrations.AlterField(
model_name="lighthouseproviderconfiguration",
name="provider_type",
field=models.CharField(
choices=[
("openai", "OpenAI"),
("bedrock", "AWS Bedrock"),
("openai_compatible", "OpenAI Compatible"),
],
help_text="LLM provider name",
max_length=50,
),
)
]
@@ -1,35 +0,0 @@
# Generated by Django 5.1.10 on 2025-09-09 09:25
from django.db import migrations
import api.db_utils
class Migration(migrations.Migration):
dependencies = [
("api", "0053_lighthouse_bedrock_openai_compatible"),
]
operations = [
migrations.AlterField(
model_name="provider",
name="provider",
field=api.db_utils.ProviderEnumField(
choices=[
("aws", "AWS"),
("azure", "Azure"),
("gcp", "GCP"),
("kubernetes", "Kubernetes"),
("m365", "M365"),
("github", "GitHub"),
("oraclecloud", "Oracle Cloud Infrastructure"),
("iac", "IaC"),
],
default="aws",
),
),
migrations.RunSQL(
"ALTER TYPE provider ADD VALUE IF NOT EXISTS 'iac';",
reverse_sql=migrations.RunSQL.noop,
),
]
@@ -1,36 +0,0 @@
# Generated by Django 5.1.13 on 2025-11-05 08:37
from django.db import migrations
import api.db_utils
class Migration(migrations.Migration):
dependencies = [
("api", "0054_iac_provider"),
]
operations = [
migrations.AlterField(
model_name="provider",
name="provider",
field=api.db_utils.ProviderEnumField(
choices=[
("aws", "AWS"),
("azure", "Azure"),
("gcp", "GCP"),
("kubernetes", "Kubernetes"),
("m365", "M365"),
("github", "GitHub"),
("mongodbatlas", "MongoDB Atlas"),
("iac", "IaC"),
("oraclecloud", "Oracle Cloud Infrastructure"),
],
default="aws",
),
),
migrations.RunSQL(
"ALTER TYPE provider ADD VALUE IF NOT EXISTS 'mongodbatlas';",
reverse_sql=migrations.RunSQL.noop,
),
]
@@ -1,24 +0,0 @@
# Generated by Django 5.1.13 on 2025-11-06 09:20
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("api", "0055_mongodbatlas_provider"),
]
operations = [
migrations.RemoveConstraint(
model_name="provider",
name="unique_provider_uids",
),
migrations.AddConstraint(
model_name="provider",
constraint=models.UniqueConstraint(
condition=models.Q(("is_deleted", False)),
fields=("tenant_id", "provider", "uid"),
name="unique_provider_uids",
),
),
]
@@ -1,170 +0,0 @@
# Generated by Django 5.1.13 on 2025-10-31 09:04
import uuid
import django.db.models.deletion
from django.db import migrations, models
import api.rls
class Migration(migrations.Migration):
dependencies = [
("api", "0056_remove_provider_unique_provider_uids_and_more"),
]
operations = [
migrations.CreateModel(
name="ThreatScoreSnapshot",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("inserted_at", models.DateTimeField(auto_now_add=True)),
(
"compliance_id",
models.CharField(
help_text="Compliance framework ID (e.g., 'prowler_threatscore_aws')",
max_length=100,
),
),
(
"overall_score",
models.DecimalField(
decimal_places=2,
help_text="Overall ThreatScore percentage (0-100)",
max_digits=5,
),
),
(
"score_delta",
models.DecimalField(
blank=True,
decimal_places=2,
help_text="Score change compared to previous snapshot (positive = improvement)",
max_digits=5,
null=True,
),
),
(
"section_scores",
models.JSONField(
blank=True,
default=dict,
help_text="ThreatScore breakdown by section",
),
),
(
"critical_requirements",
models.JSONField(
blank=True,
default=list,
help_text="List of critical failed requirements (risk >= 4)",
),
),
(
"total_requirements",
models.IntegerField(
default=0, help_text="Total number of requirements evaluated"
),
),
(
"passed_requirements",
models.IntegerField(
default=0, help_text="Number of requirements with PASS status"
),
),
(
"failed_requirements",
models.IntegerField(
default=0, help_text="Number of requirements with FAIL status"
),
),
(
"manual_requirements",
models.IntegerField(
default=0, help_text="Number of requirements with MANUAL status"
),
),
(
"total_findings",
models.IntegerField(
default=0,
help_text="Total number of findings across all requirements",
),
),
(
"passed_findings",
models.IntegerField(
default=0, help_text="Number of findings with PASS status"
),
),
(
"failed_findings",
models.IntegerField(
default=0, help_text="Number of findings with FAIL status"
),
),
(
"provider",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="threatscore_snapshots",
related_query_name="threatscore_snapshot",
to="api.provider",
),
),
(
"scan",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="threatscore_snapshots",
related_query_name="threatscore_snapshot",
to="api.scan",
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
),
),
],
options={
"db_table": "threatscore_snapshots",
"abstract": False,
},
),
migrations.AddIndex(
model_name="threatscoresnapshot",
index=models.Index(
fields=["tenant_id", "scan_id"], name="threatscore_snap_t_scan_idx"
),
),
migrations.AddIndex(
model_name="threatscoresnapshot",
index=models.Index(
fields=["tenant_id", "provider_id"], name="threatscore_snap_t_prov_idx"
),
),
migrations.AddIndex(
model_name="threatscoresnapshot",
index=models.Index(
fields=["tenant_id", "inserted_at"], name="threatscore_snap_t_time_idx"
),
),
migrations.AddConstraint(
model_name="threatscoresnapshot",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_threatscoresnapshot",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
]
@@ -1,29 +0,0 @@
from django.contrib.postgres.operations import RemoveIndexConcurrently
from django.db import migrations
class Migration(migrations.Migration):
atomic = False
dependencies = [
("api", "0057_threatscoresnapshot"),
]
operations = [
RemoveIndexConcurrently(
model_name="compliancerequirementoverview",
name="cro_tenant_scan_idx",
),
RemoveIndexConcurrently(
model_name="compliancerequirementoverview",
name="cro_scan_comp_idx",
),
RemoveIndexConcurrently(
model_name="compliancerequirementoverview",
name="cro_scan_comp_req_idx",
),
RemoveIndexConcurrently(
model_name="compliancerequirementoverview",
name="cro_scan_comp_req_reg_idx",
),
]
@@ -1,75 +0,0 @@
# Generated by Django 5.1.13 on 2025-10-30 15:23
import uuid
import django.db.models.deletion
from django.db import migrations, models
import api.rls
class Migration(migrations.Migration):
dependencies = [
("api", "0058_drop_redundant_compliance_requirement_indexes"),
]
operations = [
migrations.CreateModel(
name="ComplianceOverviewSummary",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("inserted_at", models.DateTimeField(auto_now_add=True)),
("compliance_id", models.TextField()),
("requirements_passed", models.IntegerField(default=0)),
("requirements_failed", models.IntegerField(default=0)),
("requirements_manual", models.IntegerField(default=0)),
("total_requirements", models.IntegerField(default=0)),
(
"scan",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="compliance_summaries",
related_query_name="compliance_summary",
to="api.scan",
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
),
),
],
options={
"db_table": "compliance_overview_summaries",
"abstract": False,
"indexes": [
models.Index(
fields=["tenant_id", "scan_id"], name="cos_tenant_scan_idx"
)
],
"constraints": [
models.UniqueConstraint(
fields=("tenant_id", "scan_id", "compliance_id"),
name="unique_compliance_summary_per_scan",
)
],
},
),
migrations.AddConstraint(
model_name="complianceoverviewsummary",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_complianceoverviewsummary",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
]
+23 -272
View File
@@ -284,9 +284,7 @@ class Provider(RowLevelSecurityProtectedModel):
KUBERNETES = "kubernetes", _("Kubernetes")
M365 = "m365", _("M365")
GITHUB = "github", _("GitHub")
MONGODBATLAS = "mongodbatlas", _("MongoDB Atlas")
IAC = "iac", _("IaC")
ORACLECLOUD = "oraclecloud", _("Oracle Cloud Infrastructure")
OCI = "oci", _("Oracle Cloud Infrastructure")
@staticmethod
def validate_aws_uid(value):
@@ -358,36 +356,14 @@ class Provider(RowLevelSecurityProtectedModel):
)
@staticmethod
def validate_iac_uid(value):
# Validate that it's a valid repository URL (git URL format)
if not re.match(
r"^(https?://|git@|ssh://)[^\s/]+[^\s]*\.git$|^(https?://)[^\s/]+[^\s]*$",
value,
):
raise ModelValidationError(
detail="IaC provider ID must be a valid repository URL (e.g., https://github.com/user/repo or https://github.com/user/repo.git).",
code="iac-uid",
pointer="/data/attributes/uid",
)
@staticmethod
def validate_oraclecloud_uid(value):
def validate_oci_uid(value):
if not re.match(
r"^ocid1\.([a-z0-9_-]+)\.([a-z0-9_-]+)\.([a-z0-9_-]*)\.([a-z0-9]+)$", value
):
raise ModelValidationError(
detail="Oracle Cloud Infrastructure provider ID must be a valid tenancy OCID in the format: "
"ocid1.<resource_type>.<realm>.<region>.<unique_id>",
code="oraclecloud-uid",
pointer="/data/attributes/uid",
)
@staticmethod
def validate_mongodbatlas_uid(value):
if not re.match(r"^[0-9a-fA-F]{24}$", value):
raise ModelValidationError(
detail="MongoDB Atlas organization ID must be a 24-character hexadecimal string.",
code="mongodbatlas-uid",
code="oci-uid",
pointer="/data/attributes/uid",
)
@@ -425,8 +401,7 @@ class Provider(RowLevelSecurityProtectedModel):
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "provider", "uid"),
condition=Q(is_deleted=False),
fields=("tenant_id", "provider", "uid", "is_deleted"),
name="unique_provider_uids",
),
RowLevelSecurityConstraint(
@@ -848,9 +823,6 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
muted_reason = models.TextField(
blank=True, null=True, validators=[MinLengthValidator(3)], max_length=500
)
muted_at = models.DateTimeField(
null=True, blank=True, help_text="Timestamp when this finding was muted"
)
compliance = models.JSONField(default=dict, null=True, blank=True)
# Denormalize resource data for performance
@@ -1371,70 +1343,35 @@ class ComplianceRequirementOverview(RowLevelSecurityProtectedModel):
),
]
indexes = [
models.Index(fields=["tenant_id", "scan_id"], name="cro_tenant_scan_idx"),
models.Index(
fields=["tenant_id", "scan_id", "compliance_id"],
name="cro_scan_comp_idx",
),
models.Index(
fields=["tenant_id", "scan_id", "compliance_id", "region"],
name="cro_scan_comp_reg_idx",
),
models.Index(
fields=["tenant_id", "scan_id", "compliance_id", "requirement_id"],
name="cro_scan_comp_req_idx",
),
models.Index(
fields=[
"tenant_id",
"scan_id",
"compliance_id",
"requirement_id",
"region",
],
name="cro_scan_comp_req_reg_idx",
),
]
class JSONAPIMeta:
resource_name = "compliance-requirements-overviews"
class ComplianceOverviewSummary(RowLevelSecurityProtectedModel):
"""
Pre-aggregated compliance overview aggregated across ALL regions.
One row per (scan_id, compliance_id) combination.
This table optimizes the common case where users view overall compliance
without filtering by region. For region-specific views, the detailed
ComplianceRequirementOverview table is used instead.
"""
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
scan = models.ForeignKey(
Scan,
on_delete=models.CASCADE,
related_name="compliance_summaries",
related_query_name="compliance_summary",
)
compliance_id = models.TextField(blank=False)
# Pre-aggregated scores (computed across ALL regions)
requirements_passed = models.IntegerField(default=0)
requirements_failed = models.IntegerField(default=0)
requirements_manual = models.IntegerField(default=0)
total_requirements = models.IntegerField(default=0)
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "compliance_overview_summaries"
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "scan_id", "compliance_id"),
name="unique_compliance_summary_per_scan",
),
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "DELETE"],
),
]
indexes = [
models.Index(
fields=["tenant_id", "scan_id"],
name="cos_tenant_scan_idx",
),
]
class JSONAPIMeta:
resource_name = "compliance-overview-summaries"
class ScanSummary(RowLevelSecurityProtectedModel):
objects = ActiveProviderManager()
all_objects = models.Manager()
@@ -1998,59 +1935,6 @@ class LighthouseConfiguration(RowLevelSecurityProtectedModel):
resource_name = "lighthouse-configurations"
class MuteRule(RowLevelSecurityProtectedModel):
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
# Rule metadata
name = models.CharField(
max_length=100,
validators=[MinLengthValidator(3)],
help_text="Human-readable name for this rule",
)
reason = models.TextField(
validators=[MinLengthValidator(3)],
max_length=500,
help_text="Reason for muting",
)
enabled = models.BooleanField(
default=True, help_text="Whether this rule is currently enabled"
)
# Audit fields
created_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
null=True,
related_name="created_mute_rules",
help_text="User who created this rule",
)
# Rule criteria - array of finding UIDs
finding_uids = ArrayField(
models.CharField(max_length=255), help_text="List of finding UIDs to mute"
)
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "mute_rules"
constraints = [
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
models.UniqueConstraint(
fields=("tenant_id", "name"),
name="unique_mute_rule_name_per_tenant",
),
]
class JSONAPIMeta:
resource_name = "mute-rules"
class Processor(RowLevelSecurityProtectedModel):
class ProcessorChoices(models.TextChoices):
MUTELIST = "mutelist", _("Mutelist")
@@ -2099,8 +1983,6 @@ class LighthouseProviderConfiguration(RowLevelSecurityProtectedModel):
class LLMProviderChoices(models.TextChoices):
OPENAI = "openai", _("OpenAI")
BEDROCK = "bedrock", _("AWS Bedrock")
OPENAI_COMPATIBLE = "openai_compatible", _("OpenAI Compatible")
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
@@ -2274,134 +2156,3 @@ class LighthouseProviderModels(RowLevelSecurityProtectedModel):
class JSONAPIMeta:
resource_name = "lighthouse-models"
class ThreatScoreSnapshot(RowLevelSecurityProtectedModel):
"""
Stores historical ThreatScore metrics for a given scan.
Snapshots are created automatically after each ThreatScore report generation.
"""
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
scan = models.ForeignKey(
Scan,
on_delete=models.CASCADE,
related_name="threatscore_snapshots",
related_query_name="threatscore_snapshot",
)
provider = models.ForeignKey(
Provider,
on_delete=models.CASCADE,
related_name="threatscore_snapshots",
related_query_name="threatscore_snapshot",
)
compliance_id = models.CharField(
max_length=100,
blank=False,
null=False,
help_text="Compliance framework ID (e.g., 'prowler_threatscore_aws')",
)
# Overall ThreatScore metrics
overall_score = models.DecimalField(
max_digits=5,
decimal_places=2,
help_text="Overall ThreatScore percentage (0-100)",
)
# Score improvement/degradation compared to previous snapshot
score_delta = models.DecimalField(
max_digits=5,
decimal_places=2,
null=True,
blank=True,
help_text="Score change compared to previous snapshot (positive = improvement)",
)
# Section breakdown stored as JSON
# Format: {"1. IAM": 85.5, "2. Attack Surface": 92.3, ...}
section_scores = models.JSONField(
default=dict,
blank=True,
help_text="ThreatScore breakdown by section",
)
# Critical requirements metadata stored as JSON
# Format: [{"requirement_id": "...", "risk_level": 5, "weight": 150, ...}, ...]
critical_requirements = models.JSONField(
default=list,
blank=True,
help_text="List of critical failed requirements (risk >= 4)",
)
# Summary statistics
total_requirements = models.IntegerField(
default=0,
help_text="Total number of requirements evaluated",
)
passed_requirements = models.IntegerField(
default=0,
help_text="Number of requirements with PASS status",
)
failed_requirements = models.IntegerField(
default=0,
help_text="Number of requirements with FAIL status",
)
manual_requirements = models.IntegerField(
default=0,
help_text="Number of requirements with MANUAL status",
)
total_findings = models.IntegerField(
default=0,
help_text="Total number of findings across all requirements",
)
passed_findings = models.IntegerField(
default=0,
help_text="Number of findings with PASS status",
)
failed_findings = models.IntegerField(
default=0,
help_text="Number of findings with FAIL status",
)
def __str__(self):
return f"ThreatScore {self.overall_score}% for scan {self.scan_id} ({self.inserted_at})"
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "threatscore_snapshots"
constraints = [
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
indexes = [
models.Index(
fields=["tenant_id", "scan_id"],
name="threatscore_snap_t_scan_idx",
),
models.Index(
fields=["tenant_id", "provider_id"],
name="threatscore_snap_t_prov_idx",
),
models.Index(
fields=["tenant_id", "inserted_at"],
name="threatscore_snap_t_time_idx",
),
]
class JSONAPIMeta:
resource_name = "threatscore-snapshots"
File diff suppressed because it is too large Load Diff
@@ -1,39 +0,0 @@
"""Tests for rls_transaction retry and fallback logic."""
import pytest
from django.db import DEFAULT_DB_ALIAS
from rest_framework_json_api.serializers import ValidationError
from api.db_utils import rls_transaction
@pytest.mark.django_db
class TestRLSTransaction:
"""Simple integration tests for rls_transaction using real DB."""
@pytest.fixture
def tenant(self, tenants_fixture):
return tenants_fixture[0]
def test_success_on_primary(self, tenant):
"""Basic: transaction succeeds on primary database."""
with rls_transaction(str(tenant.id), using=DEFAULT_DB_ALIAS) as cursor:
cursor.execute("SELECT 1")
result = cursor.fetchone()
assert result == (1,)
def test_invalid_uuid_raises_validation_error(self):
"""Invalid UUID raises ValidationError before DB operations."""
with pytest.raises(ValidationError, match="Must be a valid UUID"):
with rls_transaction("not-a-uuid", using=DEFAULT_DB_ALIAS):
pass
def test_custom_parameter_name(self, tenant):
"""Test custom RLS parameter name."""
custom_param = "api.custom_id"
with rls_transaction(
str(tenant.id), parameter=custom_param, using=DEFAULT_DB_ALIAS
) as cursor:
cursor.execute("SELECT current_setting(%s, true)", [custom_param])
result = cursor.fetchone()
assert result == (str(tenant.id),)
+1 -510
View File
@@ -1,15 +1,12 @@
from datetime import datetime, timezone
from enum import Enum
from unittest.mock import MagicMock, patch
from unittest.mock import patch
import pytest
from django.conf import settings
from django.db import DEFAULT_DB_ALIAS, OperationalError
from freezegun import freeze_time
from rest_framework_json_api.serializers import ValidationError
from api.db_utils import (
POSTGRES_TENANT_VAR,
_should_create_index_on_partition,
batch_delete,
create_objects_in_batches,
@@ -17,22 +14,11 @@ from api.db_utils import (
generate_api_key_prefix,
generate_random_token,
one_week_from_now,
rls_transaction,
update_objects_in_batches,
)
from api.models import Provider
@pytest.fixture
def enable_read_replica():
"""
Fixture to enable READ_REPLICA_ALIAS for tests that need replica functionality.
This avoids polluting the global test configuration.
"""
with patch("api.db_utils.READ_REPLICA_ALIAS", "replica"):
yield "replica"
class TestEnumToChoices:
def test_enum_to_choices_simple(self):
class Color(Enum):
@@ -353,498 +339,3 @@ class TestGenerateApiKeyPrefix:
prefix = generate_api_key_prefix()
random_part = prefix[3:] # Strip 'pk_'
assert all(char in allowed_chars for char in random_part)
@pytest.mark.django_db
class TestRlsTransaction:
def test_rls_transaction_valid_uuid_string(self, tenants_fixture):
"""Test rls_transaction with valid UUID string."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with rls_transaction(tenant_id) as cursor:
assert cursor is not None
cursor.execute("SELECT current_setting(%s)", [POSTGRES_TENANT_VAR])
result = cursor.fetchone()
assert result[0] == tenant_id
def test_rls_transaction_valid_uuid_object(self, tenants_fixture):
"""Test rls_transaction with UUID object."""
tenant = tenants_fixture[0]
with rls_transaction(tenant.id) as cursor:
assert cursor is not None
cursor.execute("SELECT current_setting(%s)", [POSTGRES_TENANT_VAR])
result = cursor.fetchone()
assert result[0] == str(tenant.id)
def test_rls_transaction_invalid_uuid_raises_validation_error(self):
"""Test rls_transaction raises ValidationError for invalid UUID."""
invalid_uuid = "not-a-valid-uuid"
with pytest.raises(ValidationError, match="Must be a valid UUID"):
with rls_transaction(invalid_uuid):
pass
def test_rls_transaction_uses_default_database_when_no_alias(self, tenants_fixture):
"""Test rls_transaction uses DEFAULT_DB_ALIAS when no alias specified."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=None):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic"):
with rls_transaction(tenant_id):
pass
mock_connections.__getitem__.assert_called_with(DEFAULT_DB_ALIAS)
def test_rls_transaction_uses_specified_alias(self, tenants_fixture):
"""Test rls_transaction uses specified database alias via using parameter."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
custom_alias = "custom_db"
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic"):
with patch("api.db_utils.set_read_db_alias") as mock_set_alias:
with patch("api.db_utils.reset_read_db_alias") as mock_reset_alias:
mock_set_alias.return_value = "test_token"
with rls_transaction(tenant_id, using=custom_alias):
pass
mock_connections.__getitem__.assert_called_with(custom_alias)
mock_set_alias.assert_called_once_with(custom_alias)
mock_reset_alias.assert_called_once_with("test_token")
def test_rls_transaction_uses_read_replica_from_router(
self, tenants_fixture, enable_read_replica
):
"""Test rls_transaction uses read replica alias from router."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic"):
with patch("api.db_utils.set_read_db_alias") as mock_set_alias:
with patch(
"api.db_utils.reset_read_db_alias"
) as mock_reset_alias:
mock_set_alias.return_value = "test_token"
with rls_transaction(tenant_id):
pass
mock_connections.__getitem__.assert_called()
mock_set_alias.assert_called_once()
mock_reset_alias.assert_called_once()
def test_rls_transaction_fallback_to_default_when_alias_not_in_connections(
self, tenants_fixture
):
"""Test rls_transaction falls back to DEFAULT_DB_ALIAS when alias not in connections."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
invalid_alias = "nonexistent_db"
with patch("api.db_utils.get_read_db_alias", return_value=invalid_alias):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
def contains_check(alias):
return alias == DEFAULT_DB_ALIAS
mock_connections.__contains__.side_effect = contains_check
mock_connections.__getitem__.return_value = mock_conn
with patch("api.db_utils.transaction.atomic"):
with rls_transaction(tenant_id):
pass
mock_connections.__getitem__.assert_called_with(DEFAULT_DB_ALIAS)
def test_rls_transaction_successful_execution_on_replica_no_retries(
self, tenants_fixture, enable_read_replica
):
"""Test successful execution on replica without retries."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic"):
with patch("api.db_utils.set_read_db_alias", return_value="token"):
with patch("api.db_utils.reset_read_db_alias"):
with rls_transaction(tenant_id):
pass
assert mock_cursor.execute.call_count == 1
def test_rls_transaction_retry_with_exponential_backoff_on_operational_error(
self, tenants_fixture, enable_read_replica
):
"""Test retry with exponential backoff on OperationalError on replica."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
call_count = 0
def atomic_side_effect(*args, **kwargs):
nonlocal call_count
call_count += 1
if call_count < 3:
raise OperationalError("Connection error")
return MagicMock(
__enter__=MagicMock(return_value=None),
__exit__=MagicMock(return_value=False),
)
with patch(
"api.db_utils.transaction.atomic", side_effect=atomic_side_effect
):
with patch("api.db_utils.time.sleep") as mock_sleep:
with patch(
"api.db_utils.set_read_db_alias", return_value="token"
):
with patch("api.db_utils.reset_read_db_alias"):
with patch("api.db_utils.logger") as mock_logger:
with rls_transaction(tenant_id):
pass
assert mock_sleep.call_count == 2
mock_sleep.assert_any_call(0.5)
mock_sleep.assert_any_call(1.0)
assert mock_logger.info.call_count == 2
def test_rls_transaction_max_three_attempts_for_replica(
self, tenants_fixture, enable_read_replica
):
"""Test maximum 3 attempts for replica database."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic") as mock_atomic:
mock_atomic.side_effect = OperationalError("Persistent error")
with patch("api.db_utils.time.sleep"):
with patch(
"api.db_utils.set_read_db_alias", return_value="token"
):
with patch("api.db_utils.reset_read_db_alias"):
with pytest.raises(OperationalError):
with rls_transaction(tenant_id):
pass
assert mock_atomic.call_count == 3
def test_rls_transaction_only_one_attempt_for_primary(self, tenants_fixture):
"""Test only 1 attempt for primary database."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=None):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic") as mock_atomic:
mock_atomic.side_effect = OperationalError("Primary error")
with pytest.raises(OperationalError):
with rls_transaction(tenant_id):
pass
assert mock_atomic.call_count == 1
def test_rls_transaction_fallback_to_primary_after_max_attempts(
self, tenants_fixture, enable_read_replica
):
"""Test fallback to primary DB after max attempts on replica."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
call_count = 0
def atomic_side_effect(*args, **kwargs):
nonlocal call_count
call_count += 1
if call_count < 3:
raise OperationalError("Replica error")
return MagicMock(
__enter__=MagicMock(return_value=None),
__exit__=MagicMock(return_value=False),
)
with patch(
"api.db_utils.transaction.atomic", side_effect=atomic_side_effect
):
with patch("api.db_utils.time.sleep"):
with patch(
"api.db_utils.set_read_db_alias", return_value="token"
):
with patch("api.db_utils.reset_read_db_alias"):
with patch("api.db_utils.logger") as mock_logger:
with rls_transaction(tenant_id):
pass
mock_logger.warning.assert_called_once()
warning_msg = mock_logger.warning.call_args[0][0]
assert "falling back to primary DB" in warning_msg
def test_rls_transaction_logger_warning_on_fallback(
self, tenants_fixture, enable_read_replica
):
"""Test logger warnings are emitted on fallback to primary."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
call_count = 0
def atomic_side_effect(*args, **kwargs):
nonlocal call_count
call_count += 1
if call_count < 3:
raise OperationalError("Replica error")
return MagicMock(
__enter__=MagicMock(return_value=None),
__exit__=MagicMock(return_value=False),
)
with patch(
"api.db_utils.transaction.atomic", side_effect=atomic_side_effect
):
with patch("api.db_utils.time.sleep"):
with patch(
"api.db_utils.set_read_db_alias", return_value="token"
):
with patch("api.db_utils.reset_read_db_alias"):
with patch("api.db_utils.logger") as mock_logger:
with rls_transaction(tenant_id):
pass
assert mock_logger.info.call_count == 2
assert mock_logger.warning.call_count == 1
def test_rls_transaction_operational_error_raised_immediately_on_primary(
self, tenants_fixture
):
"""Test OperationalError raised immediately on primary without retry."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=None):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic") as mock_atomic:
mock_atomic.side_effect = OperationalError("Primary error")
with patch("api.db_utils.time.sleep") as mock_sleep:
with pytest.raises(OperationalError):
with rls_transaction(tenant_id):
pass
mock_sleep.assert_not_called()
def test_rls_transaction_operational_error_raised_after_max_attempts(
self, tenants_fixture, enable_read_replica
):
"""Test OperationalError raised after max attempts on replica."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic") as mock_atomic:
mock_atomic.side_effect = OperationalError(
"Persistent replica error"
)
with patch("api.db_utils.time.sleep"):
with patch(
"api.db_utils.set_read_db_alias", return_value="token"
):
with patch("api.db_utils.reset_read_db_alias"):
with pytest.raises(OperationalError):
with rls_transaction(tenant_id):
pass
def test_rls_transaction_router_token_set_for_non_default_alias(
self, tenants_fixture
):
"""Test router token is set when using non-default alias."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
custom_alias = "custom_db"
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic"):
with patch("api.db_utils.set_read_db_alias") as mock_set_alias:
with patch("api.db_utils.reset_read_db_alias") as mock_reset_alias:
mock_set_alias.return_value = "test_token"
with rls_transaction(tenant_id, using=custom_alias):
pass
mock_set_alias.assert_called_once_with(custom_alias)
mock_reset_alias.assert_called_once_with("test_token")
def test_rls_transaction_router_token_reset_in_finally_block(self, tenants_fixture):
"""Test router token is reset in finally block even on error."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
custom_alias = "custom_db"
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic") as mock_atomic:
mock_atomic.side_effect = Exception("Unexpected error")
with patch("api.db_utils.set_read_db_alias", return_value="test_token"):
with patch("api.db_utils.reset_read_db_alias") as mock_reset_alias:
with pytest.raises(Exception):
with rls_transaction(tenant_id, using=custom_alias):
pass
mock_reset_alias.assert_called_once_with("test_token")
def test_rls_transaction_router_token_not_set_for_default_alias(
self, tenants_fixture
):
"""Test router token is not set when using default alias."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with patch("api.db_utils.get_read_db_alias", return_value=None):
with patch("api.db_utils.connections") as mock_connections:
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_connections.__getitem__.return_value = mock_conn
mock_connections.__contains__.return_value = True
with patch("api.db_utils.transaction.atomic"):
with patch("api.db_utils.set_read_db_alias") as mock_set_alias:
with patch(
"api.db_utils.reset_read_db_alias"
) as mock_reset_alias:
with rls_transaction(tenant_id):
pass
mock_set_alias.assert_not_called()
mock_reset_alias.assert_not_called()
def test_rls_transaction_set_config_query_executed_with_correct_params(
self, tenants_fixture
):
"""Test SET_CONFIG_QUERY executed with correct parameters."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with rls_transaction(tenant_id) as cursor:
cursor.execute("SELECT current_setting(%s)", [POSTGRES_TENANT_VAR])
result = cursor.fetchone()
assert result[0] == tenant_id
def test_rls_transaction_custom_parameter(self, tenants_fixture):
"""Test rls_transaction with custom parameter name."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
custom_param = "api.user_id"
with rls_transaction(tenant_id, parameter=custom_param) as cursor:
cursor.execute("SELECT current_setting(%s)", [custom_param])
result = cursor.fetchone()
assert result[0] == tenant_id
def test_rls_transaction_cursor_yielded_correctly(self, tenants_fixture):
"""Test cursor is yielded correctly."""
tenant = tenants_fixture[0]
tenant_id = str(tenant.id)
with rls_transaction(tenant_id) as cursor:
assert cursor is not None
cursor.execute("SELECT 1")
result = cursor.fetchone()
assert result[0] == 1
+3 -79
View File
@@ -20,12 +20,9 @@ from prowler.providers.aws.aws_provider import AwsProvider
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
from prowler.providers.azure.azure_provider import AzureProvider
from prowler.providers.gcp.gcp_provider import GcpProvider
from prowler.providers.github.github_provider import GithubProvider
from prowler.providers.iac.iac_provider import IacProvider
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
from prowler.providers.m365.m365_provider import M365Provider
from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider
from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
from prowler.providers.oraclecloud.oci_provider import OciProvider
class TestMergeDicts:
@@ -112,10 +109,7 @@ class TestReturnProwlerProvider:
(Provider.ProviderChoices.AZURE.value, AzureProvider),
(Provider.ProviderChoices.KUBERNETES.value, KubernetesProvider),
(Provider.ProviderChoices.M365.value, M365Provider),
(Provider.ProviderChoices.GITHUB.value, GithubProvider),
(Provider.ProviderChoices.MONGODBATLAS.value, MongodbatlasProvider),
(Provider.ProviderChoices.ORACLECLOUD.value, OraclecloudProvider),
(Provider.ProviderChoices.IAC.value, IacProvider),
(Provider.ProviderChoices.OCI.value, OciProvider),
],
)
def test_return_prowler_provider(self, provider_type, expected_provider):
@@ -212,13 +206,9 @@ class TestGetProwlerProviderKwargs:
{"organizations": ["provider_uid"]},
),
(
Provider.ProviderChoices.ORACLECLOUD.value,
Provider.ProviderChoices.OCI.value,
{},
),
(
Provider.ProviderChoices.MONGODBATLAS.value,
{"atlas_organization_id": "provider_uid"},
),
],
)
def test_get_prowler_provider_kwargs(self, provider_type, expected_extra_kwargs):
@@ -256,72 +246,6 @@ class TestGetProwlerProviderKwargs:
expected_result = {**secret_dict, "mutelist_content": {"key": "value"}}
assert result == expected_result
def test_get_prowler_provider_kwargs_iac_provider(self):
"""Test that IaC provider gets correct kwargs with repository URL."""
provider_uid = "https://github.com/org/repo"
secret_dict = {"access_token": "test_token"}
secret_mock = MagicMock()
secret_mock.secret = secret_dict
provider = MagicMock()
provider.provider = Provider.ProviderChoices.IAC.value
provider.secret = secret_mock
provider.uid = provider_uid
result = get_prowler_provider_kwargs(provider)
expected_result = {
"scan_repository_url": provider_uid,
"oauth_app_token": "test_token",
}
assert result == expected_result
def test_get_prowler_provider_kwargs_iac_provider_without_token(self):
"""Test that IaC provider works without access token for public repos."""
provider_uid = "https://github.com/org/public-repo"
secret_dict = {}
secret_mock = MagicMock()
secret_mock.secret = secret_dict
provider = MagicMock()
provider.provider = Provider.ProviderChoices.IAC.value
provider.secret = secret_mock
provider.uid = provider_uid
result = get_prowler_provider_kwargs(provider)
expected_result = {"scan_repository_url": provider_uid}
assert result == expected_result
def test_get_prowler_provider_kwargs_iac_provider_ignores_mutelist(self):
"""Test that IaC provider does NOT receive mutelist_content.
IaC provider uses Trivy's built-in mutelist logic, so it should not
receive mutelist_content even when a mutelist processor is configured.
"""
provider_uid = "https://github.com/org/repo"
secret_dict = {"access_token": "test_token"}
secret_mock = MagicMock()
secret_mock.secret = secret_dict
mutelist_processor = MagicMock()
mutelist_processor.configuration = {"Mutelist": {"key": "value"}}
provider = MagicMock()
provider.provider = Provider.ProviderChoices.IAC.value
provider.secret = secret_mock
provider.uid = provider_uid
result = get_prowler_provider_kwargs(provider, mutelist_processor)
# IaC provider should NOT have mutelist_content
assert "mutelist_content" not in result
expected_result = {
"scan_repository_url": provider_uid,
"oauth_app_token": "test_token",
}
assert result == expected_result
def test_get_prowler_provider_kwargs_unsupported_provider(self):
# Setup
provider_uid = "provider_uid"
File diff suppressed because it is too large Load Diff
+12 -52
View File
@@ -18,11 +18,9 @@ from prowler.providers.azure.azure_provider import AzureProvider
from prowler.providers.common.models import Connection
from prowler.providers.gcp.gcp_provider import GcpProvider
from prowler.providers.github.github_provider import GithubProvider
from prowler.providers.iac.iac_provider import IacProvider
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
from prowler.providers.m365.m365_provider import M365Provider
from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider
from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
from prowler.providers.oraclecloud.oci_provider import OciProvider
class CustomOAuth2Client(OAuth2Client):
@@ -68,11 +66,9 @@ def return_prowler_provider(
| AzureProvider
| GcpProvider
| GithubProvider
| IacProvider
| KubernetesProvider
| M365Provider
| MongodbatlasProvider
| OraclecloudProvider
| OciProvider
]:
"""Return the Prowler provider class based on the given provider type.
@@ -80,7 +76,7 @@ def return_prowler_provider(
provider (Provider): The provider object containing the provider type and associated secrets.
Returns:
AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: The corresponding provider class.
AwsProvider | AzureProvider | GcpProvider | GithubProvider | KubernetesProvider | M365Provider | OciProvider: The corresponding provider class.
Raises:
ValueError: If the provider type specified in `provider.provider` is not supported.
@@ -98,12 +94,8 @@ def return_prowler_provider(
prowler_provider = M365Provider
case Provider.ProviderChoices.GITHUB.value:
prowler_provider = GithubProvider
case Provider.ProviderChoices.MONGODBATLAS.value:
prowler_provider = MongodbatlasProvider
case Provider.ProviderChoices.IAC.value:
prowler_provider = IacProvider
case Provider.ProviderChoices.ORACLECLOUD.value:
prowler_provider = OraclecloudProvider
case Provider.ProviderChoices.OCI.value:
prowler_provider = OciProvider
case _:
raise ValueError(f"Provider type {provider.provider} not supported")
return prowler_provider
@@ -140,26 +132,10 @@ def get_prowler_provider_kwargs(
**prowler_provider_kwargs,
"organizations": [provider.uid],
}
elif provider.provider == Provider.ProviderChoices.IAC.value:
# For IaC provider, uid contains the repository URL
# Extract the access token if present in the secret
prowler_provider_kwargs = {
"scan_repository_url": provider.uid,
}
if "access_token" in provider.secret.secret:
prowler_provider_kwargs["oauth_app_token"] = provider.secret.secret[
"access_token"
]
elif provider.provider == Provider.ProviderChoices.MONGODBATLAS.value:
prowler_provider_kwargs = {
**prowler_provider_kwargs,
"atlas_organization_id": provider.uid,
}
if mutelist_processor:
mutelist_content = mutelist_processor.configuration.get("Mutelist", {})
# IaC provider doesn't support mutelist (uses Trivy's built-in logic)
if mutelist_content and provider.provider != Provider.ProviderChoices.IAC.value:
if mutelist_content:
prowler_provider_kwargs["mutelist_content"] = mutelist_content
return prowler_provider_kwargs
@@ -173,11 +149,9 @@ def initialize_prowler_provider(
| AzureProvider
| GcpProvider
| GithubProvider
| IacProvider
| KubernetesProvider
| M365Provider
| MongodbatlasProvider
| OraclecloudProvider
| OciProvider
):
"""Initialize a Prowler provider instance based on the given provider type.
@@ -186,8 +160,8 @@ def initialize_prowler_provider(
mutelist_processor (Processor): The mutelist processor object containing the mutelist configuration.
Returns:
AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: An instance of the corresponding provider class
(`AwsProvider`, `AzureProvider`, `GcpProvider`, `GithubProvider`, `IacProvider`, `KubernetesProvider`, `M365Provider`, `OraclecloudProvider` or `MongodbatlasProvider`) initialized with the
AwsProvider | AzureProvider | GcpProvider | GithubProvider | KubernetesProvider | M365Provider | OciProvider: An instance of the corresponding provider class
(`AwsProvider`, `AzureProvider`, `GcpProvider`, `GithubProvider`, `KubernetesProvider`, `M365Provider` or `OciProvider`) initialized with the
provider's secrets.
"""
prowler_provider = return_prowler_provider(provider)
@@ -211,23 +185,9 @@ def prowler_provider_connection_test(provider: Provider) -> Connection:
except Provider.secret.RelatedObjectDoesNotExist as secret_error:
return Connection(is_connected=False, error=secret_error)
# For IaC provider, construct the kwargs properly for test_connection
if provider.provider == Provider.ProviderChoices.IAC.value:
# Don't pass repository_url from secret, use scan_repository_url with the UID
iac_test_kwargs = {
"scan_repository_url": provider.uid,
"raise_on_exception": False,
}
# Add access_token if present in the secret
if "access_token" in prowler_provider_kwargs:
iac_test_kwargs["access_token"] = prowler_provider_kwargs["access_token"]
return prowler_provider.test_connection(**iac_test_kwargs)
else:
return prowler_provider.test_connection(
**prowler_provider_kwargs,
provider_id=provider.uid,
raise_on_exception=False,
)
return prowler_provider.test_connection(
**prowler_provider_kwargs, provider_id=provider.uid, raise_on_exception=False
)
def prowler_integration_connection_test(integration: Integration) -> Connection:
@@ -1,6 +1,5 @@
import re
from drf_spectacular.utils import extend_schema_field
from rest_framework_json_api import serializers
@@ -12,198 +11,3 @@ class OpenAICredentialsSerializer(serializers.Serializer):
if not re.match(pattern, value or ""):
raise serializers.ValidationError("Invalid OpenAI API key format.")
return value
def to_internal_value(self, data):
"""Check for unknown fields before DRF filters them out."""
if not isinstance(data, dict):
raise serializers.ValidationError(
{"non_field_errors": ["Credentials must be an object"]}
)
allowed_fields = set(self.fields.keys())
provided_fields = set(data.keys())
extra_fields = provided_fields - allowed_fields
if extra_fields:
raise serializers.ValidationError(
{
"non_field_errors": [
f"Unknown fields in credentials: {', '.join(sorted(extra_fields))}"
]
}
)
return super().to_internal_value(data)
class BedrockCredentialsSerializer(serializers.Serializer):
"""
Serializer for AWS Bedrock credentials validation.
Validates long-term AWS credentials (AKIA) and region format.
"""
access_key_id = serializers.CharField()
secret_access_key = serializers.CharField()
region = serializers.CharField()
def validate_access_key_id(self, value: str) -> str:
"""Validate AWS access key ID format (AKIA for long-term credentials)."""
pattern = r"^AKIA[0-9A-Z]{16}$"
if not re.match(pattern, value or ""):
raise serializers.ValidationError(
"Invalid AWS access key ID format. Must be AKIA followed by 16 alphanumeric characters."
)
return value
def validate_secret_access_key(self, value: str) -> str:
"""Validate AWS secret access key format (40 base64 characters)."""
pattern = r"^[A-Za-z0-9/+=]{40}$"
if not re.match(pattern, value or ""):
raise serializers.ValidationError(
"Invalid AWS secret access key format. Must be 40 base64 characters."
)
return value
def validate_region(self, value: str) -> str:
"""Validate AWS region format."""
pattern = r"^[a-z]{2}-[a-z]+-\d+$"
if not re.match(pattern, value or ""):
raise serializers.ValidationError(
"Invalid AWS region format. Expected format like 'us-east-1' or 'eu-west-2'."
)
return value
def to_internal_value(self, data):
"""Check for unknown fields before DRF filters them out."""
if not isinstance(data, dict):
raise serializers.ValidationError(
{"non_field_errors": ["Credentials must be an object"]}
)
allowed_fields = set(self.fields.keys())
provided_fields = set(data.keys())
extra_fields = provided_fields - allowed_fields
if extra_fields:
raise serializers.ValidationError(
{
"non_field_errors": [
f"Unknown fields in credentials: {', '.join(sorted(extra_fields))}"
]
}
)
return super().to_internal_value(data)
class BedrockCredentialsUpdateSerializer(BedrockCredentialsSerializer):
"""
Serializer for AWS Bedrock credentials during UPDATE operations.
Inherits all validation logic from BedrockCredentialsSerializer but makes
all fields optional to support partial updates.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Make all fields optional for updates
for field in self.fields.values():
field.required = False
class OpenAICompatibleCredentialsSerializer(serializers.Serializer):
"""
Minimal serializer for OpenAI-compatible credentials.
Many OpenAI-compatible providers do not use the same key format as OpenAI.
We only require a non-empty API key string. Additional fields can be added later
without breaking existing configurations.
"""
api_key = serializers.CharField()
def validate_api_key(self, value: str) -> str:
if not isinstance(value, str) or not value.strip():
raise serializers.ValidationError("API key is required.")
return value.strip()
def to_internal_value(self, data):
"""Check for unknown fields before DRF filters them out."""
if not isinstance(data, dict):
raise serializers.ValidationError(
{"non_field_errors": ["Credentials must be an object"]}
)
allowed_fields = set(self.fields.keys())
provided_fields = set(data.keys())
extra_fields = provided_fields - allowed_fields
if extra_fields:
raise serializers.ValidationError(
{
"non_field_errors": [
f"Unknown fields in credentials: {', '.join(sorted(extra_fields))}"
]
}
)
return super().to_internal_value(data)
@extend_schema_field(
{
"oneOf": [
{
"type": "object",
"title": "OpenAI Credentials",
"properties": {
"api_key": {
"type": "string",
"description": "OpenAI API key. Must start with 'sk-' followed by alphanumeric characters, "
"hyphens, or underscores.",
"pattern": "^sk-[\\w-]+$",
}
},
"required": ["api_key"],
},
{
"type": "object",
"title": "AWS Bedrock Credentials",
"properties": {
"access_key_id": {
"type": "string",
"description": "AWS access key ID.",
"pattern": "^AKIA[0-9A-Z]{16}$",
},
"secret_access_key": {
"type": "string",
"description": "AWS secret access key.",
"pattern": "^[A-Za-z0-9/+=]{40}$",
},
"region": {
"type": "string",
"description": "AWS region identifier where Bedrock is available. Examples: us-east-1, "
"us-west-2, eu-west-1, ap-northeast-1.",
"pattern": "^[a-z]{2}-[a-z]+-\\d+$",
},
},
"required": ["access_key_id", "secret_access_key", "region"],
},
{
"type": "object",
"title": "OpenAI Compatible Credentials",
"properties": {
"api_key": {
"type": "string",
"description": "API key for OpenAI-compatible provider. The format varies by provider. "
"Note: The 'base_url' field (separate from credentials) is required when using this provider type.",
}
},
"required": ["api_key"],
},
]
}
)
class LighthouseCredentialsField(serializers.JSONField):
pass
@@ -239,21 +239,6 @@ from rest_framework_json_api import serializers
},
"required": ["github_app_id", "github_app_key"],
},
{
"type": "object",
"title": "IaC Repository Credentials",
"properties": {
"repository_url": {
"type": "string",
"description": "Repository URL to scan for IaC files.",
},
"access_token": {
"type": "string",
"description": "Optional access token for private repositories.",
},
},
"required": ["repository_url"],
},
{
"type": "object",
"title": "Oracle Cloud Infrastructure (OCI) API Key Credentials",
@@ -289,21 +274,6 @@ from rest_framework_json_api import serializers
},
"required": ["user", "fingerprint", "tenancy", "region"],
},
{
"type": "object",
"title": "MongoDB Atlas API Key",
"properties": {
"atlas_public_key": {
"type": "string",
"description": "MongoDB Atlas API public key.",
},
"atlas_private_key": {
"type": "string",
"description": "MongoDB Atlas API private key.",
},
},
"required": ["atlas_public_key", "atlas_private_key"],
},
]
}
)
+7 -384
View File
@@ -31,7 +31,6 @@ from api.models import (
LighthouseProviderModels,
LighthouseTenantConfiguration,
Membership,
MuteRule,
Processor,
Provider,
ProviderGroup,
@@ -47,7 +46,6 @@ from api.models import (
StatusChoices,
Task,
TenantAPIKey,
ThreatScoreSnapshot,
User,
UserRoleRelationship,
)
@@ -61,13 +59,7 @@ from api.v1.serializer_utils.integrations import (
S3ConfigSerializer,
SecurityHubConfigSerializer,
)
from api.v1.serializer_utils.lighthouse import (
BedrockCredentialsSerializer,
BedrockCredentialsUpdateSerializer,
LighthouseCredentialsField,
OpenAICompatibleCredentialsSerializer,
OpenAICredentialsSerializer,
)
from api.v1.serializer_utils.lighthouse import OpenAICredentialsSerializer
from api.v1.serializer_utils.processors import ProcessorConfigField
from api.v1.serializer_utils.providers import ProviderSecretField
from prowler.lib.mutelist.mutelist import Mutelist
@@ -1167,17 +1159,11 @@ class ResourceSerializer(RLSSerializer):
"findings",
"failed_findings_count",
"url",
"metadata",
"details",
"partition",
]
extra_kwargs = {
"id": {"read_only": True},
"inserted_at": {"read_only": True},
"updated_at": {"read_only": True},
"metadata": {"read_only": True},
"details": {"read_only": True},
"partition": {"read_only": True},
}
included_serializers = {
@@ -1234,15 +1220,11 @@ class ResourceIncludeSerializer(RLSSerializer):
"service",
"type_",
"tags",
"details",
"partition",
]
extra_kwargs = {
"id": {"read_only": True},
"inserted_at": {"read_only": True},
"updated_at": {"read_only": True},
"details": {"read_only": True},
"partition": {"read_only": True},
}
@extend_schema_field(
@@ -1373,16 +1355,12 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
serializer = GCPProviderSecret(data=secret)
elif provider_type == Provider.ProviderChoices.GITHUB.value:
serializer = GithubProviderSecret(data=secret)
elif provider_type == Provider.ProviderChoices.IAC.value:
serializer = IacProviderSecret(data=secret)
elif provider_type == Provider.ProviderChoices.KUBERNETES.value:
serializer = KubernetesProviderSecret(data=secret)
elif provider_type == Provider.ProviderChoices.M365.value:
serializer = M365ProviderSecret(data=secret)
elif provider_type == Provider.ProviderChoices.ORACLECLOUD.value:
elif provider_type == Provider.ProviderChoices.OCI.value:
serializer = OracleCloudProviderSecret(data=secret)
elif provider_type == Provider.ProviderChoices.MONGODBATLAS.value:
serializer = MongoDBAtlasProviderSecret(data=secret)
else:
raise serializers.ValidationError(
{"provider": f"Provider type not supported {provider_type}"}
@@ -1479,14 +1457,6 @@ class GCPServiceAccountProviderSecret(serializers.Serializer):
resource_name = "provider-secrets"
class MongoDBAtlasProviderSecret(serializers.Serializer):
atlas_public_key = serializers.CharField()
atlas_private_key = serializers.CharField()
class Meta:
resource_name = "provider-secrets"
class KubernetesProviderSecret(serializers.Serializer):
kubeconfig_content = serializers.CharField()
@@ -1504,14 +1474,6 @@ class GithubProviderSecret(serializers.Serializer):
resource_name = "provider-secrets"
class IacProviderSecret(serializers.Serializer):
repository_url = serializers.CharField()
access_token = serializers.CharField(required=False)
class Meta:
resource_name = "provider-secrets"
class OracleCloudProviderSecret(serializers.Serializer):
user = serializers.CharField()
fingerprint = serializers.CharField()
@@ -2229,30 +2191,6 @@ class OverviewServiceSerializer(serializers.Serializer):
return {"version": "v1"}
class OverviewRegionSerializer(serializers.Serializer):
id = serializers.SerializerMethodField()
provider_type = serializers.CharField()
region = serializers.CharField()
total = serializers.IntegerField()
_pass = serializers.IntegerField()
fail = serializers.IntegerField()
muted = serializers.IntegerField()
class JSONAPIMeta:
resource_name = "regions-overview"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fields["pass"] = self.fields.pop("_pass")
def get_id(self, obj):
"""Generate unique ID from provider_type and region."""
return f"{obj['provider_type']}:{obj['region']}"
def get_root_meta(self, _resource, _many):
return {"version": "v1"}
# Schedules
@@ -3137,12 +3075,7 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize
Accepts credentials as JSON; stored encrypted via credentials_decoded.
"""
credentials = LighthouseCredentialsField(write_only=True, required=True)
base_url = serializers.URLField(
required=False,
allow_null=True,
help_text="Base URL for the LLM provider API. Required for 'openai_compatible' provider type.",
)
credentials = serializers.JSONField(write_only=True, required=True)
class Meta:
model = LighthouseProviderConfiguration
@@ -3154,10 +3087,7 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize
]
extra_kwargs = {
"is_active": {"required": False},
"provider_type": {
"help_text": "LLM provider type. Determines which credential format to use. "
"See 'credentials' field documentation for provider-specific requirements."
},
"base_url": {"required": False, "allow_null": True},
}
def create(self, validated_data):
@@ -3180,7 +3110,6 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize
def validate(self, attrs):
provider_type = attrs.get("provider_type")
credentials = attrs.get("credentials") or {}
base_url = attrs.get("base_url")
if provider_type == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI:
try:
@@ -3193,35 +3122,6 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize
e.detail[f"credentials/{key}"] = value
del e.detail[key]
raise e
elif (
provider_type == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
):
try:
BedrockCredentialsSerializer(data=credentials).is_valid(
raise_exception=True
)
except ValidationError as e:
details = e.detail.copy()
for key, value in details.items():
e.detail[f"credentials/{key}"] = value
del e.detail[key]
raise e
elif (
provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE
):
if not base_url:
raise ValidationError({"base_url": "Base URL is required."})
try:
OpenAICompatibleCredentialsSerializer(data=credentials).is_valid(
raise_exception=True
)
except ValidationError as e:
details = e.detail.copy()
for key, value in details.items():
e.detail[f"credentials/{key}"] = value
del e.detail[key]
raise e
return super().validate(attrs)
@@ -3231,12 +3131,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
Update serializer for LighthouseProviderConfiguration.
"""
credentials = LighthouseCredentialsField(write_only=True, required=False)
base_url = serializers.URLField(
required=False,
allow_null=True,
help_text="Base URL for the LLM provider API. Required for 'openai_compatible' provider type.",
)
credentials = serializers.JSONField(write_only=True, required=False)
class Meta:
model = LighthouseProviderConfiguration
@@ -3250,6 +3145,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
extra_kwargs = {
"id": {"read_only": True},
"provider_type": {"read_only": True},
"base_url": {"required": False, "allow_null": True},
"is_active": {"required": False},
}
@@ -3260,11 +3156,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
setattr(instance, attr, value)
if credentials is not None:
# Merge partial credentials with existing ones
# New values overwrite existing ones, but unspecified fields are preserved
existing_credentials = instance.credentials_decoded or {}
merged_credentials = {**existing_credentials, **credentials}
instance.credentials_decoded = merged_credentials
instance.credentials_decoded = credentials
instance.save()
return instance
@@ -3272,7 +3164,6 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
def validate(self, attrs):
provider_type = getattr(self.instance, "provider_type", None)
credentials = attrs.get("credentials", None)
base_url = attrs.get("base_url", None)
if (
credentials is not None
@@ -3289,40 +3180,6 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
e.detail[f"credentials/{key}"] = value
del e.detail[key]
raise e
elif (
credentials is not None
and provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
):
try:
BedrockCredentialsUpdateSerializer(data=credentials).is_valid(
raise_exception=True
)
except ValidationError as e:
details = e.detail.copy()
for key, value in details.items():
e.detail[f"credentials/{key}"] = value
del e.detail[key]
raise e
elif (
credentials is not None
and provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE
):
if base_url is None:
pass
elif not base_url:
raise ValidationError({"base_url": "Base URL cannot be empty."})
try:
OpenAICompatibleCredentialsSerializer(data=credentials).is_valid(
raise_exception=True
)
except ValidationError as e:
details = e.detail.copy()
for key, value in details.items():
e.detail[f"credentials/{key}"] = value
del e.detail[key]
raise e
return super().validate(attrs)
@@ -3488,237 +3345,3 @@ class LighthouseProviderModelsUpdateSerializer(BaseWriteSerializer):
extra_kwargs = {
"id": {"read_only": True},
}
# Mute Rules
class MuteRuleSerializer(RLSSerializer):
"""
Serializer for reading MuteRule instances.
"""
finding_uids = serializers.ListField(
child=serializers.CharField(),
read_only=True,
help_text="List of finding UIDs that are muted by this rule",
)
class Meta:
model = MuteRule
fields = [
"id",
"inserted_at",
"updated_at",
"name",
"reason",
"enabled",
"created_by",
"finding_uids",
]
included_serializers = {
"created_by": "api.v1.serializers.UserIncludeSerializer",
}
class MuteRuleCreateSerializer(RLSSerializer, BaseWriteSerializer):
"""
Serializer for creating new MuteRule instances.
Accepts finding_ids in the request, converts them to UIDs, and stores in finding_uids.
"""
finding_ids = serializers.ListField(
child=serializers.UUIDField(),
write_only=True,
required=True,
help_text="List of Finding IDs to mute (will be converted to UIDs)",
)
finding_uids = serializers.ListField(
child=serializers.CharField(),
read_only=True,
help_text="List of finding UIDs that are muted by this rule",
)
class Meta:
model = MuteRule
fields = [
"id",
"inserted_at",
"updated_at",
"name",
"reason",
"enabled",
"created_by",
"finding_ids",
"finding_uids",
]
extra_kwargs = {
"id": {"read_only": True},
"inserted_at": {"read_only": True},
"updated_at": {"read_only": True},
"enabled": {"read_only": True},
"created_by": {"read_only": True},
}
def validate_name(self, value):
"""Validate that the name is unique within the tenant."""
tenant_id = self.context.get("tenant_id")
if MuteRule.objects.filter(tenant_id=tenant_id, name=value).exists():
raise ValidationError("A mute rule with this name already exists.")
return value
def validate_finding_ids(self, value):
"""Validate that all finding IDs exist and belong to the tenant."""
if not value:
raise ValidationError("At least one finding_id must be provided.")
tenant_id = self.context.get("tenant_id")
# Check that all findings exist and belong to this tenant
findings = Finding.all_objects.filter(tenant_id=tenant_id, id__in=value)
found_ids = set(findings.values_list("id", flat=True))
provided_ids = set(value)
missing_ids = provided_ids - found_ids
if missing_ids:
raise ValidationError(
f"The following finding IDs do not exist or do not belong to your tenant: {missing_ids}"
)
return value
def validate(self, data):
"""Validate the entire mute rule, including overlap detection."""
data = super().validate(data)
tenant_id = self.context.get("tenant_id")
finding_ids = data.get("finding_ids", [])
if not finding_ids:
return data
# Convert finding IDs to UIDs (deduplicate in case multiple findings have same UID)
findings = Finding.all_objects.filter(id__in=finding_ids, tenant_id=tenant_id)
finding_uids = list(set(findings.values_list("uid", flat=True)))
# Check for overlaps with existing enabled rules
existing_rules = MuteRule.objects.filter(tenant_id=tenant_id, enabled=True)
for rule in existing_rules:
overlap = set(finding_uids) & set(rule.finding_uids)
if overlap:
raise ConflictException(
detail=f"The following finding UIDs are already muted by rule '{rule.name}': {overlap}"
)
# Store finding_uids in validated_data for create
data["finding_uids"] = finding_uids
return data
def create(self, validated_data):
"""Create a new mute rule and set created_by."""
# Remove finding_ids from validated_data (we've already converted to finding_uids)
validated_data.pop("finding_ids", None)
# Set created_by to the current user
request = self.context.get("request")
if request and hasattr(request, "user"):
validated_data["created_by"] = request.user
return super().create(validated_data)
class MuteRuleUpdateSerializer(BaseWriteSerializer):
"""
Serializer for updating MuteRule instances.
"""
class Meta:
model = MuteRule
fields = [
"id",
"name",
"reason",
"enabled",
]
extra_kwargs = {
"id": {"read_only": True},
"name": {"required": False},
"reason": {"required": False},
"enabled": {"required": False},
}
def validate_name(self, value):
"""Validate that the name is unique within the tenant, excluding current instance."""
tenant_id = self.context.get("tenant_id")
if (
MuteRule.objects.filter(tenant_id=tenant_id, name=value)
.exclude(id=self.instance.id)
.exists()
):
raise ValidationError("A mute rule with this name already exists.")
return value
# ThreatScore Snapshots
class ThreatScoreSnapshotSerializer(RLSSerializer):
"""
Serializer for ThreatScore snapshots.
Read-only serializer for retrieving historical ThreatScore metrics.
"""
id = serializers.SerializerMethodField()
class Meta:
model = ThreatScoreSnapshot
fields = [
"id",
"inserted_at",
"scan",
"provider",
"compliance_id",
"overall_score",
"score_delta",
"section_scores",
"critical_requirements",
"total_requirements",
"passed_requirements",
"failed_requirements",
"manual_requirements",
"total_findings",
"passed_findings",
"failed_findings",
]
extra_kwargs = {
"id": {"read_only": True},
"inserted_at": {"read_only": True},
"scan": {"read_only": True},
"provider": {"read_only": True},
"compliance_id": {"read_only": True},
"overall_score": {"read_only": True},
"score_delta": {"read_only": True},
"section_scores": {"read_only": True},
"critical_requirements": {"read_only": True},
"total_requirements": {"read_only": True},
"passed_requirements": {"read_only": True},
"failed_requirements": {"read_only": True},
"manual_requirements": {"read_only": True},
"total_findings": {"read_only": True},
"passed_findings": {"read_only": True},
"failed_findings": {"read_only": True},
}
included_serializers = {
"scan": "api.v1.serializers.ScanIncludeSerializer",
"provider": "api.v1.serializers.ProviderIncludeSerializer",
}
def get_id(self, obj):
if getattr(obj, "_aggregated", False):
return "n/a"
return str(obj.id)
+1 -2
View File
@@ -21,7 +21,6 @@ from api.v1.views import (
LighthouseProviderModelsViewSet,
LighthouseTenantConfigViewSet,
MembershipViewSet,
MuteRuleViewSet,
OverviewViewSet,
ProcessorViewSet,
ProviderGroupProvidersRelationshipView,
@@ -81,7 +80,6 @@ router.register(
LighthouseProviderModelsViewSet,
basename="lighthouse-models",
)
router.register(r"mute-rules", MuteRuleViewSet, basename="mute-rule")
tenants_router = routers.NestedSimpleRouter(router, r"tenants", lookup="tenant")
tenants_router.register(
@@ -152,6 +150,7 @@ urlpatterns = [
),
name="provider_group-providers-relationship",
),
# Lighthouse tenant config as singleton endpoint
path(
"lighthouse/configuration",
LighthouseTenantConfigViewSet.as_view(
File diff suppressed because it is too large Load Diff
+11 -55
View File
@@ -23,7 +23,6 @@ from api.models import (
Invitation,
LighthouseConfiguration,
Membership,
MuteRule,
Processor,
Provider,
ProviderGroup,
@@ -501,28 +500,13 @@ def providers_fixture(tenants_fixture):
tenant_id=tenant.id,
)
provider7 = Provider.objects.create(
provider="oraclecloud",
provider="oci",
uid="ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda",
alias="oci_testing",
tenant_id=tenant.id,
)
provider8 = Provider.objects.create(
provider="mongodbatlas",
uid="64b1d3c0e4b03b1234567890",
alias="mongodbatlas_testing",
tenant_id=tenant.id,
)
return (
provider1,
provider2,
provider3,
provider4,
provider5,
provider6,
provider7,
provider8,
)
return provider1, provider2, provider3, provider4, provider5, provider6, provider7
@pytest.fixture
@@ -1108,8 +1092,8 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
region="region1",
_pass=1,
fail=0,
muted=2,
total=3,
muted=0,
total=1,
new=1,
changed=0,
unchanged=0,
@@ -1117,7 +1101,7 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
fail_changed=0,
pass_new=1,
pass_changed=0,
muted_new=2,
muted_new=0,
muted_changed=0,
scan=scan,
)
@@ -1130,8 +1114,8 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
region="region2",
_pass=0,
fail=1,
muted=3,
total=4,
muted=1,
total=2,
new=2,
changed=0,
unchanged=0,
@@ -1139,7 +1123,7 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
fail_changed=0,
pass_new=0,
pass_changed=0,
muted_new=3,
muted_new=1,
muted_changed=0,
scan=scan,
)
@@ -1152,8 +1136,8 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
region="region1",
_pass=1,
fail=0,
muted=1,
total=2,
muted=0,
total=1,
new=1,
changed=0,
unchanged=0,
@@ -1161,7 +1145,7 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
fail_changed=0,
pass_new=1,
pass_changed=0,
muted_new=1,
muted_new=0,
muted_changed=0,
scan=scan,
)
@@ -1441,34 +1425,6 @@ def api_keys_fixture(tenants_fixture, create_test_user):
return [api_key1, api_key2, api_key3]
@pytest.fixture
def mute_rules_fixture(tenants_fixture, create_test_user, findings_fixture):
"""Create test mute rules for testing."""
tenant = tenants_fixture[0]
user = create_test_user
# Create two mute rules: one enabled, one disabled
mute_rule1 = MuteRule.objects.create(
tenant_id=tenant.id,
name="Test Rule 1",
reason="Security exception for testing",
enabled=True,
created_by=user,
finding_uids=[findings_fixture[0].uid],
)
mute_rule2 = MuteRule.objects.create(
tenant_id=tenant.id,
name="Test Rule 2",
reason="Compliance exception approved",
enabled=False,
created_by=user,
finding_uids=[findings_fixture[1].uid],
)
return mute_rule1, mute_rule2
def get_authorization_header(access_token: str) -> dict:
return {"Authorization": f"Bearer {access_token}"}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 95 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 94 KiB

+1 -117
View File
@@ -1,10 +1,5 @@
from collections import defaultdict
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import (
ComplianceOverviewSummary,
ComplianceRequirementOverview,
Resource,
ResourceFindingMapping,
ResourceScanSummary,
@@ -14,7 +9,7 @@ from api.models import (
def backfill_resource_scan_summaries(tenant_id: str, scan_id: str):
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
with rls_transaction(tenant_id):
if ResourceScanSummary.objects.filter(
tenant_id=tenant_id, scan_id=scan_id
).exists():
@@ -64,114 +59,3 @@ def backfill_resource_scan_summaries(tenant_id: str, scan_id: str):
)
return {"status": "backfilled", "inserted": len(summaries)}
def backfill_compliance_summaries(tenant_id: str, scan_id: str):
"""
Backfill ComplianceOverviewSummary records for a completed scan.
This function checks if summary records already exist for the scan.
If not, it aggregates compliance requirement data and creates the summaries.
Args:
tenant_id: Target tenant UUID
scan_id: Scan UUID to backfill
Returns:
dict: Status indicating whether backfill was performed
"""
with rls_transaction(tenant_id):
if ComplianceOverviewSummary.objects.filter(
tenant_id=tenant_id, scan_id=scan_id
).exists():
return {"status": "already backfilled"}
with rls_transaction(tenant_id):
if not Scan.objects.filter(
tenant_id=tenant_id,
id=scan_id,
state__in=(StateChoices.COMPLETED, StateChoices.FAILED),
).exists():
return {"status": "scan is not completed"}
# Fetch all compliance requirement overview rows for this scan
requirement_rows = ComplianceRequirementOverview.objects.filter(
tenant_id=tenant_id, scan_id=scan_id
).values(
"compliance_id",
"requirement_id",
"requirement_status",
)
if not requirement_rows:
return {"status": "no compliance data to backfill"}
# Group by (compliance_id, requirement_id) across regions
requirement_statuses = defaultdict(
lambda: {"fail_count": 0, "pass_count": 0, "total_count": 0}
)
for row in requirement_rows:
compliance_id = row["compliance_id"]
requirement_id = row["requirement_id"]
requirement_status = row["requirement_status"]
# Aggregate requirement status across regions
key = (compliance_id, requirement_id)
requirement_statuses[key]["total_count"] += 1
if requirement_status == "FAIL":
requirement_statuses[key]["fail_count"] += 1
elif requirement_status == "PASS":
requirement_statuses[key]["pass_count"] += 1
# Determine per-requirement status and aggregate to compliance level
compliance_summaries = defaultdict(
lambda: {
"total_requirements": 0,
"requirements_passed": 0,
"requirements_failed": 0,
"requirements_manual": 0,
}
)
for (compliance_id, requirement_id), counts in requirement_statuses.items():
# Apply business rule: any FAIL → requirement fails
if counts["fail_count"] > 0:
req_status = "FAIL"
elif counts["pass_count"] == counts["total_count"]:
req_status = "PASS"
else:
req_status = "MANUAL"
# Aggregate to compliance level
compliance_summaries[compliance_id]["total_requirements"] += 1
if req_status == "PASS":
compliance_summaries[compliance_id]["requirements_passed"] += 1
elif req_status == "FAIL":
compliance_summaries[compliance_id]["requirements_failed"] += 1
else:
compliance_summaries[compliance_id]["requirements_manual"] += 1
# Create summary objects
summary_objects = []
for compliance_id, data in compliance_summaries.items():
summary_objects.append(
ComplianceOverviewSummary(
tenant_id=tenant_id,
scan_id=scan_id,
compliance_id=compliance_id,
requirements_passed=data["requirements_passed"],
requirements_failed=data["requirements_failed"],
requirements_manual=data["requirements_manual"],
total_requirements=data["total_requirements"],
)
)
# Bulk insert summaries
if summary_objects:
ComplianceOverviewSummary.objects.bulk_create(
summary_objects, batch_size=500, ignore_conflicts=True
)
return {"status": "backfilled", "inserted": len(summary_objects)}
+36 -129
View File
@@ -15,7 +15,6 @@ from prowler.config.config import (
html_file_suffix,
json_asff_file_suffix,
json_ocsf_file_suffix,
set_output_timestamp,
)
from prowler.lib.outputs.asff.asff import ASFF
from prowler.lib.outputs.compliance.aws_well_architected.aws_well_architected import (
@@ -33,7 +32,7 @@ from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS
from prowler.lib.outputs.compliance.cis.cis_github import GithubCIS
from prowler.lib.outputs.compliance.cis.cis_kubernetes import KubernetesCIS
from prowler.lib.outputs.compliance.cis.cis_m365 import M365CIS
from prowler.lib.outputs.compliance.cis.cis_oraclecloud import OracleCloudCIS
from prowler.lib.outputs.compliance.cis.cis_oci import OCICIS
from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
@@ -59,9 +58,6 @@ from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_azur
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_gcp import (
ProwlerThreatScoreGCP,
)
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_kubernetes import (
ProwlerThreatScoreKubernetes,
)
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_m365 import (
ProwlerThreatScoreM365,
)
@@ -108,10 +104,6 @@ COMPLIANCE_CLASS_MAP = {
"kubernetes": [
(lambda name: name.startswith("cis_"), KubernetesCIS),
(lambda name: name.startswith("iso27001_"), KubernetesISO27001),
(
lambda name: name == "prowler_threatscore_kubernetes",
ProwlerThreatScoreKubernetes,
),
],
"m365": [
(lambda name: name.startswith("cis_"), M365CIS),
@@ -121,12 +113,8 @@ COMPLIANCE_CLASS_MAP = {
"github": [
(lambda name: name.startswith("cis_"), GithubCIS),
],
"iac": [
# IaC provider doesn't have specific compliance frameworks yet
# Trivy handles its own compliance checks
],
"oraclecloud": [
(lambda name: name.startswith("cis_"), OracleCloudCIS),
"oci": [
(lambda name: name.startswith("cis_"), OCICIS),
],
}
@@ -242,33 +230,36 @@ def _upload_to_s3(
logger.error(f"S3 upload failed: {str(e)}")
def _build_output_path(
output_directory: str,
prowler_provider: str,
tenant_id: str,
scan_id: str,
subdirectory: str = None,
) -> str:
def _generate_output_directory(
output_directory, prowler_provider: object, tenant_id: str, scan_id: str
) -> tuple[str, str, str]:
"""
Build a file system path for the output directory of a prowler scan.
Generate a file system path for the output directory of a prowler scan.
This function constructs the output directory path by combining a base
temporary output directory, the tenant ID, the scan ID, and details about
the prowler provider along with a timestamp. The resulting path is used to
store the output files of a prowler scan.
Note:
This function depends on one external variable:
- `output_file_timestamp`: A timestamp (as a string) used to uniquely identify the output.
Args:
output_directory (str): The base output directory.
prowler_provider (str): An identifier or descriptor for the prowler provider.
Typically, this is a string indicating the provider (e.g., "aws").
prowler_provider (object): An identifier or descriptor for the prowler provider.
Typically, this is a string indicating the provider (e.g., "aws").
tenant_id (str): The unique identifier for the tenant.
scan_id (str): The unique identifier for the scan.
subdirectory (str, optional): Optional subdirectory to include in the path
(e.g., "compliance", "threatscore", "ens").
Returns:
str: The constructed path with directory created.
str: The constructed file system path for the prowler scan output directory.
Example:
>>> _build_output_path("/tmp", "aws", "tenant-1234", "scan-5678")
'/tmp/tenant-1234/scan-5678/prowler-output-aws-20230215123456'
>>> _build_output_path("/tmp", "aws", "tenant-1234", "scan-5678", "threatscore")
'/tmp/tenant-1234/scan-5678/threatscore/prowler-output-aws-20230215123456'
>>> _generate_output_directory("/tmp", "aws", "tenant-1234", "scan-5678")
'/tmp/tenant-1234/aws/scan-5678/prowler-output-2023-02-15T12:34:56',
'/tmp/tenant-1234/aws/scan-5678/compliance/prowler-output-2023-02-15T12:34:56'
'/tmp/tenant-1234/aws/scan-5678/threatscore/prowler-output-2023-02-15T12:34:56'
"""
# Sanitize the prowler provider name to ensure it is a valid directory name
prowler_provider_sanitized = re.sub(r"[^\w\-]", "-", prowler_provider)
@@ -276,107 +267,23 @@ def _build_output_path(
with rls_transaction(tenant_id):
started_at = Scan.objects.get(id=scan_id).started_at
set_output_timestamp(started_at)
timestamp = started_at.strftime("%Y%m%d%H%M%S")
if subdirectory:
path = (
f"{output_directory}/{tenant_id}/{scan_id}/{subdirectory}/prowler-output-"
f"{prowler_provider_sanitized}-{timestamp}"
)
else:
path = (
f"{output_directory}/{tenant_id}/{scan_id}/prowler-output-"
f"{prowler_provider_sanitized}-{timestamp}"
)
# Create directory for the path if it doesn't exist
path = (
f"{output_directory}/{tenant_id}/{scan_id}/prowler-output-"
f"{prowler_provider_sanitized}-{timestamp}"
)
os.makedirs("/".join(path.split("/")[:-1]), exist_ok=True)
return path
def _generate_compliance_output_directory(
output_directory: str,
prowler_provider: str,
tenant_id: str,
scan_id: str,
compliance_framework: str,
) -> str:
"""
Generate a file system path for a compliance framework output directory.
This function constructs the output directory path specifically for a compliance
framework (e.g., "threatscore", "ens") by combining a base temporary output directory,
the tenant ID, the scan ID, the compliance framework name, and details about the
prowler provider along with a timestamp.
Args:
output_directory (str): The base output directory.
prowler_provider (str): An identifier or descriptor for the prowler provider.
Typically, this is a string indicating the provider (e.g., "aws").
tenant_id (str): The unique identifier for the tenant.
scan_id (str): The unique identifier for the scan.
compliance_framework (str): The compliance framework name (e.g., "threatscore", "ens").
Returns:
str: The path for the compliance framework output directory.
Example:
>>> _generate_compliance_output_directory("/tmp", "aws", "tenant-1234", "scan-5678", "threatscore")
'/tmp/tenant-1234/scan-5678/threatscore/prowler-output-aws-20230215123456'
>>> _generate_compliance_output_directory("/tmp", "aws", "tenant-1234", "scan-5678", "ens")
'/tmp/tenant-1234/scan-5678/ens/prowler-output-aws-20230215123456'
>>> _generate_compliance_output_directory("/tmp", "aws", "tenant-1234", "scan-5678", "nis2")
'/tmp/tenant-1234/scan-5678/nis2/prowler-output-aws-20230215123456'
"""
return _build_output_path(
output_directory,
prowler_provider,
tenant_id,
scan_id,
subdirectory=compliance_framework,
compliance_path = (
f"{output_directory}/{tenant_id}/{scan_id}/compliance/prowler-output-"
f"{prowler_provider_sanitized}-{timestamp}"
)
os.makedirs("/".join(compliance_path.split("/")[:-1]), exist_ok=True)
def _generate_output_directory(
output_directory: str,
prowler_provider: str,
tenant_id: str,
scan_id: str,
) -> tuple[str, str]:
"""
Generate file system paths for the standard and compliance output directories of a prowler scan.
This function constructs both the standard output directory path and the compliance
output directory path by combining a base temporary output directory, the tenant ID,
the scan ID, and details about the prowler provider along with a timestamp.
Args:
output_directory (str): The base output directory.
prowler_provider (str): An identifier or descriptor for the prowler provider.
Typically, this is a string indicating the provider (e.g., "aws").
tenant_id (str): The unique identifier for the tenant.
scan_id (str): The unique identifier for the scan.
Returns:
tuple[str, str]: A tuple containing (standard_path, compliance_path).
Example:
>>> _generate_output_directory("/tmp", "aws", "tenant-1234", "scan-5678")
('/tmp/tenant-1234/scan-5678/prowler-output-aws-20230215123456',
'/tmp/tenant-1234/scan-5678/compliance/prowler-output-aws-20230215123456')
"""
standard_path = _build_output_path(
output_directory, prowler_provider, tenant_id, scan_id
)
compliance_path = _build_output_path(
output_directory,
prowler_provider,
tenant_id,
scan_id,
subdirectory="compliance",
threatscore_path = (
f"{output_directory}/{tenant_id}/{scan_id}/threatscore/prowler-output-"
f"{prowler_provider_sanitized}-{timestamp}"
)
os.makedirs("/".join(threatscore_path.split("/")[:-1]), exist_ok=True)
return standard_path, compliance_path
return path, compliance_path, threatscore_path
@@ -1,8 +1,6 @@
from typing import Dict
from typing import Dict, Set
import boto3
import openai
from botocore.exceptions import BotoCoreError, ClientError
from celery.utils.log import get_task_logger
from api.models import LighthouseProviderConfiguration, LighthouseProviderModels
@@ -32,71 +30,16 @@ def _extract_openai_api_key(
return api_key
def _extract_openai_compatible_params(
provider_cfg: LighthouseProviderConfiguration,
) -> Dict[str, str] | None:
"""
Extract base_url and api_key for OpenAI-compatible providers.
"""
creds = provider_cfg.credentials_decoded
base_url = provider_cfg.base_url
if not isinstance(creds, dict):
return None
api_key = creds.get("api_key")
if not isinstance(api_key, str) or not api_key:
return None
if not isinstance(base_url, str) or not base_url:
return None
return {"base_url": base_url, "api_key": api_key}
def _extract_bedrock_credentials(
provider_cfg: LighthouseProviderConfiguration,
) -> Dict[str, str] | None:
"""
Safely extract AWS Bedrock credentials from a provider configuration.
Args:
provider_cfg (LighthouseProviderConfiguration): The provider configuration instance
containing the credentials.
Returns:
Dict[str, str] | None: Dictionary with 'access_key_id', 'secret_access_key', and
'region' if present and valid, otherwise None.
"""
creds = provider_cfg.credentials_decoded
if not isinstance(creds, dict):
return None
access_key_id = creds.get("access_key_id")
secret_access_key = creds.get("secret_access_key")
region = creds.get("region")
# Validate all required fields are present and are strings
if (
not isinstance(access_key_id, str)
or not access_key_id
or not isinstance(secret_access_key, str)
or not secret_access_key
or not isinstance(region, str)
or not region
):
return None
return {
"access_key_id": access_key_id,
"secret_access_key": secret_access_key,
"region": region,
}
def check_lighthouse_provider_connection(provider_config_id: str) -> Dict:
"""
Validate a Lighthouse provider configuration by calling the provider API and
toggle its active state accordingly.
Currently supports the OpenAI provider by invoking `models.list` to verify that
the provided credentials are valid.
Args:
provider_config_id: The primary key of the `LighthouseProviderConfiguration`
provider_config_id (str): The primary key of the `LighthouseProviderConfiguration`
to validate.
Returns:
@@ -112,238 +55,42 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> Dict:
"""
provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id)
# TODO: Add support for other providers
if (
provider_cfg.provider_type
!= LighthouseProviderConfiguration.LLMProviderChoices.OPENAI
):
return {"connected": False, "error": "Unsupported provider type"}
api_key = _extract_openai_api_key(provider_cfg)
if not api_key:
provider_cfg.is_active = False
provider_cfg.save()
return {"connected": False, "error": "API key is invalid or missing"}
try:
if (
provider_cfg.provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI
):
api_key = _extract_openai_api_key(provider_cfg)
if not api_key:
provider_cfg.is_active = False
provider_cfg.save()
return {"connected": False, "error": "API key is invalid or missing"}
# Test connection by listing models
client = openai.OpenAI(api_key=api_key)
_ = client.models.list()
elif (
provider_cfg.provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
):
bedrock_creds = _extract_bedrock_credentials(provider_cfg)
if not bedrock_creds:
provider_cfg.is_active = False
provider_cfg.save()
return {
"connected": False,
"error": "AWS credentials are invalid or missing",
}
# Test connection by listing foundation models
bedrock_client = boto3.client(
"bedrock",
aws_access_key_id=bedrock_creds["access_key_id"],
aws_secret_access_key=bedrock_creds["secret_access_key"],
region_name=bedrock_creds["region"],
)
_ = bedrock_client.list_foundation_models()
elif (
provider_cfg.provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE
):
params = _extract_openai_compatible_params(provider_cfg)
if not params:
provider_cfg.is_active = False
provider_cfg.save()
return {
"connected": False,
"error": "Base URL or API key is invalid or missing",
}
# Test connection using OpenAI SDK with custom base_url
# Note: base_url should include version (e.g., https://openrouter.ai/api/v1)
client = openai.OpenAI(
api_key=params["api_key"],
base_url=params["base_url"],
)
_ = client.models.list()
else:
return {"connected": False, "error": "Unsupported provider type"}
# Connection successful
client = openai.OpenAI(api_key=api_key)
_ = client.models.list()
provider_cfg.is_active = True
provider_cfg.save()
return {"connected": True, "error": None}
except Exception as e:
logger.warning(
"%s connection check failed: %s", provider_cfg.provider_type, str(e)
)
logger.warning("OpenAI connection check failed: %s", str(e))
provider_cfg.is_active = False
provider_cfg.save()
return {"connected": False, "error": str(e)}
def _fetch_openai_models(api_key: str) -> Dict[str, str]:
"""
Fetch available models from OpenAI API.
Args:
api_key: OpenAI API key for authentication.
Returns:
Dict mapping model_id to model_name. For OpenAI, both are the same
as the API doesn't provide separate display names.
Raises:
Exception: If the API call fails.
"""
client = openai.OpenAI(api_key=api_key)
models = client.models.list()
# OpenAI uses model.id for both ID and display name
return {m.id: m.id for m in getattr(models, "data", [])}
def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, str]:
"""
Fetch available models from an OpenAI-compatible API using the OpenAI SDK.
Returns a mapping of model_id -> model_name. Prefers the 'name' attribute
if available (e.g., from OpenRouter), otherwise falls back to 'id'.
Note: base_url should include version (e.g., https://openrouter.ai/api/v1)
"""
client = openai.OpenAI(api_key=api_key, base_url=base_url)
models = client.models.list()
available_models: Dict[str, str] = {}
for model in models.data:
model_id = model.id
# Prefer provider-supplied human-friendly name when available
name = getattr(model, "name", None)
if name:
available_models[model_id] = name
else:
available_models[model_id] = model_id
return available_models
def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]:
"""
Fetch available models from AWS Bedrock with entitlement verification.
This function:
1. Lists foundation models with TEXT modality support
2. Lists inference profiles with TEXT modality support
3. Verifies user has entitlement access to each model
Args:
bedrock_creds: Dictionary with 'access_key_id', 'secret_access_key', and 'region'.
Returns:
Dict mapping model_id to model_name for all accessible models.
Raises:
BotoCoreError, ClientError: If AWS API calls fail.
"""
bedrock_client = boto3.client(
"bedrock",
aws_access_key_id=bedrock_creds["access_key_id"],
aws_secret_access_key=bedrock_creds["secret_access_key"],
region_name=bedrock_creds["region"],
)
models_to_check: Dict[str, str] = {}
# Step 1: Get foundation models with TEXT modality
foundation_response = bedrock_client.list_foundation_models()
model_summaries = foundation_response.get("modelSummaries", [])
for model in model_summaries:
# Check if model supports TEXT input and output modality
input_modalities = model.get("inputModalities", [])
output_modalities = model.get("outputModalities", [])
if "TEXT" not in input_modalities or "TEXT" not in output_modalities:
continue
model_id = model.get("modelId")
if not model_id:
continue
inference_types = model.get("inferenceTypesSupported", [])
# Only include models with ON_DEMAND inference support
if "ON_DEMAND" in inference_types:
models_to_check[model_id] = model["modelName"]
# Step 2: Get inference profiles
try:
inference_profiles_response = bedrock_client.list_inference_profiles()
inference_profiles = inference_profiles_response.get(
"inferenceProfileSummaries", []
)
for profile in inference_profiles:
# Check if profile supports TEXT modality
input_modalities = profile.get("inputModalities", [])
output_modalities = profile.get("outputModalities", [])
if "TEXT" not in input_modalities or "TEXT" not in output_modalities:
continue
profile_id = profile.get("inferenceProfileId")
if profile_id:
models_to_check[profile_id] = profile["inferenceProfileName"]
except (BotoCoreError, ClientError) as e:
logger.info(
"Could not fetch inference profiles in %s: %s",
bedrock_creds["region"],
str(e),
)
# Step 3: Verify entitlement availability for each model
available_models: Dict[str, str] = {}
for model_id, model_name in models_to_check.items():
try:
availability = bedrock_client.get_foundation_model_availability(
modelId=model_id
)
entitlement = availability.get("entitlementAvailability")
# Only include models user has access to
if entitlement == "AVAILABLE":
available_models[model_id] = model_name
else:
logger.debug(
"Skipping model %s - entitlement status: %s", model_id, entitlement
)
except (BotoCoreError, ClientError) as e:
logger.debug(
"Could not check availability for model %s: %s", model_id, str(e)
)
continue
return available_models
def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict:
"""
Refresh the catalog of models for a Lighthouse provider configuration.
Fetches the current list of models from the provider, upserts entries into
`LighthouseProviderModels`, and deletes stale entries no longer returned.
For the OpenAI provider, this fetches the current list of models, upserts entries
into `LighthouseProviderModels`, and deletes stale entries no longer returned by
the provider.
Args:
provider_config_id: The primary key of the `LighthouseProviderConfiguration`
provider_config_id (str): The primary key of the `LighthouseProviderConfiguration`
whose models should be refreshed.
Returns:
@@ -357,81 +104,45 @@ def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict:
LighthouseProviderConfiguration.DoesNotExist: If no configuration exists with the given ID.
"""
provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id)
fetched_models: Dict[str, str] = {}
# Fetch models from the appropriate provider
if (
provider_cfg.provider_type
!= LighthouseProviderConfiguration.LLMProviderChoices.OPENAI
):
return {
"created": 0,
"updated": 0,
"deleted": 0,
"error": "Unsupported provider type",
}
api_key = _extract_openai_api_key(provider_cfg)
if not api_key:
return {
"created": 0,
"updated": 0,
"deleted": 0,
"error": "API key is invalid or missing",
}
try:
if (
provider_cfg.provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI
):
api_key = _extract_openai_api_key(provider_cfg)
if not api_key:
return {
"created": 0,
"updated": 0,
"deleted": 0,
"error": "API key is invalid or missing",
}
fetched_models = _fetch_openai_models(api_key)
elif (
provider_cfg.provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
):
bedrock_creds = _extract_bedrock_credentials(provider_cfg)
if not bedrock_creds:
return {
"created": 0,
"updated": 0,
"deleted": 0,
"error": "AWS credentials are invalid or missing",
}
fetched_models = _fetch_bedrock_models(bedrock_creds)
elif (
provider_cfg.provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE
):
params = _extract_openai_compatible_params(provider_cfg)
if not params:
return {
"created": 0,
"updated": 0,
"deleted": 0,
"error": "Base URL or API key is invalid or missing",
}
fetched_models = _fetch_openai_compatible_models(
params["base_url"], params["api_key"]
)
else:
return {
"created": 0,
"updated": 0,
"deleted": 0,
"error": "Unsupported provider type",
}
except Exception as e:
logger.warning(
"Unexpected error refreshing %s models: %s",
provider_cfg.provider_type,
str(e),
)
client = openai.OpenAI(api_key=api_key)
models = client.models.list()
fetched_ids: Set[str] = {m.id for m in getattr(models, "data", [])}
except Exception as e: # noqa: BLE001
logger.warning("OpenAI models refresh failed: %s", str(e))
return {"created": 0, "updated": 0, "deleted": 0, "error": str(e)}
# Upsert models into the catalog
created = 0
updated = 0
for model_id, model_name in fetched_models.items():
for model_id in fetched_ids:
obj, was_created = LighthouseProviderModels.objects.update_or_create(
tenant_id=provider_cfg.tenant_id,
provider_configuration=provider_cfg,
model_id=model_id,
defaults={
"model_name": model_name,
"model_name": model_id, # OpenAI doesn't return a separate display name
"default_parameters": {},
},
)
@@ -445,7 +156,7 @@ def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict:
LighthouseProviderModels.objects.filter(
tenant_id=provider_cfg.tenant_id, provider_configuration=provider_cfg
)
.exclude(model_id__in=fetched_models.keys())
.exclude(model_id__in=fetched_ids)
.delete()
)
-64
View File
@@ -1,64 +0,0 @@
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
from tasks.utils import batched
from api.db_utils import rls_transaction
from api.models import Finding, MuteRule
logger = get_task_logger(__name__)
def mute_historical_findings(tenant_id: str, mute_rule_id: str):
"""
Mute historical findings that match the given mute rule.
This function processes findings in batches, updating their muted status
and adding the mute reason.
Args:
tenant_id (str): The tenant ID for RLS context
mute_rule_id (str): The ID of the mute rule to apply
Returns:
dict: Summary of the muting operation with findings_muted count
"""
findings_muted_count = 0
# Get the list of UIDs to mute and the reason
with rls_transaction(tenant_id):
mute_rule = MuteRule.objects.get(id=mute_rule_id, tenant_id=tenant_id)
finding_uids = mute_rule.finding_uids
mute_reason = mute_rule.reason
muted_at = mute_rule.inserted_at
# Query findings that match the UIDs and are not already muted
with rls_transaction(tenant_id):
findings_to_mute = Finding.objects.filter(
tenant_id=tenant_id, uid__in=finding_uids, muted=False
)
total_findings = findings_to_mute.count()
logger.info(
f"Processing {total_findings} findings for mute rule {mute_rule_id}"
)
if total_findings > 0:
for batch, is_last in batched(
findings_to_mute.iterator(), DJANGO_FINDINGS_BATCH_SIZE
):
batch_ids = [f.id for f in batch]
updated_count = Finding.all_objects.filter(
id__in=batch_ids, tenant_id=tenant_id
).update(
muted=True,
muted_at=muted_at,
muted_reason=mute_reason,
)
findings_muted_count += updated_count
logger.info(f"Muted {findings_muted_count} findings for rule {mute_rule_id}")
return {
"findings_muted": findings_muted_count,
"rule_id": mute_rule_id,
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-214
View File
@@ -1,214 +0,0 @@
from celery.utils.log import get_task_logger
from tasks.jobs.threatscore_utils import (
_aggregate_requirement_statistics_from_database,
_calculate_requirements_data_from_statistics,
)
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Provider, StatusChoices
from prowler.lib.check.compliance_models import Compliance
logger = get_task_logger(__name__)
def compute_threatscore_metrics(
tenant_id: str,
scan_id: str,
provider_id: str,
compliance_id: str,
min_risk_level: int = 4,
) -> dict:
"""
Compute ThreatScore metrics for a given scan.
This function calculates all the metrics needed for a ThreatScore snapshot:
- Overall ThreatScore percentage
- Section-by-section scores
- Critical failed requirements (risk >= min_risk_level)
- Summary statistics (requirements and findings counts)
Args:
tenant_id (str): The tenant ID for Row-Level Security context.
scan_id (str): The ID of the scan to analyze.
provider_id (str): The ID of the provider used in the scan.
compliance_id (str): Compliance framework ID (e.g., "prowler_threatscore_aws").
min_risk_level (int): Minimum risk level for critical requirements. Defaults to 4.
Returns:
dict: A dictionary containing:
- overall_score (float): Overall ThreatScore percentage (0-100)
- section_scores (dict): Section name -> score percentage mapping
- critical_requirements (list): List of critical failed requirement dicts
- total_requirements (int): Total number of requirements
- passed_requirements (int): Number of PASS requirements
- failed_requirements (int): Number of FAIL requirements
- manual_requirements (int): Number of MANUAL requirements
- total_findings (int): Total findings count
- passed_findings (int): Passed findings count
- failed_findings (int): Failed findings count
Example:
>>> metrics = compute_threatscore_metrics(
... tenant_id="tenant-123",
... scan_id="scan-456",
... provider_id="provider-789",
... compliance_id="prowler_threatscore_aws"
... )
>>> print(f"Overall ThreatScore: {metrics['overall_score']:.2f}%")
"""
# Get provider and compliance information
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
provider_obj = Provider.objects.get(id=provider_id)
provider_type = provider_obj.provider
frameworks_bulk = Compliance.get_bulk(provider_type)
compliance_obj = frameworks_bulk[compliance_id]
# Aggregate requirement statistics from database
requirement_statistics_by_check_id = (
_aggregate_requirement_statistics_from_database(tenant_id, scan_id)
)
# Calculate requirements data using aggregated statistics
attributes_by_requirement_id, requirements_list = (
_calculate_requirements_data_from_statistics(
compliance_obj, requirement_statistics_by_check_id
)
)
# Initialize metrics
overall_numerator = 0
overall_denominator = 0
overall_has_findings = False
sections_data = {}
total_requirements = len(requirements_list)
passed_requirements = 0
failed_requirements = 0
manual_requirements = 0
total_findings = 0
passed_findings = 0
failed_findings = 0
critical_requirements_list = []
# Process each requirement
for requirement in requirements_list:
requirement_id = requirement["id"]
requirement_status = requirement["attributes"]["status"]
requirement_attributes = attributes_by_requirement_id.get(requirement_id, {})
# Count requirements by status
if requirement_status == StatusChoices.PASS:
passed_requirements += 1
elif requirement_status == StatusChoices.FAIL:
failed_requirements += 1
elif requirement_status == StatusChoices.MANUAL:
manual_requirements += 1
# Get findings data
req_passed_findings = requirement["attributes"].get("passed_findings", 0)
req_total_findings = requirement["attributes"].get("total_findings", 0)
# Accumulate findings counts
total_findings += req_total_findings
passed_findings += req_passed_findings
failed_findings += req_total_findings - req_passed_findings
# Skip requirements with no findings
if req_total_findings == 0:
continue
overall_has_findings = True
# Get requirement metadata
metadata = requirement_attributes.get("attributes", {}).get(
"req_attributes", []
)
if not metadata or len(metadata) == 0:
continue
m = metadata[0]
risk_level = getattr(m, "LevelOfRisk", 0)
weight = getattr(m, "Weight", 0)
section = getattr(m, "Section", "Unknown")
# Calculate ThreatScore components using formula from UI
rate_i = req_passed_findings / req_total_findings
rfac_i = 1 + 0.25 * risk_level
# Update overall score
overall_numerator += rate_i * req_total_findings * weight * rfac_i
overall_denominator += req_total_findings * weight * rfac_i
# Update section scores
if section not in sections_data:
sections_data[section] = {
"numerator": 0,
"denominator": 0,
"has_findings": False,
}
sections_data[section]["has_findings"] = True
sections_data[section]["numerator"] += (
rate_i * req_total_findings * weight * rfac_i
)
sections_data[section]["denominator"] += req_total_findings * weight * rfac_i
# Identify critical failed requirements
if requirement_status == StatusChoices.FAIL and risk_level >= min_risk_level:
critical_requirements_list.append(
{
"requirement_id": requirement_id,
"title": getattr(m, "Title", "N/A"),
"section": section,
"subsection": getattr(m, "SubSection", "N/A"),
"risk_level": risk_level,
"weight": weight,
"passed_findings": req_passed_findings,
"total_findings": req_total_findings,
"description": getattr(m, "AttributeDescription", "N/A"),
}
)
# Calculate overall ThreatScore
if not overall_has_findings:
overall_score = 100.0
elif overall_denominator > 0:
overall_score = (overall_numerator / overall_denominator) * 100
else:
overall_score = 0.0
# Calculate section scores
section_scores = {}
for section, data in sections_data.items():
if data["has_findings"] and data["denominator"] > 0:
section_scores[section] = (data["numerator"] / data["denominator"]) * 100
else:
section_scores[section] = 100.0
# Sort critical requirements by risk level (desc) and weight (desc)
critical_requirements_list.sort(
key=lambda x: (x["risk_level"], x["weight"]), reverse=True
)
logger.info(
f"ThreatScore computed: {overall_score:.2f}% "
f"({passed_requirements}/{total_requirements} requirements passed, "
f"{len(critical_requirements_list)} critical failures)"
)
return {
"overall_score": round(overall_score, 2),
"section_scores": {k: round(v, 2) for k, v in section_scores.items()},
"critical_requirements": critical_requirements_list,
"total_requirements": total_requirements,
"passed_requirements": passed_requirements,
"failed_requirements": failed_requirements,
"manual_requirements": manual_requirements,
"total_findings": total_findings,
"passed_findings": passed_findings,
"failed_findings": failed_findings,
}
@@ -1,239 +0,0 @@
from collections import defaultdict
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
from django.db.models import Count, Q
from tasks.utils import batched
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import Finding, StatusChoices
from prowler.lib.outputs.finding import Finding as FindingOutput
logger = get_task_logger(__name__)
def _aggregate_requirement_statistics_from_database(
tenant_id: str, scan_id: str
) -> dict[str, dict[str, int]]:
"""
Aggregate finding statistics by check_id using database aggregation.
This function uses Django ORM aggregation to calculate pass/fail statistics
entirely in the database, avoiding the need to load findings into memory.
Args:
tenant_id (str): The tenant ID for Row-Level Security context.
scan_id (str): The ID of the scan to retrieve findings for.
Returns:
dict[str, dict[str, int]]: Dictionary mapping check_id to statistics:
- 'passed' (int): Number of passed findings for this check
- 'total' (int): Total number of findings for this check
Example:
{
'aws_iam_user_mfa_enabled': {'passed': 10, 'total': 15},
'aws_s3_bucket_public_access': {'passed': 0, 'total': 5}
}
"""
requirement_statistics_by_check_id = {}
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
aggregated_statistics_queryset = (
Finding.all_objects.filter(
tenant_id=tenant_id, scan_id=scan_id, muted=False
)
.values("check_id")
.annotate(
total_findings=Count(
"id",
filter=Q(status__in=[StatusChoices.PASS, StatusChoices.FAIL]),
),
passed_findings=Count("id", filter=Q(status=StatusChoices.PASS)),
)
)
for aggregated_stat in aggregated_statistics_queryset:
check_id = aggregated_stat["check_id"]
requirement_statistics_by_check_id[check_id] = {
"passed": aggregated_stat["passed_findings"],
"total": aggregated_stat["total_findings"],
}
logger.info(
f"Aggregated statistics for {len(requirement_statistics_by_check_id)} unique checks"
)
return requirement_statistics_by_check_id
def _calculate_requirements_data_from_statistics(
compliance_obj, requirement_statistics_by_check_id: dict[str, dict[str, int]]
) -> tuple[dict[str, dict], list[dict]]:
"""
Calculate requirement status and statistics using pre-aggregated database statistics.
Args:
compliance_obj: The compliance framework object containing requirements.
requirement_statistics_by_check_id (dict[str, dict[str, int]]): Pre-aggregated statistics
mapping check_id to {'passed': int, 'total': int} counts.
Returns:
tuple[dict[str, dict], list[dict]]: A tuple containing:
- attributes_by_requirement_id: Dictionary mapping requirement IDs to their attributes.
- requirements_list: List of requirement dictionaries with status and statistics.
"""
attributes_by_requirement_id = {}
requirements_list = []
compliance_framework = getattr(compliance_obj, "Framework", "N/A")
compliance_version = getattr(compliance_obj, "Version", "N/A")
for requirement in compliance_obj.Requirements:
requirement_id = requirement.Id
requirement_description = getattr(requirement, "Description", "")
requirement_checks = getattr(requirement, "Checks", [])
requirement_attributes = getattr(requirement, "Attributes", [])
attributes_by_requirement_id[requirement_id] = {
"attributes": {
"req_attributes": requirement_attributes,
"checks": requirement_checks,
},
"description": requirement_description,
}
total_passed_findings = 0
total_findings_count = 0
for check_id in requirement_checks:
if check_id in requirement_statistics_by_check_id:
check_statistics = requirement_statistics_by_check_id[check_id]
total_findings_count += check_statistics["total"]
total_passed_findings += check_statistics["passed"]
if total_findings_count > 0:
if total_passed_findings == total_findings_count:
requirement_status = StatusChoices.PASS
else:
requirement_status = StatusChoices.FAIL
else:
requirement_status = StatusChoices.MANUAL
requirements_list.append(
{
"id": requirement_id,
"attributes": {
"framework": compliance_framework,
"version": compliance_version,
"status": requirement_status,
"description": requirement_description,
"passed_findings": total_passed_findings,
"total_findings": total_findings_count,
},
}
)
return attributes_by_requirement_id, requirements_list
def _load_findings_for_requirement_checks(
tenant_id: str,
scan_id: str,
check_ids: list[str],
prowler_provider,
findings_cache: dict[str, list[FindingOutput]] | None = None,
) -> dict[str, list[FindingOutput]]:
"""
Load findings for specific check IDs on-demand with optional caching.
This function loads only the findings needed for a specific set of checks,
minimizing memory usage by avoiding loading all findings at once. This is used
when generating detailed findings tables for specific requirements in the PDF.
Supports optional caching to avoid duplicate queries when generating multiple
reports for the same scan.
Args:
tenant_id (str): The tenant ID for Row-Level Security context.
scan_id (str): The ID of the scan to retrieve findings for.
check_ids (list[str]): List of check IDs to load findings for.
prowler_provider: The initialized Prowler provider instance.
findings_cache (dict, optional): Cache of already loaded findings.
If provided, checks are first looked up in cache before querying database.
Returns:
dict[str, list[FindingOutput]]: Dictionary mapping check_id to list of FindingOutput objects.
Example:
{
'aws_iam_user_mfa_enabled': [FindingOutput(...), FindingOutput(...)],
'aws_s3_bucket_public_access': [FindingOutput(...)]
}
"""
findings_by_check_id = defaultdict(list)
if not check_ids:
return dict(findings_by_check_id)
# Initialize cache if not provided
if findings_cache is None:
findings_cache = {}
# Separate cached and non-cached check_ids
check_ids_to_load = []
cache_hits = 0
cache_misses = 0
for check_id in check_ids:
if check_id in findings_cache:
# Reuse from cache
findings_by_check_id[check_id] = findings_cache[check_id]
cache_hits += 1
else:
# Need to load from database
check_ids_to_load.append(check_id)
cache_misses += 1
if cache_hits > 0:
logger.info(
f"Findings cache: {cache_hits} hits, {cache_misses} misses "
f"({cache_hits / (cache_hits + cache_misses) * 100:.1f}% hit rate)"
)
# If all check_ids were in cache, return early
if not check_ids_to_load:
return dict(findings_by_check_id)
logger.info(f"Loading findings for {len(check_ids_to_load)} checks on-demand")
findings_queryset = (
Finding.all_objects.filter(
tenant_id=tenant_id, scan_id=scan_id, check_id__in=check_ids_to_load
)
.order_by("uid")
.iterator()
)
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
for batch, is_last_batch in batched(
findings_queryset, DJANGO_FINDINGS_BATCH_SIZE
):
for finding_model in batch:
finding_output = FindingOutput.transform_api_finding(
finding_model, prowler_provider
)
findings_by_check_id[finding_output.check_id].append(finding_output)
# Update cache with newly loaded findings
if finding_output.check_id not in findings_cache:
findings_cache[finding_output.check_id] = []
findings_cache[finding_output.check_id].append(finding_output)
total_findings_loaded = sum(
len(findings) for findings in findings_by_check_id.values()
)
logger.info(
f"Loaded {total_findings_loaded} findings for {len(findings_by_check_id)} checks"
)
return dict(findings_by_check_id)
+9 -65
View File
@@ -8,10 +8,7 @@ from celery.utils.log import get_task_logger
from config.celery import RLSTask
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY
from django_celery_beat.models import PeriodicTask
from tasks.jobs.backfill import (
backfill_compliance_summaries,
backfill_resource_scan_summaries,
)
from tasks.jobs.backfill import backfill_resource_scan_summaries
from tasks.jobs.connection import (
check_integration_connection,
check_lighthouse_connection,
@@ -34,8 +31,7 @@ from tasks.jobs.lighthouse_providers import (
check_lighthouse_provider_connection,
refresh_lighthouse_provider_models,
)
from tasks.jobs.muting import mute_historical_findings
from tasks.jobs.report import generate_compliance_reports_job
from tasks.jobs.report import generate_threatscore_report_job
from tasks.jobs.scan import (
aggregate_findings,
create_compliance_requirements,
@@ -75,8 +71,7 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id
),
group(
# Use optimized task that generates both reports with shared queries
generate_compliance_reports_task.si(
generate_threatscore_report_task.si(
tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id
),
check_integrations_task.si(
@@ -320,7 +315,7 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
frameworks_bulk = Compliance.get_bulk(provider_type)
frameworks_avail = get_compliance_frameworks(provider_type)
out_dir, comp_dir = _generate_output_directory(
out_dir, comp_dir, _ = _generate_output_directory(
DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id
)
@@ -498,21 +493,6 @@ def backfill_scan_resource_summaries_task(tenant_id: str, scan_id: str):
return backfill_resource_scan_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="backfill-compliance-summaries", queue="backfill")
def backfill_compliance_summaries_task(tenant_id: str, scan_id: str):
"""
Tries to backfill compliance overview summaries for a completed scan.
This task aggregates compliance requirement data across regions
to create pre-computed summary records for fast compliance overview queries.
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
"""
return backfill_compliance_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(base=RLSTask, name="scan-compliance-overviews", queue="compliance")
def create_compliance_requirements_task(tenant_id: str, scan_id: str):
"""
@@ -687,53 +667,17 @@ def jira_integration_task(
@shared_task(
base=RLSTask,
name="scan-compliance-reports",
name="scan-threatscore-report",
queue="scan-reports",
)
def generate_compliance_reports_task(tenant_id: str, scan_id: str, provider_id: str):
def generate_threatscore_report_task(tenant_id: str, scan_id: str, provider_id: str):
"""
Optimized task to generate ThreatScore, ENS, and NIS2 reports with shared queries.
This task is more efficient than running separate report tasks because it reuses database queries:
- Provider object fetched once (instead of three times)
- Requirement statistics aggregated once (instead of three times)
- Can reduce database load by up to 50-70%
Task to generate a threatscore report for a given scan.
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
provider_id (str): The provider identifier.
Returns:
dict: Results for all reports containing upload status and paths.
"""
return generate_compliance_reports_job(
tenant_id=tenant_id,
scan_id=scan_id,
provider_id=provider_id,
generate_threatscore=True,
generate_ens=True,
generate_nis2=True,
return generate_threatscore_report_job(
tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id
)
@shared_task(name="findings-mute-historical")
def mute_historical_findings_task(tenant_id: str, mute_rule_id: str):
"""
Background task to mute all historical findings matching a mute rule.
This task processes findings in batches to avoid memory issues with large datasets.
It updates the Finding.muted, Finding.muted_at, and Finding.muted_reason fields
for all findings whose UID is in the mute rule's finding_uids list.
Args:
tenant_id (str): The tenant ID for RLS context.
mute_rule_id (str): The primary key of the MuteRule to apply.
Returns:
dict: A dictionary containing:
- 'findings_muted' (int): Total number of findings muted.
- 'rule_id' (str): The mute rule ID.
- 'status' (str): Final status ('completed').
"""
return mute_historical_findings(tenant_id, mute_rule_id)
+32 -127
View File
@@ -1,53 +1,43 @@
from uuid import uuid4
import pytest
from tasks.jobs.backfill import (
backfill_compliance_summaries,
backfill_resource_scan_summaries,
)
from tasks.jobs.backfill import backfill_resource_scan_summaries
from api.models import (
ComplianceOverviewSummary,
ResourceScanSummary,
Scan,
StateChoices,
)
@pytest.fixture(scope="function")
def resource_scan_summary_data(scans_fixture):
scan = scans_fixture[0]
return ResourceScanSummary.objects.create(
tenant_id=scan.tenant_id,
scan_id=scan.id,
resource_id=str(uuid4()),
service="aws",
region="us-east-1",
resource_type="instance",
)
@pytest.fixture(scope="function")
def get_not_completed_scans(providers_fixture):
provider_id = providers_fixture[0].id
tenant_id = providers_fixture[0].tenant_id
scan_1 = Scan.objects.create(
tenant_id=tenant_id,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.EXECUTING,
provider_id=provider_id,
)
scan_2 = Scan.objects.create(
tenant_id=tenant_id,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.AVAILABLE,
provider_id=provider_id,
)
return scan_1, scan_2
from api.models import ResourceScanSummary, Scan, StateChoices
@pytest.mark.django_db
class TestBackfillResourceScanSummaries:
@pytest.fixture(scope="function")
def resource_scan_summary_data(self, scans_fixture):
scan = scans_fixture[0]
return ResourceScanSummary.objects.create(
tenant_id=scan.tenant_id,
scan_id=scan.id,
resource_id=str(uuid4()),
service="aws",
region="us-east-1",
resource_type="instance",
)
@pytest.fixture(scope="function")
def get_not_completed_scans(self, providers_fixture):
provider_id = providers_fixture[0].id
tenant_id = providers_fixture[0].tenant_id
scan_1 = Scan.objects.create(
tenant_id=tenant_id,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.EXECUTING,
provider_id=provider_id,
)
scan_2 = Scan.objects.create(
tenant_id=tenant_id,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.AVAILABLE,
provider_id=provider_id,
)
return scan_1, scan_2
def test_already_backfilled(self, resource_scan_summary_data):
tenant_id = resource_scan_summary_data.tenant_id
scan_id = resource_scan_summary_data.scan_id
@@ -87,88 +77,3 @@ class TestBackfillResourceScanSummaries:
assert summary.service == resource.service
assert summary.region == resource.region
assert summary.resource_type == resource.type
def test_no_resources_to_backfill(self, scans_fixture):
scan = scans_fixture[1] # Failed scan with no findings/resources
tenant_id = str(scan.tenant_id)
scan_id = str(scan.id)
result = backfill_resource_scan_summaries(tenant_id, scan_id)
assert result == {"status": "no resources to backfill"}
@pytest.mark.django_db
class TestBackfillComplianceSummaries:
def test_already_backfilled(self, scans_fixture):
scan = scans_fixture[0]
tenant_id = str(scan.tenant_id)
ComplianceOverviewSummary.objects.create(
tenant_id=scan.tenant_id,
scan=scan,
compliance_id="aws_account_security_onboarding_aws",
requirements_passed=1,
requirements_failed=0,
requirements_manual=0,
total_requirements=1,
)
result = backfill_compliance_summaries(tenant_id, str(scan.id))
assert result == {"status": "already backfilled"}
def test_not_completed_scan(self, get_not_completed_scans):
for scan in get_not_completed_scans:
result = backfill_compliance_summaries(str(scan.tenant_id), str(scan.id))
assert result == {"status": "scan is not completed"}
def test_no_compliance_data(self, scans_fixture):
scan = scans_fixture[1] # Failed scan with no compliance rows
result = backfill_compliance_summaries(str(scan.tenant_id), str(scan.id))
assert result == {"status": "no compliance data to backfill"}
def test_backfill_creates_compliance_summaries(
self, tenants_fixture, scans_fixture, compliance_requirements_overviews_fixture
):
tenant = tenants_fixture[0]
scan = scans_fixture[0]
result = backfill_compliance_summaries(str(tenant.id), str(scan.id))
expected = {
"aws_account_security_onboarding_aws": {
"requirements_passed": 1,
"requirements_failed": 1,
"requirements_manual": 1,
"total_requirements": 3,
},
"cis_1.4_aws": {
"requirements_passed": 0,
"requirements_failed": 1,
"requirements_manual": 0,
"total_requirements": 1,
},
"mitre_attack_aws": {
"requirements_passed": 0,
"requirements_failed": 1,
"requirements_manual": 0,
"total_requirements": 1,
},
}
assert result == {"status": "backfilled", "inserted": len(expected)}
summaries = ComplianceOverviewSummary.objects.filter(
tenant_id=str(tenant.id), scan_id=str(scan.id)
)
assert summaries.count() == len(expected)
for summary in summaries:
assert summary.compliance_id in expected
expected_counts = expected[summary.compliance_id]
assert summary.requirements_passed == expected_counts["requirements_passed"]
assert summary.requirements_failed == expected_counts["requirements_failed"]
assert summary.requirements_manual == expected_counts["requirements_manual"]
assert summary.total_requirements == expected_counts["total_requirements"]
+8 -39
View File
@@ -9,7 +9,6 @@ import pytest
from botocore.exceptions import ClientError
from tasks.jobs.export import (
_compress_output_files,
_generate_compliance_output_directory,
_generate_output_directory,
_upload_to_s3,
get_s3_client,
@@ -148,11 +147,10 @@ class TestOutputs:
)
mock_logger.assert_called()
@patch("tasks.jobs.export.set_output_timestamp")
@patch("tasks.jobs.export.rls_transaction")
@patch("tasks.jobs.export.Scan")
def test_generate_output_directory_creates_paths(
self, mock_scan, mock_rls_transaction, mock_set_timestamp, tmpdir
self, mock_scan, mock_rls_transaction, tmpdir
):
# Mock the scan object with a started_at timestamp
mock_scan_instance = MagicMock()
@@ -170,40 +168,22 @@ class TestOutputs:
provider = "aws"
expected_timestamp = "20230615103045"
# Test _generate_output_directory (returns standard and compliance paths)
path, compliance = _generate_output_directory(
path, compliance, threatscore = _generate_output_directory(
base_dir, provider, tenant_id, scan_id
)
assert os.path.isdir(os.path.dirname(path))
assert os.path.isdir(os.path.dirname(compliance))
assert os.path.isdir(os.path.dirname(threatscore))
assert path.endswith(f"{provider}-{expected_timestamp}")
assert compliance.endswith(f"{provider}-{expected_timestamp}")
assert "/compliance/" in compliance
# Test _generate_compliance_output_directory with "threatscore"
threatscore = _generate_compliance_output_directory(
base_dir, provider, tenant_id, scan_id, compliance_framework="threatscore"
)
assert os.path.isdir(os.path.dirname(threatscore))
assert threatscore.endswith(f"{provider}-{expected_timestamp}")
assert "/threatscore/" in threatscore
# Test _generate_compliance_output_directory with "ens"
ens = _generate_compliance_output_directory(
base_dir, provider, tenant_id, scan_id, compliance_framework="ens"
)
assert os.path.isdir(os.path.dirname(ens))
assert ens.endswith(f"{provider}-{expected_timestamp}")
assert "/ens/" in ens
@patch("tasks.jobs.export.set_output_timestamp")
@patch("tasks.jobs.export.rls_transaction")
@patch("tasks.jobs.export.Scan")
def test_generate_output_directory_invalid_character(
self, mock_scan, mock_rls_transaction, mock_set_timestamp, tmpdir
self, mock_scan, mock_rls_transaction, tmpdir
):
# Mock the scan object with a started_at timestamp
mock_scan_instance = MagicMock()
@@ -221,25 +201,14 @@ class TestOutputs:
provider = "aws/test@check"
expected_timestamp = "20230615103045"
# Test provider name sanitization with _generate_output_directory
path, compliance = _generate_output_directory(
path, compliance, threatscore = _generate_output_directory(
base_dir, provider, tenant_id, scan_id
)
assert os.path.isdir(os.path.dirname(path))
assert os.path.isdir(os.path.dirname(compliance))
assert os.path.isdir(os.path.dirname(threatscore))
assert path.endswith(f"aws-test-check-{expected_timestamp}")
assert compliance.endswith(f"aws-test-check-{expected_timestamp}")
# Test provider name sanitization with _generate_compliance_output_directory
threatscore = _generate_compliance_output_directory(
base_dir, provider, tenant_id, scan_id, compliance_framework="threatscore"
)
ens = _generate_compliance_output_directory(
base_dir, provider, tenant_id, scan_id, compliance_framework="ens"
)
assert os.path.isdir(os.path.dirname(threatscore))
assert os.path.isdir(os.path.dirname(ens))
assert threatscore.endswith(f"aws-test-check-{expected_timestamp}")
assert ens.endswith(f"aws-test-check-{expected_timestamp}")
-532
View File
@@ -1,532 +0,0 @@
from datetime import datetime, timezone
from uuid import uuid4
import pytest
from django.core.exceptions import ObjectDoesNotExist
from tasks.jobs.muting import mute_historical_findings
from api.models import Finding, MuteRule
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
@pytest.mark.django_db
class TestMuteHistoricalFindings:
"""
Test suite for the mute_historical_findings function.
This class tests the batch processing of findings to update their muted status
based on MuteRule criteria.
"""
@pytest.fixture(scope="function")
def test_user(self, create_test_user):
"""Create a test user for mute rule creation."""
return create_test_user
@pytest.fixture(scope="function")
def mute_rule_with_findings(self, tenants_fixture, findings_fixture, test_user):
"""
Create a mute rule that targets the first finding in the fixture.
"""
tenant = tenants_fixture[0]
finding = findings_fixture[0]
mute_rule = MuteRule.objects.create(
tenant_id=tenant.id,
name="Test Mute Rule",
reason="Testing mute functionality",
enabled=True,
created_by=test_user,
finding_uids=[finding.uid],
)
return mute_rule
@pytest.fixture(scope="function")
def mute_rule_multiple_findings(self, scans_fixture, test_user):
"""
Create multiple unmuted findings and a mute rule targeting all of them.
"""
scan = scans_fixture[0]
tenant_id = scan.tenant_id
# Create 5 unmuted findings
finding_uids = []
for i in range(5):
finding = Finding.objects.create(
tenant_id=tenant_id,
uid=f"test_finding_uid_mute_{i}",
scan=scan,
status=Status.FAIL,
status_extended=f"Test status {i}",
impact=Severity.high,
severity=Severity.high,
raw_result={
"status": Status.FAIL,
"impact": Severity.high,
"severity": Severity.high,
},
check_id=f"test_check_id_{i}",
check_metadata={
"CheckId": f"test_check_id_{i}",
"Description": f"Test description {i}",
},
muted=False,
)
finding_uids.append(finding.uid)
# Create mute rule targeting all findings
mute_rule = MuteRule.objects.create(
tenant_id=tenant_id,
name="Test Multiple Findings Mute Rule",
reason="Testing batch muting",
enabled=True,
created_by=test_user,
finding_uids=finding_uids,
)
return mute_rule, finding_uids
@pytest.fixture(scope="function")
def mute_rule_already_muted(self, findings_fixture, test_user):
"""
Create a mute rule that targets an already-muted finding.
"""
tenant_id = findings_fixture[1].tenant_id
already_muted_finding = findings_fixture[1]
mute_rule = MuteRule.objects.create(
tenant_id=tenant_id,
name="Test Already Muted Rule",
reason="Testing already muted findings",
enabled=True,
created_by=test_user,
finding_uids=[already_muted_finding.uid],
)
return mute_rule
@pytest.fixture(scope="function")
def mute_rule_mixed_findings(self, scans_fixture, test_user):
"""
Create a mute rule with a mix of muted and unmuted findings.
"""
scan = scans_fixture[0]
tenant_id = scan.tenant_id
# Create 3 unmuted findings
unmuted_uids = []
for i in range(3):
finding = Finding.objects.create(
tenant_id=tenant_id,
uid=f"unmuted_finding_{i}",
scan=scan,
status=Status.FAIL,
status_extended=f"Unmuted status {i}",
impact=Severity.medium,
severity=Severity.medium,
raw_result={
"status": Status.FAIL,
"impact": Severity.medium,
"severity": Severity.medium,
},
check_id=f"unmuted_check_{i}",
check_metadata={
"CheckId": f"unmuted_check_{i}",
"Description": f"Unmuted description {i}",
},
muted=False,
)
unmuted_uids.append(finding.uid)
# Create 2 already muted findings
muted_uids = []
for i in range(2):
finding = Finding.objects.create(
tenant_id=tenant_id,
uid=f"muted_finding_{i}",
scan=scan,
status=Status.FAIL,
status_extended=f"Muted status {i}",
impact=Severity.low,
severity=Severity.low,
raw_result={
"status": Status.FAIL,
"impact": Severity.low,
"severity": Severity.low,
},
check_id=f"muted_check_{i}",
check_metadata={
"CheckId": f"muted_check_{i}",
"Description": f"Muted description {i}",
},
muted=True,
muted_at=datetime.now(timezone.utc),
muted_reason="Already muted",
)
muted_uids.append(finding.uid)
# Create mute rule targeting all findings
all_uids = unmuted_uids + muted_uids
mute_rule = MuteRule.objects.create(
tenant_id=tenant_id,
name="Test Mixed Findings Rule",
reason="Testing mixed muted/unmuted findings",
enabled=True,
created_by=test_user,
finding_uids=all_uids,
)
return mute_rule, unmuted_uids, muted_uids
@pytest.fixture(scope="function")
def mute_rule_batch_test(self, scans_fixture, test_user):
"""
Create enough findings to test batch processing (>1000 for default batch size).
"""
scan = scans_fixture[0]
tenant_id = scan.tenant_id
# Create 1500 findings to exceed default batch size of 1000
finding_uids = []
for i in range(1500):
finding = Finding.objects.create(
tenant_id=tenant_id,
uid=f"batch_test_finding_{i}",
scan=scan,
status=Status.FAIL,
status_extended=f"Batch test status {i}",
impact=Severity.critical,
severity=Severity.critical,
raw_result={
"status": Status.FAIL,
"impact": Severity.critical,
"severity": Severity.critical,
},
check_id=f"batch_test_check_{i}",
check_metadata={
"CheckId": f"batch_test_check_{i}",
"Description": f"Batch test description {i}",
},
muted=False,
)
finding_uids.append(finding.uid)
# Create mute rule targeting all findings
mute_rule = MuteRule.objects.create(
tenant_id=tenant_id,
name="Test Batch Processing Rule",
reason="Testing batch processing functionality",
enabled=True,
created_by=test_user,
finding_uids=finding_uids,
)
return mute_rule, finding_uids
def test_mute_historical_findings_single_finding(
self, mute_rule_with_findings, findings_fixture
):
"""
Test muting a single historical finding.
"""
mute_rule = mute_rule_with_findings
tenant_id = str(mute_rule.tenant_id)
finding = findings_fixture[0]
# Ensure the finding is not muted before execution
finding.refresh_from_db()
assert finding.muted is False
assert finding.muted_at is None
assert finding.muted_reason is None
# Execute the muting function
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify return value
assert result["findings_muted"] == 1
assert result["rule_id"] == str(mute_rule.id)
# Verify the finding was muted
finding.refresh_from_db()
assert finding.muted is True
assert finding.muted_at == mute_rule.inserted_at
assert finding.muted_reason == mute_rule.reason
def test_mute_historical_findings_multiple_findings(
self, mute_rule_multiple_findings
):
"""
Test muting multiple historical findings.
"""
mute_rule, finding_uids = mute_rule_multiple_findings
tenant_id = str(mute_rule.tenant_id)
# Verify all findings are unmuted
findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids)
assert findings.count() == 5
for finding in findings:
assert finding.muted is False
# Execute the muting function
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify return value
assert result["findings_muted"] == 5
assert result["rule_id"] == str(mute_rule.id)
# Verify all findings were muted
findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids)
for finding in findings:
assert finding.muted is True
assert finding.muted_at == mute_rule.inserted_at
assert finding.muted_reason == mute_rule.reason
def test_mute_historical_findings_already_muted(
self, mute_rule_already_muted, findings_fixture
):
"""
Test that already-muted findings are not counted or updated.
"""
mute_rule = mute_rule_already_muted
tenant_id = str(mute_rule.tenant_id)
finding = findings_fixture[1]
# Verify the finding is already muted
finding.refresh_from_db()
assert finding.muted is True
original_muted_at = finding.muted_at
original_muted_reason = finding.muted_reason
# Execute the muting function
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify no findings were muted
assert result["findings_muted"] == 0
assert result["rule_id"] == str(mute_rule.id)
# Verify the finding's mute status did not change
finding.refresh_from_db()
assert finding.muted is True
assert finding.muted_at == original_muted_at
assert finding.muted_reason == original_muted_reason
def test_mute_historical_findings_mixed_status(self, mute_rule_mixed_findings):
"""
Test muting when some findings are already muted and others are not.
"""
mute_rule, unmuted_uids, muted_uids = mute_rule_mixed_findings
tenant_id = str(mute_rule.tenant_id)
# Execute the muting function
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify only unmuted findings were counted
assert result["findings_muted"] == 3
assert result["rule_id"] == str(mute_rule.id)
# Verify unmuted findings are now muted
unmuted_findings = Finding.objects.filter(
tenant_id=tenant_id, uid__in=unmuted_uids
)
for finding in unmuted_findings:
assert finding.muted is True
assert finding.muted_at == mute_rule.inserted_at
assert finding.muted_reason == mute_rule.reason
# Verify already-muted findings remained unchanged
already_muted_findings = Finding.objects.filter(
tenant_id=tenant_id, uid__in=muted_uids
)
for finding in already_muted_findings:
assert finding.muted is True
assert finding.muted_reason == "Already muted"
def test_mute_historical_findings_nonexistent_rule(self, tenants_fixture):
"""
Test that a nonexistent mute rule raises ObjectDoesNotExist.
"""
tenant_id = str(tenants_fixture[0].id)
nonexistent_rule_id = str(uuid4())
with pytest.raises(ObjectDoesNotExist):
mute_historical_findings(tenant_id, nonexistent_rule_id)
def test_mute_historical_findings_no_matching_findings(
self, tenants_fixture, test_user
):
"""
Test muting when no findings match the rule's UIDs.
"""
tenant_id = str(tenants_fixture[0].id)
# Create a mute rule with non-existent finding UIDs
mute_rule = MuteRule.objects.create(
tenant_id=tenant_id,
name="Test No Match Rule",
reason="Testing no matching findings",
enabled=True,
created_by=test_user,
finding_uids=[
"nonexistent_uid_1",
"nonexistent_uid_2",
"nonexistent_uid_3",
],
)
# Execute the muting function
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify no findings were muted
assert result["findings_muted"] == 0
assert result["rule_id"] == str(mute_rule.id)
def test_mute_historical_findings_batch_processing(self, mute_rule_batch_test):
"""
Test that large numbers of findings are processed in batches correctly.
"""
mute_rule, finding_uids = mute_rule_batch_test
tenant_id = str(mute_rule.tenant_id)
# Verify all findings exist and are unmuted
findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids)
assert findings.count() == 1500
for finding in findings:
assert finding.muted is False
# Execute the muting function
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify return value
assert result["findings_muted"] == 1500
assert result["rule_id"] == str(mute_rule.id)
# Verify all findings were muted
findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids)
for finding in findings:
assert finding.muted is True
assert finding.muted_at == mute_rule.inserted_at
assert finding.muted_reason == mute_rule.reason
def test_mute_historical_findings_preserves_muted_at_timestamp(
self, mute_rule_with_findings, findings_fixture
):
"""
Test that muted_at is set to the rule's inserted_at, not the current time.
"""
mute_rule = mute_rule_with_findings
tenant_id = str(mute_rule.tenant_id)
finding = findings_fixture[0]
# Execute the muting function
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify the finding was muted
assert result["findings_muted"] == 1
# Verify muted_at matches the rule's inserted_at timestamp
finding.refresh_from_db()
assert finding.muted_at == mute_rule.inserted_at
assert finding.muted_at is not None
def test_mute_historical_findings_partial_match(self, scans_fixture, test_user):
"""
Test muting when only some of the rule's UIDs exist as findings.
"""
scan = scans_fixture[0]
tenant_id = str(scan.tenant_id)
# Create 3 findings
existing_uids = []
for i in range(3):
finding = Finding.objects.create(
tenant_id=tenant_id,
uid=f"partial_match_finding_{i}",
scan=scan,
status=Status.FAIL,
status_extended=f"Partial match status {i}",
impact=Severity.high,
severity=Severity.high,
raw_result={
"status": Status.FAIL,
"impact": Severity.high,
"severity": Severity.high,
},
check_id=f"partial_match_check_{i}",
check_metadata={
"CheckId": f"partial_match_check_{i}",
"Description": f"Partial match description {i}",
},
muted=False,
)
existing_uids.append(finding.uid)
# Create a mute rule with both existing and non-existing UIDs
all_uids = existing_uids + [
"nonexistent_uid_1",
"nonexistent_uid_2",
]
mute_rule = MuteRule.objects.create(
tenant_id=tenant_id,
name="Test Partial Match Rule",
reason="Testing partial matching",
enabled=True,
created_by=test_user,
finding_uids=all_uids,
)
# Execute the muting function
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify only existing findings were muted
assert result["findings_muted"] == 3
assert result["rule_id"] == str(mute_rule.id)
# Verify the existing findings were muted
findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=existing_uids)
assert findings.count() == 3
for finding in findings:
assert finding.muted is True
assert finding.muted_at == mute_rule.inserted_at
assert finding.muted_reason == mute_rule.reason
def test_mute_historical_findings_empty_uids(self, tenants_fixture, test_user):
"""
Test muting when the rule has an empty finding_uids array.
"""
tenant_id = str(tenants_fixture[0].id)
# Create a mute rule with empty finding_uids
mute_rule = MuteRule.objects.create(
tenant_id=tenant_id,
name="Test Empty UIDs Rule",
reason="Testing empty UIDs",
enabled=True,
created_by=test_user,
finding_uids=[],
)
# Execute the muting function
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify no findings were muted
assert result["findings_muted"] == 0
assert result["rule_id"] == str(mute_rule.id)
def test_mute_historical_findings_return_format(self, mute_rule_with_findings):
"""
Test that the return value has the correct format and fields.
"""
mute_rule = mute_rule_with_findings
tenant_id = str(mute_rule.tenant_id)
result = mute_historical_findings(tenant_id, str(mute_rule.id))
# Verify return value structure
assert isinstance(result, dict)
assert "findings_muted" in result
assert "rule_id" in result
assert isinstance(result["findings_muted"], int)
assert isinstance(result["rule_id"], str)
assert result["rule_id"] == str(mute_rule.id)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+16 -392
View File
@@ -1,24 +1,16 @@
import uuid
from unittest.mock import MagicMock, patch
import openai
import pytest
from botocore.exceptions import ClientError
from tasks.tasks import (
_perform_scan_complete_tasks,
check_integrations_task,
check_lighthouse_provider_connection_task,
generate_outputs_task,
refresh_lighthouse_provider_models_task,
s3_integration_task,
security_hub_integration_task,
)
from api.models import (
Integration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
)
from api.models import Integration
# TODO Move this to outputs/reports jobs
@@ -109,6 +101,7 @@ class TestGenerateOutputs:
return_value=(
"/tmp/test/out-dir",
"/tmp/test/comp-dir",
"/tmp/test/threat-dir",
),
),
patch("tasks.tasks.Scan.all_objects.filter") as mock_scan_update,
@@ -138,7 +131,7 @@ class TestGenerateOutputs:
patch("tasks.tasks.Finding.all_objects.filter") as mock_findings,
patch(
"tasks.tasks._generate_output_directory",
return_value=("/tmp/test/out", "/tmp/test/comp"),
return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"),
),
patch("tasks.tasks.FindingOutput._transform_findings_stats"),
patch("tasks.tasks.FindingOutput.transform_api_finding"),
@@ -208,7 +201,7 @@ class TestGenerateOutputs:
patch("tasks.tasks.Finding.all_objects.filter") as mock_findings,
patch(
"tasks.tasks._generate_output_directory",
return_value=("/tmp/test/out", "/tmp/test/comp"),
return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"),
),
patch(
"tasks.tasks.FindingOutput._transform_findings_stats",
@@ -288,6 +281,7 @@ class TestGenerateOutputs:
return_value=(
"/tmp/test/outdir",
"/tmp/test/compdir",
"/tmp/test/threatdir",
),
),
patch("tasks.tasks._compress_output_files", return_value="outdir.zip"),
@@ -366,6 +360,7 @@ class TestGenerateOutputs:
return_value=(
"/tmp/test/outdir",
"/tmp/test/compdir",
"/tmp/test/threatdir",
),
),
patch("tasks.tasks.FindingOutput._transform_findings_stats"),
@@ -433,7 +428,7 @@ class TestGenerateOutputs:
patch("tasks.tasks.Finding.all_objects.filter") as mock_findings,
patch(
"tasks.tasks._generate_output_directory",
return_value=("/tmp/test/out", "/tmp/test/comp"),
return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"),
),
patch(
"tasks.tasks.FindingOutput._transform_findings_stats",
@@ -491,7 +486,7 @@ class TestGenerateOutputs:
patch("tasks.tasks.Finding.all_objects.filter") as mock_findings,
patch(
"tasks.tasks._generate_output_directory",
return_value=("/tmp/test/out", "/tmp/test/comp"),
return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"),
),
patch("tasks.tasks.FindingOutput._transform_findings_stats"),
patch("tasks.tasks.FindingOutput.transform_api_finding"),
@@ -532,45 +527,34 @@ class TestScanCompleteTasks:
@patch("tasks.tasks.create_compliance_requirements_task.apply_async")
@patch("tasks.tasks.perform_scan_summary_task.si")
@patch("tasks.tasks.generate_outputs_task.si")
@patch("tasks.tasks.generate_compliance_reports_task.si")
@patch("tasks.tasks.generate_threatscore_report_task.si")
@patch("tasks.tasks.check_integrations_task.si")
def test_scan_complete_tasks(
self,
mock_check_integrations_task,
mock_compliance_reports_task,
mock_threatscore_task,
mock_outputs_task,
mock_scan_summary_task,
mock_compliance_requirements_task,
mock_compliance_tasks,
):
"""Test that scan complete tasks are properly orchestrated with optimized reports."""
_perform_scan_complete_tasks("tenant-id", "scan-id", "provider-id")
# Verify compliance requirements task is called
mock_compliance_requirements_task.assert_called_once_with(
mock_compliance_tasks.assert_called_once_with(
kwargs={"tenant_id": "tenant-id", "scan_id": "scan-id"},
)
# Verify scan summary task is called
mock_scan_summary_task.assert_called_once_with(
scan_id="scan-id",
tenant_id="tenant-id",
)
# Verify outputs task is called
mock_outputs_task.assert_called_once_with(
scan_id="scan-id",
provider_id="provider-id",
tenant_id="tenant-id",
)
# Verify optimized compliance reports task is called (replaces individual tasks)
mock_compliance_reports_task.assert_called_once_with(
mock_threatscore_task.assert_called_once_with(
tenant_id="tenant-id",
scan_id="scan-id",
provider_id="provider-id",
)
# Verify integrations task is called
mock_check_integrations_task.assert_called_once_with(
tenant_id="tenant-id",
provider_id="provider-id",
@@ -746,7 +730,7 @@ class TestCheckIntegrationsTask:
mock_initialize_provider.return_value = MagicMock()
mock_compliance_bulk.return_value = {}
mock_get_frameworks.return_value = []
mock_generate_dir.return_value = ("out-dir", "comp-dir")
mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir")
mock_transform_stats.return_value = {"stats": "data"}
# Mock findings
@@ -871,7 +855,7 @@ class TestCheckIntegrationsTask:
mock_initialize_provider.return_value = MagicMock()
mock_compliance_bulk.return_value = {}
mock_get_frameworks.return_value = []
mock_generate_dir.return_value = ("out-dir", "comp-dir")
mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir")
mock_transform_stats.return_value = {"stats": "data"}
# Mock findings
@@ -987,7 +971,7 @@ class TestCheckIntegrationsTask:
mock_initialize_provider.return_value = MagicMock()
mock_compliance_bulk.return_value = {}
mock_get_frameworks.return_value = []
mock_generate_dir.return_value = ("out-dir", "comp-dir")
mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir")
mock_transform_stats.return_value = {"stats": "data"}
# Mock findings
@@ -1113,363 +1097,3 @@ class TestCheckIntegrationsTask:
assert result is False
mock_upload.assert_called_once_with(self.tenant_id, self.provider_id, scan_id)
@pytest.mark.django_db
class TestCheckLighthouseProviderConnectionTask:
def setup_method(self):
self.tenant_id = str(uuid.uuid4())
@pytest.mark.parametrize(
"provider_type,credentials,base_url,expected_result",
[
(
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI,
{"api_key": "sk-test123"},
None,
{"connected": True, "error": None},
),
(
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
{"api_key": "sk-test123"},
"https://openrouter.ai/api/v1",
{"connected": True, "error": None},
),
(
LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
{
"access_key_id": "AKIA123",
"secret_access_key": "secret",
"region": "us-east-1",
},
None,
{"connected": True, "error": None},
),
],
)
def test_check_connection_success_all_providers(
self, tenants_fixture, provider_type, credentials, base_url, expected_result
):
"""Test successful connection check for all provider types."""
# Create provider configuration
provider_cfg = LighthouseProviderConfiguration(
tenant_id=tenants_fixture[0].id,
provider_type=provider_type,
base_url=base_url,
is_active=False,
)
provider_cfg.credentials_decoded = credentials
provider_cfg.save()
# Mock the appropriate API calls
with (
patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai,
patch("tasks.jobs.lighthouse_providers.boto3.client") as mock_boto3,
):
mock_client = MagicMock()
mock_client.models.list.return_value = MagicMock()
mock_client.list_foundation_models.return_value = {}
mock_openai.return_value = mock_client
mock_boto3.return_value = mock_client
# Execute
result = check_lighthouse_provider_connection_task(
provider_config_id=str(provider_cfg.id),
tenant_id=str(tenants_fixture[0].id),
)
# Assert
assert result == expected_result
provider_cfg.refresh_from_db()
assert provider_cfg.is_active is True
@pytest.mark.parametrize(
"provider_type,credentials,base_url,exception_to_raise",
[
(
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI,
{"api_key": "sk-invalid"},
None,
openai.AuthenticationError(
"Invalid API key", response=MagicMock(), body=None
),
),
(
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
{"api_key": "sk-invalid"},
"https://openrouter.ai/api/v1",
openai.APIConnectionError(request=MagicMock()),
),
(
LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
{
"access_key_id": "AKIA123",
"secret_access_key": "secret",
"region": "us-east-1",
},
None,
ClientError(
{"Error": {"Code": "AccessDenied", "Message": "Access Denied"}},
"list_foundation_models",
),
),
],
)
def test_check_connection_api_failure(
self,
tenants_fixture,
provider_type,
credentials,
base_url,
exception_to_raise,
):
"""Test connection check when API calls fail."""
# Create provider configuration
provider_cfg = LighthouseProviderConfiguration(
tenant_id=tenants_fixture[0].id,
provider_type=provider_type,
base_url=base_url,
is_active=True,
)
provider_cfg.credentials_decoded = credentials
provider_cfg.save()
# Mock the API to raise exception
with (
patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai,
patch("tasks.jobs.lighthouse_providers.boto3.client") as mock_boto3,
):
mock_client = MagicMock()
if (
provider_type
== LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
):
mock_client.list_foundation_models.side_effect = exception_to_raise
mock_boto3.return_value = mock_client
else:
mock_client.models.list.side_effect = exception_to_raise
mock_openai.return_value = mock_client
# Execute
result = check_lighthouse_provider_connection_task(
provider_config_id=str(provider_cfg.id),
tenant_id=str(tenants_fixture[0].id),
)
# Assert
assert result["connected"] is False
assert result["error"] is not None
provider_cfg.refresh_from_db()
assert provider_cfg.is_active is False
def test_check_connection_updates_active_status(self, tenants_fixture):
"""Test that connection check toggles is_active from True to False on failure."""
# Create provider with is_active=True
provider_cfg = LighthouseProviderConfiguration(
tenant_id=tenants_fixture[0].id,
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI,
base_url=None,
is_active=True,
)
provider_cfg.credentials_decoded = {"api_key": "sk-test123"}
provider_cfg.save()
# Mock API to fail
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
mock_client = MagicMock()
mock_client.models.list.side_effect = openai.AuthenticationError(
"Invalid", response=MagicMock(), body=None
)
mock_openai.return_value = mock_client
# Execute
result = check_lighthouse_provider_connection_task(
provider_config_id=str(provider_cfg.id),
tenant_id=str(tenants_fixture[0].id),
)
# Assert status changed
assert result["connected"] is False
provider_cfg.refresh_from_db()
assert provider_cfg.is_active is False
def test_check_connection_provider_does_not_exist(self, tenants_fixture):
"""Test that checking non-existent provider raises DoesNotExist."""
non_existent_id = str(uuid.uuid4())
with pytest.raises(LighthouseProviderConfiguration.DoesNotExist):
check_lighthouse_provider_connection_task(
provider_config_id=non_existent_id,
tenant_id=str(tenants_fixture[0].id),
)
@pytest.mark.django_db
class TestRefreshLighthouseProviderModelsTask:
def setup_method(self):
self.tenant_id = str(uuid.uuid4())
@pytest.mark.parametrize(
"provider_type,credentials,base_url,mock_models,expected_count",
[
(
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI,
{"api_key": "sk-test123"},
None,
{"gpt-5": "gpt-5", "gpt-4o": "gpt-4o"},
2,
),
(
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
{"api_key": "sk-test123"},
"https://openrouter.ai/api/v1",
{"model-1": "Model One", "model-2": "Model Two"},
2,
),
(
LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
{
"access_key_id": "AKIA123",
"secret_access_key": "secret",
"region": "us-east-1",
},
None,
{"openai.gpt-oss-120b-1:0": "gpt-oss-120b"},
1,
),
],
)
def test_refresh_models_create_new(
self,
tenants_fixture,
provider_type,
credentials,
base_url,
mock_models,
expected_count,
):
"""Test creating new models for all provider types."""
# Create provider configuration
provider_cfg = LighthouseProviderConfiguration(
tenant_id=tenants_fixture[0].id,
provider_type=provider_type,
base_url=base_url,
is_active=True,
)
provider_cfg.credentials_decoded = credentials
provider_cfg.save()
# Mock the fetch functions
with (
patch(
"tasks.jobs.lighthouse_providers._fetch_openai_models",
return_value=mock_models,
),
patch(
"tasks.jobs.lighthouse_providers._fetch_openai_compatible_models",
return_value=mock_models,
),
patch(
"tasks.jobs.lighthouse_providers._fetch_bedrock_models",
return_value=mock_models,
),
):
# Execute
result = refresh_lighthouse_provider_models_task(
provider_config_id=str(provider_cfg.id),
tenant_id=str(tenants_fixture[0].id),
)
# Assert
assert result["created"] == expected_count
assert result["updated"] == 0
assert result["deleted"] == 0
assert (
LighthouseProviderModels.objects.filter(
provider_configuration=provider_cfg
).count()
== expected_count
)
def test_refresh_models_mixed_operations(self, tenants_fixture):
"""Test mixed create, update, and delete operations."""
# Create provider configuration
provider_cfg = LighthouseProviderConfiguration(
tenant_id=tenants_fixture[0].id,
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI,
base_url=None,
is_active=True,
)
provider_cfg.credentials_decoded = {"api_key": "sk-test123"}
provider_cfg.save()
# Create 2 existing models (A, B)
LighthouseProviderModels.objects.create(
tenant_id=tenants_fixture[0].id,
provider_configuration=provider_cfg,
model_id="model-a",
model_name="Model A",
)
LighthouseProviderModels.objects.create(
tenant_id=tenants_fixture[0].id,
provider_configuration=provider_cfg,
model_id="model-b",
model_name="Model B",
)
# Mock API to return models B (existing), C (new) - A will be deleted
mock_models = {"model-b": "Model B", "model-c": "Model C"}
with patch(
"tasks.jobs.lighthouse_providers._fetch_openai_models",
return_value=mock_models,
):
# Execute
result = refresh_lighthouse_provider_models_task(
provider_config_id=str(provider_cfg.id),
tenant_id=str(tenants_fixture[0].id),
)
# Assert
assert result["created"] == 1 # model-c created
assert result["updated"] == 1 # model-b updated
assert result["deleted"] == 1 # model-a deleted
# Verify only B and C exist
remaining_models = LighthouseProviderModels.objects.filter(
provider_configuration=provider_cfg
)
assert remaining_models.count() == 2
assert set(remaining_models.values_list("model_id", flat=True)) == {
"model-b",
"model-c",
}
def test_refresh_models_api_exception(self, tenants_fixture):
"""Test refresh when API raises an exception."""
# Create provider configuration
provider_cfg = LighthouseProviderConfiguration(
tenant_id=tenants_fixture[0].id,
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI,
base_url=None,
is_active=True,
)
provider_cfg.credentials_decoded = {"api_key": "sk-test123"}
provider_cfg.save()
# Mock fetch to raise exception
with patch(
"tasks.jobs.lighthouse_providers._fetch_openai_models",
side_effect=openai.APIError("API Error", request=MagicMock(), body=None),
):
# Execute
result = refresh_lighthouse_provider_models_task(
provider_config_id=str(provider_cfg.id),
tenant_id=str(tenants_fixture[0].id),
)
# Assert
assert result["created"] == 0
assert result["updated"] == 0
assert result["deleted"] == 0
assert "error" in result
assert result["error"] is not None
@@ -1,46 +0,0 @@
import warnings
from dashboard.common_methods import get_section_containers_cis
warnings.filterwarnings("ignore")
def get_table(data):
aux = data[
[
"REQUIREMENTS_ID",
"REQUIREMENTS_DESCRIPTION",
"REQUIREMENTS_ATTRIBUTES_SECTION",
"CHECKID",
"STATUS",
"REGION",
"ACCOUNTID",
"RESOURCEID",
]
].copy()
# Shorten the long FedRAMP KSI descriptions for better display
ksi_short_names = {
"A secure cloud service offering will protect user data, control access, and apply zero trust principles": "Identity and Access Management",
"A secure cloud service offering will use cloud native architecture and design principles to enforce and enhance the Confidentiality, Integrity and Availability of the system": "Cloud Native Architecture",
"A secure cloud service provider will ensure that all system changes are properly documented and configuration baselines are updated accordingly": "Change Management",
"A secure cloud service provider will continuously educate their employees on cybersecurity measures, testing them regularly": "Cybersecurity Education",
"A secure cloud service offering will document, report, and analyze security incidents to ensure regulatory compliance and continuous security improvement": "Incident Reporting",
"A secure cloud service offering will monitor, log, and audit all important events, activity, and changes": "Monitoring, Logging, and Auditing",
"A secure cloud service offering will have intentional, organized, universal guidance for how every information resource, including personnel, is secured": "Policy and Inventory",
"A secure cloud service offering will define, maintain, and test incident response plan(s) and recovery capabilities to ensure minimal service disruption and data loss": "Recovery Planning",
"A secure cloud service offering will follow FedRAMP encryption policies, continuously verify information resource integrity, and restrict access to third-party information resources": "Service Configuration",
"A secure cloud service offering will understand, monitor, and manage supply chain risks from third-party information resources": "Third-Party Information Resources",
}
# Replace long descriptions with short names - use contains for partial matching
if not aux.empty:
for long_desc, short_name in ksi_short_names.items():
mask = aux["REQUIREMENTS_DESCRIPTION"].str.contains(
long_desc, na=False, regex=False
)
aux.loc[mask, "REQUIREMENTS_DESCRIPTION"] = short_name
return get_section_containers_cis(
aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION"
)
@@ -1,46 +0,0 @@
import warnings
from dashboard.common_methods import get_section_containers_cis
warnings.filterwarnings("ignore")
def get_table(data):
aux = data[
[
"REQUIREMENTS_ID",
"REQUIREMENTS_DESCRIPTION",
"REQUIREMENTS_ATTRIBUTES_SECTION",
"CHECKID",
"STATUS",
"REGION",
"ACCOUNTID",
"RESOURCEID",
]
].copy()
# Shorten the long FedRAMP KSI descriptions for better display
ksi_short_names = {
"A secure cloud service offering will protect user data, control access, and apply zero trust principles": "Identity and Access Management",
"A secure cloud service offering will use cloud native architecture and design principles to enforce and enhance the Confidentiality, Integrity and Availability of the system": "Cloud Native Architecture",
"A secure cloud service provider will ensure that all system changes are properly documented and configuration baselines are updated accordingly": "Change Management",
"A secure cloud service provider will continuously educate their employees on cybersecurity measures, testing them regularly": "Cybersecurity Education",
"A secure cloud service offering will document, report, and analyze security incidents to ensure regulatory compliance and continuous security improvement": "Incident Reporting",
"A secure cloud service offering will monitor, log, and audit all important events, activity, and changes": "Monitoring, Logging, and Auditing",
"A secure cloud service offering will have intentional, organized, universal guidance for how every information resource, including personnel, is secured": "Policy and Inventory",
"A secure cloud service offering will define, maintain, and test incident response plan(s) and recovery capabilities to ensure minimal service disruption and data loss": "Recovery Planning",
"A secure cloud service offering will follow FedRAMP encryption policies, continuously verify information resource integrity, and restrict access to third-party information resources": "Service Configuration",
"A secure cloud service offering will understand, monitor, and manage supply chain risks from third-party information resources": "Third-Party Information Resources",
}
# Replace long descriptions with short names - use contains for partial matching
if not aux.empty:
for long_desc, short_name in ksi_short_names.items():
mask = aux["REQUIREMENTS_DESCRIPTION"].str.contains(
long_desc, na=False, regex=False
)
aux.loc[mask, "REQUIREMENTS_DESCRIPTION"] = short_name
return get_section_containers_cis(
aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION"
)
@@ -1,46 +0,0 @@
import warnings
from dashboard.common_methods import get_section_containers_cis
warnings.filterwarnings("ignore")
def get_table(data):
aux = data[
[
"REQUIREMENTS_ID",
"REQUIREMENTS_DESCRIPTION",
"REQUIREMENTS_ATTRIBUTES_SECTION",
"CHECKID",
"STATUS",
"REGION",
"ACCOUNTID",
"RESOURCEID",
]
].copy()
# Shorten the long FedRAMP KSI descriptions for better display
ksi_short_names = {
"A secure cloud service offering will protect user data, control access, and apply zero trust principles": "Identity and Access Management",
"A secure cloud service offering will use cloud native architecture and design principles to enforce and enhance the Confidentiality, Integrity and Availability of the system": "Cloud Native Architecture",
"A secure cloud service provider will ensure that all system changes are properly documented and configuration baselines are updated accordingly": "Change Management",
"A secure cloud service provider will continuously educate their employees on cybersecurity measures, testing them regularly": "Cybersecurity Education",
"A secure cloud service offering will document, report, and analyze security incidents to ensure regulatory compliance and continuous security improvement": "Incident Reporting",
"A secure cloud service offering will monitor, log, and audit all important events, activity, and changes": "Monitoring, Logging, and Auditing",
"A secure cloud service offering will have intentional, organized, universal guidance for how every information resource, including personnel, is secured": "Policy and Inventory",
"A secure cloud service offering will define, maintain, and test incident response plan(s) and recovery capabilities to ensure minimal service disruption and data loss": "Recovery Planning",
"A secure cloud service offering will follow FedRAMP encryption policies, continuously verify information resource integrity, and restrict access to third-party information resources": "Service Configuration",
"A secure cloud service offering will understand, monitor, and manage supply chain risks from third-party information resources": "Third-Party Information Resources",
}
# Replace long descriptions with short names - use contains for partial matching
if not aux.empty:
for long_desc, short_name in ksi_short_names.items():
mask = aux["REQUIREMENTS_DESCRIPTION"].str.contains(
long_desc, na=False, regex=False
)
aux.loc[mask, "REQUIREMENTS_DESCRIPTION"] = short_name
return get_section_containers_cis(
aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION"
)
-25
View File
@@ -1,25 +0,0 @@
import warnings
from dashboard.common_methods import get_section_containers_format3
warnings.filterwarnings("ignore")
def get_table(data):
aux = data[
[
"REQUIREMENTS_ID",
"REQUIREMENTS_ATTRIBUTES_SECTION",
"REQUIREMENTS_DESCRIPTION",
"CHECKID",
"STATUS",
"REGION",
"ACCOUNTID",
"RESOURCEID",
]
].copy()
return get_section_containers_format3(
aux, "REQUIREMENTS_ATTRIBUTES_SECTION", "REQUIREMENTS_ID"
)
-24
View File
@@ -1,24 +0,0 @@
import warnings
from dashboard.common_methods import get_section_containers_format3
warnings.filterwarnings("ignore")
def get_table(data):
aux = data[
[
"REQUIREMENTS_ID",
"REQUIREMENTS_ATTRIBUTES_SECTION",
"REQUIREMENTS_DESCRIPTION",
"CHECKID",
"STATUS",
"REGION",
"ACCOUNTID",
"RESOURCEID",
]
].copy()
return get_section_containers_format3(
aux, "REQUIREMENTS_ATTRIBUTES_SECTION", "REQUIREMENTS_ID"
)
@@ -1,28 +0,0 @@
import warnings
from dashboard.common_methods import get_section_containers_threatscore
warnings.filterwarnings("ignore")
def get_table(data):
aux = data[
[
"REQUIREMENTS_ID",
"REQUIREMENTS_DESCRIPTION",
"REQUIREMENTS_ATTRIBUTES_SECTION",
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
"CHECKID",
"STATUS",
"REGION",
"ACCOUNTID",
"RESOURCEID",
]
].copy()
return get_section_containers_threatscore(
aux,
"REQUIREMENTS_ATTRIBUTES_SECTION",
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
"REQUIREMENTS_ID",
)
+335
View File
@@ -0,0 +1,335 @@
# Prowler API Reference Documentation
This directory contains the API reference documentation for Prowler Cloud, integrated with Mintlify.
## Structure
```
api-reference/
├── README.md # This file
├── openapi.yaml # OpenAPI specification (auto-synced from api/src/backend/api/specs/v1.yaml)
├── introduction.mdx # API introduction and getting started guide
├── tokens/ # Authentication endpoints
│ ├── create.mdx # Create JWT token
│ ├── refresh.mdx # Refresh JWT token
│ └── switch.mdx # Switch tenant context
├── api-keys/ # API Keys endpoints
│ ├── list.mdx
│ ├── create.mdx
│ ├── retrieve.mdx
│ ├── update.mdx
│ └── revoke.mdx
├── users/ # User endpoints
│ └── me.mdx # Get current user
├── tenants/ # Tenant management
│ ├── list.mdx
│ ├── invitations-list.mdx
│ └── invitations-create.mdx
├── invitations/ # Invitation endpoints
│ └── accept.mdx
├── providers/ # Cloud provider management
│ ├── list.mdx
│ ├── create.mdx
│ ├── retrieve.mdx
│ ├── update.mdx
│ ├── delete.mdx
│ └── check-connection.mdx
├── scans/ # Security scan endpoints
│ ├── list.mdx
│ ├── retrieve.mdx
│ ├── compliance.mdx
│ ├── report.mdx
│ └── threatscore.mdx
├── findings/ # Security findings endpoints
│ ├── list.mdx
│ ├── retrieve.mdx
│ ├── latest.mdx
│ ├── services-regions.mdx
│ ├── metadata.mdx
│ └── metadata-latest.mdx
├── resources/ # Cloud resource endpoints
│ ├── list.mdx
│ ├── retrieve.mdx
│ ├── latest.mdx
│ ├── metadata.mdx
│ └── metadata-latest.mdx
├── compliance/ # Compliance framework endpoints
│ ├── list.mdx
│ ├── requirements.mdx
│ ├── attributes.mdx
│ └── metadata.mdx
├── overviews/ # Dashboard overview endpoints
│ ├── findings.mdx
│ ├── findings-severity.mdx
│ ├── providers.mdx
│ ├── providers-count.mdx
│ └── services.mdx
├── integrations/ # External integrations
│ ├── list.mdx
│ ├── create.mdx
│ ├── retrieve.mdx
│ ├── update.mdx
│ ├── delete.mdx
│ ├── check-connection.mdx
│ └── jira-dispatch.mdx
├── lighthouse/ # Lighthouse AI endpoints
│ ├── configuration-get.mdx
│ ├── configuration-update.mdx
│ ├── providers-list.mdx
│ ├── providers-create.mdx
│ └── models-list.mdx
├── processors/ # Finding processors (mutelists)
│ ├── list.mdx
│ └── create.mdx
├── schedules/ # Automated scan scheduling
│ └── daily.mdx
└── tasks/ # Asynchronous task management
├── list.mdx
└── retrieve.mdx
```
**Total: 131 endpoint documentation files organized in 18 groups**
**100% API Coverage** - All 123 operation IDs documented
## How It Works
The API documentation uses Mintlify's native OpenAPI support to automatically generate interactive API documentation from the OpenAPI specification file.
### Components
1. **openapi.yaml**: The source of truth for API endpoints, copied from `api/src/backend/api/specs/v1.yaml`
2. **MDX files**: Enhanced documentation for each endpoint with examples, tips, and additional context
3. **docs.json**: Mintlify configuration that references the OpenAPI spec
## Updating the Documentation
### When the API Spec Changes
When you update the OpenAPI specification in `api/src/backend/api/specs/v1.yaml`, you need to sync it to the docs:
```bash
# From the root of the repository
cp api/src/backend/api/specs/v1.yaml docs/api-reference/openapi.yaml
```
Consider automating this with a pre-commit hook or CI/CD pipeline.
### Adding New Endpoints
To document a new API endpoint:
1. **Update the OpenAPI spec** in `api/src/backend/api/specs/v1.yaml`
2. **Sync to docs**: Copy the spec to `docs/api-reference/openapi.yaml`
3. **Create MDX file**: Create a new `.mdx` file in the appropriate directory
4. **Update navigation**: Add the new page to `docs/docs.json` in the API Reference tab
#### MDX File Template
```mdx
---
title: "Endpoint Title"
api: "METHOD /api/v1/endpoint"
description: "Brief description of what this endpoint does."
---
Detailed description of the endpoint.
## Path Parameters
- `param` (required/optional) - Description
## Query Parameters
- `param` (required/optional) - Description
## Request Body
```json
{
"example": "request"
}
```
## Example Request
```bash
curl -X METHOD "https://api.prowler.com/api/v1/endpoint" \
-H "Authorization: Bearer YOUR_API_KEY" \
-H "Content-Type: application/vnd.api+json"
```
## Response
Description of the response.
```
### Testing Changes Locally
To preview documentation changes locally:
```bash
cd docs
mintlify dev
```
Then open http://localhost:3000 in your browser.
## Mintlify Configuration
The API documentation is configured in `docs/docs.json`:
- **openapi**: Points to the OpenAPI spec file
- **api.baseUrl**: The base URL for the API
- **api.auth.method**: Authentication method (bearer token)
- **api.playground.mode**: Interactive API playground mode
## API Endpoint Groups
### Core Endpoints
- **Authentication (3)**: JWT token management and tenant switching
- **API Keys (5)**: Programmatic API access management
- **Users (1)**: User profile and information
- **Tenants (3)**: Organization and invitation management
### Cloud Infrastructure
- **Providers (6)**: Cloud provider (AWS, Azure, GCP, etc.) configuration
- **Scans (5)**: Security scan execution and results
- **Findings (6)**: Security findings and vulnerabilities
- **Resources (5)**: Cloud resource inventory and metadata
### Compliance & Reporting
- **Compliance (4)**: Compliance framework assessments (CIS, PCI-DSS, etc.)
- **Overviews (5)**: Dashboard aggregated statistics
### Integrations & Automation
- **Integrations (7)**: External service integrations (S3, Security Hub, JIRA, Slack)
- **Lighthouse AI (5)**: AI-powered security insights configuration
- **Processors (2)**: Finding processors and mutelists
- **Schedules (1)**: Automated scan scheduling
### System
- **Tasks (2)**: Asynchronous task monitoring
## Best Practices
1. **Keep OpenAPI spec up to date**: Always update the source spec first, then sync to docs
2. **Add examples**: Include real-world examples in MDX files
3. **Use callouts**: Leverage Mintlify components like `<Note>`, `<Tip>`, `<Warning>` for important information
4. **Test playground**: Verify that the interactive API playground works for each endpoint
5. **Document filters**: For list endpoints, clearly document all available filters
6. **Include rate limits**: Document any rate limiting or pagination requirements
7. **Group related endpoints**: Keep related endpoints in the same directory
8. **Use consistent naming**: Follow the pattern `action.mdx` (e.g., `list.mdx`, `create.mdx`, `retrieve.mdx`)
## Resources
- [Mintlify OpenAPI Guide](https://mintlify.com/docs/api-playground/openapi-support)
- [Mintlify Components](https://mintlify.com/docs/content/components)
- [JSON:API Specification](https://jsonapi.org/)
- [Prowler Cloud API](https://api.prowler.com/api/v1/docs)
## Syncing OpenAPI Spec
The OpenAPI specification is maintained in the API repository and should be synced regularly:
```bash
# Using the provided sync script
cd docs
./sync-api-spec.sh
# Or manually
cp ../api/src/backend/api/specs/v1.yaml ./api-reference/openapi.yaml
```
The `sync-api-spec.sh` script automates this process and can be integrated into your CI/CD pipeline.
## Automation 🤖
**NEW**: Documentation can now be auto-generated from the OpenAPI specification!
### Quick Start with Automation
```bash
cd docs
# 1. Sync latest OpenAPI spec from GitHub
./sync-api-spec.sh
# 2. Generate MDX files automatically
python3 generate-api-docs.py
# 3. Test locally
mintlify dev
```
### Available Scripts
1. **`sync-api-spec.sh`** - Downloads latest OpenAPI spec from GitHub
2. **`generate-api-docs.py`** - Generates/updates MDX files from OpenAPI spec
### Features
- ✅ Auto-generate MDX files from OpenAPI spec
- ✅ Update existing or create new endpoints
- ✅ Dry-run mode to preview changes
- ✅ Proper directory structure and naming
- ✅ Frontmatter generation (title, API path, description)
- ✅ Parameter documentation extraction
- ✅ Request/response examples
### Documentation
See **[AUTOMATION.md](./AUTOMATION.md)** for complete automation guide including:
- Detailed usage instructions
- CI/CD integration examples
- Troubleshooting
- Best practices
## Recent Updates
### January 2025 - Complete API Documentation & Automation
#### Documentation Expansion
- ✅ **100% Coverage**: All 123 operation IDs documented (131 MDX files)
- ✅ **18 Endpoint Groups**: Complete organization
- ✅ **New Groups Added**:
- Provider Secrets (5 endpoints)
- Provider Groups (8 endpoints)
- Roles (8 endpoints)
- SAML Configuration (5 endpoints)
- Lighthouse AI expanded (17 endpoints with nested structure)
- Processors (5 endpoints)
- Tasks (3 endpoints)
- Compliance Overview (4 endpoints)
- Overviews (5 endpoints)
#### Automation Implementation
- ✅ **Auto-generation script**: `generate-api-docs.py`
- ✅ **Sync script**: `sync-api-spec.sh`
- ✅ **Complete automation guide**: AUTOMATION.md
- ✅ **CI/CD ready**: GitHub Actions example included
#### Quality Improvements
- ✅ Field name corrections verified against OpenAPI spec
- ✅ JSON:API compliance in all examples
- ✅ Proper nested structures (Provider Secrets, Tenant Memberships/Invitations)
- ✅ Comprehensive documentation files (CORRECTIONS.md, VERIFICATION.md, IMPROVEMENTS.md)
**Total coverage increased from 14 to 131 documented endpoints (843% increase)**
+182
View File
@@ -0,0 +1,182 @@
---
title: "Create API Key"
api: "POST /api/v1/api-keys"
description: "Create a new API key for the tenant."
---
Create a new API key for programmatic access to the Prowler API. The API key will be returned in the response and should be stored securely as it cannot be retrieved later.
## Request Body
The request must follow the JSON:API specification format:
```json
{
"data": {
"type": "api-keys",
"attributes": {
"name": "string",
"expires_at": "2024-12-31T23:59:59Z"
}
}
}
```
### Attributes
- `name` (required) - A descriptive name for the API key
- `expires_at` (optional) - Expiration date for the API key in ISO 8601 format
## Example Request
```bash
curl -X POST "https://api.prowler.com/api/v1/api-keys" \
-H "Authorization: Bearer YOUR_JWT_TOKEN" \
-H "Content-Type: application/vnd.api+json" \
-d '{
"data": {
"type": "api-keys",
"attributes": {
"name": "Production API Key",
"expires_at": "2025-12-31T23:59:59Z"
}
}
}'
```
## Response
### Success Response (201 Created)
Returns the created API key **including the full key value** (only shown once):
```json
{
"data": {
"type": "api-keys",
"id": "api-key-uuid",
"attributes": {
"name": "Production API Key",
"prefix": "pk_live_abc123",
"api_key": "pk_live_abc123def456ghi789jkl012mno345pqr678stu901vwx234yz",
"expires_at": "2025-12-31T23:59:59Z",
"revoked": false,
"inserted_at": "2024-01-15T10:30:00Z",
"last_used_at": null
},
"relationships": {
"entity": {
"data": {
"type": "users",
"id": "user-uuid"
}
}
}
}
}
```
### Response Fields
- `id` - Unique UUID for the API key record
- `name` - Descriptive name you provided (3-100 characters)
- `prefix` - First characters of the key (for identification, read-only)
- `api_key` - **Full API key value** (only returned on creation, read-only)
- `expires_at` - Expiration date in ISO 8601 format (optional)
- `revoked` - Whether key has been revoked (always false on creation, read-only)
- `inserted_at` - When key was created (read-only)
- `last_used_at` - Last usage timestamp (null initially, read-only)
### Error Responses
**400 Bad Request** - Invalid expiration date
```json
{
"errors": [
{
"status": "400",
"title": "Invalid Expiration Date",
"detail": "Expiration date must be in the future"
}
]
}
```
**422 Unprocessable Entity** - Missing required fields
```json
{
"errors": [
{
"status": "422",
"title": "Validation Error",
"detail": "Name is required",
"source": {
"pointer": "/data/attributes/name"
}
}
]
}
```
**429 Too Many Requests** - API key limit reached
```json
{
"errors": [
{
"status": "429",
"title": "API Key Limit Exceeded",
"detail": "Maximum of 10 active API keys per tenant. Revoke unused keys to create new ones."
}
]
}
```
## Usage Example
After creating the API key, use it to authenticate API requests:
```bash
# Save the API key
API_KEY="pk_live_abc123def456ghi789jkl012mno345pqr678stu901vwx234yz"
# Use it in requests
curl -X GET "https://api.prowler.com/api/v1/providers" \
-H "Authorization: Bearer $API_KEY" \
-H "Content-Type: application/vnd.api+json"
```
## Security Best Practices
1. **Store Securely**: Save the API key in a secure location immediately (password manager, secrets vault, environment variables)
2. **Never Commit**: Do not commit API keys to version control systems
3. **Use Environment Variables**: Store keys in environment variables, not in code
4. **Set Expiration**: Always set an expiration date for API keys (recommended: 90 days)
5. **Rotate Regularly**: Rotate API keys every 90 days
6. **Limit Scope**: Use separate API keys for different environments (dev, staging, production)
7. **Monitor Usage**: Regularly check `last_used_at` to identify unused keys
8. **Revoke Unused**: Revoke API keys that are no longer needed
## Naming Conventions
Use descriptive names that include:
- **Environment**: Production, Staging, Development
- **Purpose**: CI/CD, Dashboard, Integration
- **Owner/Team**: Security Team, DevOps, Monitoring
Examples:
- "Production CI/CD Pipeline"
- "Staging Dashboard API Access"
- "Security Team Automation"
- "JIRA Integration - Production"
<Warning>
**CRITICAL:** The API key is only shown once in the creation response. It cannot be retrieved later. If you lose the key, you must revoke it and create a new one. Store it securely immediately after creation.
</Warning>
<Note>
API keys have the same permissions as the user who created them. For automated systems, consider creating a dedicated service account with limited permissions.
</Note>
<Tip>
**Automation Tip:** API keys are ideal for CI/CD pipelines, scheduled scans, and integrations. Unlike JWT tokens, they don't expire hourly and don't require refresh logic.
</Tip>
+124
View File
@@ -0,0 +1,124 @@
---
title: "List API Keys"
api: "GET /api/v1/api-keys"
description: "Retrieve a list of API keys for the tenant, with filtering support."
---
Retrieve a list of all API keys associated with your tenant. This endpoint supports various filtering options to help you find specific keys.
## Query Parameters
### Filtering
- `filter[name]` - Filter by exact API key name
- `filter[name__icontains]` - Filter by API key name (case-insensitive)
- `filter[prefix]` - Filter by API key prefix
- `filter[revoked]` - Filter by revocation status (boolean)
- `filter[expires_at]`, `filter[expires_at__gte]`, `filter[expires_at__lte]` - Filter by expiration date
- `filter[inserted_at]`, `filter[inserted_at__gte]`, `filter[inserted_at__lte]` - Filter by creation date
- `filter[search]` - General search term
### Pagination
- `page[number]` - Page number to retrieve
- `page[size]` - Number of results per page
### Sorting
- `sort` - Field to sort by. Available options: `name`, `-name`, `prefix`, `-prefix`, `revoked`, `-revoked`, `inserted_at`, `-inserted_at`, `expires_at`, `-expires_at`
### Field Selection
- `fields[api-keys]` - Specify which fields to return: `name`, `prefix`, `expires_at`, `revoked`, `inserted_at`, `last_used_at`, `entity`
### Include Related Resources
- `include` - Include related resources. Available: `entity`
## Example Request
```bash
curl -X GET "https://api.prowler.com/api/v1/api-keys?page[size]=10&sort=-inserted_at" \
-H "Authorization: Bearer YOUR_API_KEY" \
-H "Content-Type: application/vnd.api+json"
```
## Response
### Success Response (200 OK)
Returns a paginated list of API keys with their metadata:
```json
{
"data": [
{
"type": "api-keys",
"id": "api-key-uuid-1",
"attributes": {
"name": "Production CI/CD Pipeline",
"prefix": "pk_live_abc123",
"expires_at": "2025-12-31T23:59:59Z",
"revoked": false,
"inserted_at": "2024-01-15T10:30:00Z",
"last_used_at": "2024-01-20T14:22:00Z"
},
"relationships": {
"entity": {
"data": {
"type": "users",
"id": "user-uuid"
}
}
}
},
{
"type": "api-keys",
"id": "api-key-uuid-2",
"attributes": {
"name": "Staging Dashboard API Access",
"prefix": "pk_live_def456",
"expires_at": null,
"revoked": false,
"inserted_at": "2024-01-10T08:15:00Z",
"last_used_at": null
},
"relationships": {
"entity": {
"data": {
"type": "users",
"id": "user-uuid"
}
}
}
}
],
"meta": {
"page": {
"number": 1,
"size": 20,
"total": 2,
"total_pages": 1
}
},
"links": {
"self": "https://api.prowler.com/api/v1/api-keys?page[number]=1",
"first": "https://api.prowler.com/api/v1/api-keys?page[number]=1",
"last": "https://api.prowler.com/api/v1/api-keys?page[number]=1"
}
}
```
### Response Fields
- `id` (UUID) - Unique identifier for the API key record
- `name` (string) - Descriptive name (3-100 characters)
- `prefix` (string, read-only) - First characters of the key for identification
- `expires_at` (datetime, nullable) - Expiration date in ISO 8601 format
- `revoked` (boolean, read-only) - Whether key has been revoked
- `inserted_at` (datetime, read-only) - When key was created
- `last_used_at` (datetime, nullable, read-only) - Last usage timestamp
<Note>
The full API key value is never returned after creation. Only the prefix is shown for identification purposes.
</Note>
+89
View File
@@ -0,0 +1,89 @@
---
title: "Retrieve API Key"
api: "GET /api/v1/api-keys/{id}"
description: "Fetch detailed information about a specific API key by its ID."
---
Retrieve detailed information about a specific API key. Note that the full API key value is never returned, only metadata about the key.
## Path Parameters
- `id` (required) - The UUID of the API key to retrieve
## Query Parameters
### Field Selection
- `fields[api-keys]` - Specify which fields to return: `name`, `prefix`, `expires_at`, `revoked`, `inserted_at`, `last_used_at`, `entity`
### Include Related Resources
- `include` - Include related resources. Available: `entity`
## Example Request
```bash
curl -X GET "https://api.prowler.com/api/v1/api-keys/{id}" \
-H "Authorization: Bearer YOUR_API_KEY" \
-H "Content-Type: application/vnd.api+json"
```
## Response
### Success Response (200 OK)
Returns detailed metadata about the API key:
```json
{
"data": {
"type": "api-keys",
"id": "api-key-uuid",
"attributes": {
"name": "Production CI/CD Pipeline",
"prefix": "pk_live_abc123",
"expires_at": "2025-12-31T23:59:59Z",
"revoked": false,
"inserted_at": "2024-01-15T10:30:00Z",
"last_used_at": "2024-01-20T14:22:00Z"
},
"relationships": {
"entity": {
"data": {
"type": "users",
"id": "user-uuid"
}
}
}
}
}
```
### Response Fields
- `id` (UUID) - Unique identifier for the API key record
- `name` (string) - Descriptive name (3-100 characters)
- `prefix` (string, read-only) - First characters of the key for identification
- `expires_at` (datetime, nullable) - Expiration date in ISO 8601 format
- `revoked` (boolean, read-only) - Whether key has been revoked
- `inserted_at` (datetime, read-only) - When key was created
- `last_used_at` (datetime, nullable, read-only) - Last usage timestamp
### Error Responses
**404 Not Found** - API key does not exist
```json
{
"errors": [
{
"status": "404",
"title": "Not Found",
"detail": "API key not found"
}
]
}
```
<Note>
The full API key value is never returned after creation. Only the prefix is shown for identification purposes.
</Note>
+86
View File
@@ -0,0 +1,86 @@
---
title: "Revoke API Key"
api: "DELETE /api/v1/api-keys/{id}/revoke"
description: "Revoke an API key by its ID. This action is irreversible."
---
Revoke an API key, preventing it from being used for further API requests. This action cannot be undone.
## Path Parameters
- `id` (required) - The UUID of the API key to revoke
## Example Request
```bash
curl -X DELETE "https://api.prowler.com/api/v1/api-keys/{id}/revoke" \
-H "Authorization: Bearer YOUR_API_KEY" \
-H "Content-Type: application/vnd.api+json"
```
## Response
### Success Response (200 OK)
Returns the API key with `revoked` status set to `true`:
```json
{
"data": {
"type": "api-keys",
"id": "api-key-uuid",
"attributes": {
"name": "Production CI/CD Pipeline",
"prefix": "pk_live_abc123",
"expires_at": "2025-12-31T23:59:59Z",
"revoked": true,
"inserted_at": "2024-01-15T10:30:00Z",
"last_used_at": "2024-01-20T14:22:00Z"
},
"relationships": {
"entity": {
"data": {
"type": "users",
"id": "user-uuid"
}
}
}
}
}
```
### Error Responses
**404 Not Found** - API key does not exist
```json
{
"errors": [
{
"status": "404",
"title": "Not Found",
"detail": "API key not found"
}
]
}
```
**409 Conflict** - API key already revoked
```json
{
"errors": [
{
"status": "409",
"title": "Already Revoked",
"detail": "This API key has already been revoked"
}
]
}
```
<Warning>
Revoking an API key is permanent and cannot be undone. Any applications using this key will immediately lose access. The `revoked` field will be set to `true` and the key will no longer authenticate API requests.
</Warning>
<Tip>
Monitor the `last_used_at` field before revoking to ensure the key is no longer in active use.
</Tip>
+88
View File
@@ -0,0 +1,88 @@
---
title: "Update API Key"
api: "PATCH /api/v1/api-keys/{id}"
description: "Modify certain fields of an existing API key without affecting other settings."
---
Partially update an API key's properties. This endpoint allows you to modify the name or expiration date without affecting other settings.
## Path Parameters
- `id` (required) - The UUID of the API key to update
## Request Body
The request must follow the JSON:API specification format:
```json
{
"data": {
"type": "api-keys",
"id": "string",
"attributes": {
"name": "string",
"expires_at": "2024-12-31T23:59:59Z"
}
}
}
```
### Attributes
- `name` (optional, string) - Update the API key name (3-100 characters)
<Note>
Only the `name` field can be updated. Other fields like `expires_at`, `prefix`, and `revoked` are read-only and cannot be modified after creation.
</Note>
## Example Request
```bash
curl -X PATCH "https://api.prowler.com/api/v1/api-keys/{id}" \
-H "Authorization: Bearer YOUR_API_KEY" \
-H "Content-Type: application/vnd.api+json" \
-d '{
"data": {
"type": "api-keys",
"id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
"attributes": {
"name": "Updated API Key Name"
}
}
}'
```
## Response
### Success Response (200 OK)
Returns the updated API key with modified metadata:
```json
{
"data": {
"type": "api-keys",
"id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
"attributes": {
"name": "Updated API Key Name",
"prefix": "pk_live_abc123",
"expires_at": "2025-12-31T23:59:59Z",
"revoked": false,
"inserted_at": "2024-01-15T10:30:00Z",
"last_used_at": "2024-01-20T14:22:00Z"
},
"relationships": {
"entity": {
"data": {
"type": "users",
"id": "user-uuid"
}
}
}
}
}
```
<Warning>
The full API key value is never returned in responses after creation. Only the prefix is shown for identification.
</Warning>
@@ -0,0 +1,7 @@
---
title: "Retrieve Compliance Attributes"
api: "GET /api/v1/compliance-overviews/attributes"
description: "Get compliance framework attributes."
---
Retrieve detailed attributes for compliance frameworks.
+17
View File
@@ -0,0 +1,17 @@
---
title: "List Compliance Overviews"
api: "GET /api/v1/compliance-overviews"
description: "Retrieve compliance framework coverage for scans."
---
Get compliance status across frameworks like CIS, PCI-DSS, HIPAA, etc.
## Query Parameters
- `filter[scan]` - Filter by scan UUID (required)
- `filter[framework]` - Filter by framework name
## Example
\`\`\`bash
curl "https://api.prowler.com/api/v1/compliance-overviews?filter[scan]=scan-uuid" \\
-H "Authorization: Bearer YOUR_API_KEY"
\`\`\`
@@ -0,0 +1,7 @@
---
title: "Retrieve Compliance Metadata"
api: "GET /api/v1/compliance-overviews/metadata"
description: "Get available compliance frameworks and metadata."
---
Fetch list of available compliance frameworks and their details.
@@ -0,0 +1,7 @@
---
title: "Retrieve Compliance Requirements"
api: "GET /api/v1/compliance-overviews/requirements"
description: "Get compliance requirements details."
---
Fetch specific requirements for compliance frameworks.

Some files were not shown because too many files have changed in this diff Show More