diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index 0ba3557f38..3e3510496f 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -8,7 +8,7 @@ body: attributes: label: Feature search options: - - label: I have searched the existing issues and this feature has not been requested yet + - label: I have searched the existing issues and this feature has not been requested yet or is already in our [Public Roadmap](https://roadmap.prowler.com/roadmap) required: true - type: dropdown id: component diff --git a/.github/workflows/find-secrets.yml b/.github/workflows/find-secrets.yml index d1258b3827..6428cf8f08 100644 --- a/.github/workflows/find-secrets.yml +++ b/.github/workflows/find-secrets.yml @@ -1,19 +1,33 @@ name: 'Tools: TruffleHog' -on: pull_request +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - trufflehog: + scan-secrets: runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: - - name: Checkout + - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 - - name: TruffleHog OSS - uses: trufflesecurity/trufflehog@466da5b0bb161144f6afca9afe5d57975828c410 # v3.90.8 + + - name: Scan for secrets with TruffleHog + uses: trufflesecurity/trufflehog@ad6fc8fb446b8fafbf7ea8193d2d6bfd42f45690 # v3.90.11 with: - path: ./ - base: ${{ github.event.repository.default_branch }} - head: HEAD - extra_args: --only-verified + extra_args: '--results=verified,unknown' diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 9dfa8993a1..fad94177f7 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,17 +1,29 @@ -name: Prowler - PR Labeler +name: 'Tools: PR Labeler' on: - pull_request_target: - branches: - - "master" - - "v3" - - "v4.*" + pull_request_target: + branches: + - 'master' + - 'v5.*' + types: + - 'opened' + - 'reopened' + - 'synchronize' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: labeler: + runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + steps: - - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + - name: Apply labels to PR + uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + with: + sync-labels: true diff --git a/.github/workflows/mcp-container-build-push.yml b/.github/workflows/mcp-container-build-push.yml index fa4f5be578..aecec30592 100644 --- a/.github/workflows/mcp-container-build-push.yml +++ b/.github/workflows/mcp-container-build-push.yml @@ -3,21 +3,13 @@ name: 'MCP: Container Build and Push' on: push: branches: - - "master" + - 'master' paths: - - "mcp_server/**" - - ".github/workflows/mcp-container-build-push.yml" - - # Uncomment to test this workflow on PRs - # pull_request: - # branches: - # - "master" - # paths: - # - "mcp_server/**" - # - ".github/workflows/mcp-container-build-push.yml" - + - 'mcp_server/**' + - '.github/workflows/mcp-container-build-push.yml' release: - types: [published] + types: + - 'published' permissions: contents: read @@ -41,6 +33,7 @@ jobs: setup: if: github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest + timeout-minutes: 5 outputs: short-sha: ${{ steps.set-short-sha.outputs.short-sha }} steps: @@ -51,8 +44,12 @@ jobs: container-build-push: needs: setup runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write steps: - - name: Checkout + - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Login to DockerHub @@ -64,7 +61,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - name: Build and push container (latest) + - name: Build and push MCP container (latest) if: github.event_name == 'push' uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: @@ -83,7 +80,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - - name: Build and push container (release) + - name: Build and push MCP container (release) if: github.event_name == 'release' uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: @@ -103,7 +100,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - - name: Trigger deployment + - name: Trigger MCP deployment if: github.event_name == 'push' uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 with: diff --git a/.github/workflows/mcp-pull-request.yml b/.github/workflows/mcp-pull-request.yml new file mode 100644 index 0000000000..31b4934c35 --- /dev/null +++ b/.github/workflows/mcp-pull-request.yml @@ -0,0 +1,80 @@ +name: 'MCP: Pull Request' + +on: + push: + branches: + - 'master' + - 'v5.*' + paths: + - '.github/workflows/mcp-pull-request.yml' + - 'mcp_server/**' + - '!mcp_server/README.md' + - '!mcp_server/CHANGELOG.md' + pull_request: + branches: + - 'master' + - 'v5.*' + paths: + - '.github/workflows/mcp-pull-request.yml' + - 'mcp_server/**' + - '!mcp_server/README.md' + - '!mcp_server/CHANGELOG.md' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MCP_WORKING_DIR: ./mcp_server + IMAGE_NAME: prowler-mcp + +jobs: + dockerfile-lint: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Lint Dockerfile with Hadolint + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: mcp_server/Dockerfile + + container-build-and-scan: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build MCP container + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.MCP_WORKING_DIR }} + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Scan MCP container with Trivy + uses: ./.github/actions/trivy-scan + with: + image-name: ${{ env.IMAGE_NAME }} + image-tag: ${{ github.sha }} + fail-on-critical: 'false' + severity: 'CRITICAL' diff --git a/.github/workflows/pr-check-changelog.yml b/.github/workflows/pr-check-changelog.yml new file mode 100644 index 0000000000..f8de212e2a --- /dev/null +++ b/.github/workflows/pr-check-changelog.yml @@ -0,0 +1,103 @@ +name: 'Tools: Check Changelog' + +on: + pull_request: + types: + - 'opened' + - 'synchronize' + - 'reopened' + - 'labeled' + - 'unlabeled' + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check-changelog: + if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + env: + MONITORED_FOLDERS: 'api ui prowler mcp_server' + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + api/** + ui/** + prowler/** + mcp_server/** + + - name: Check for folder changes and changelog presence + id: check-folders + run: | + missing_changelogs="" + + # Check api folder + if [[ "${{ steps.changed-files.outputs.any_changed }}" == "true" ]]; then + for folder in $MONITORED_FOLDERS; do + # Get files changed in this folder + changed_in_folder=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "^${folder}/" || true) + + if [ -n "$changed_in_folder" ]; then + echo "Detected changes in ${folder}/" + + # Check if CHANGELOG.md was updated + if ! echo "$changed_in_folder" | grep -q "^${folder}/CHANGELOG.md$"; then + echo "No changelog update found for ${folder}/" + missing_changelogs="${missing_changelogs}- \`${folder}\`"$'\n' + fi + fi + done + fi + + { + echo "missing_changelogs<> $GITHUB_OUTPUT + + - name: Find existing changelog comment + if: github.event.pull_request.head.repo.full_name == github.repository + id: find-comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: '' + + - name: Update PR comment with changelog status + if: github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-id: ${{ steps.find-comment.outputs.comment-id }} + edit-mode: replace + body: | + + ${{ steps.check-folders.outputs.missing_changelogs != '' && format('⚠️ **Changes detected in the following folders without a corresponding update to the `CHANGELOG.md`:** + + {0} + + Please add an entry to the corresponding `CHANGELOG.md` file to maintain a clear history of changes.', steps.check-folders.outputs.missing_changelogs) || '✅ All necessary `CHANGELOG.md` files have been updated.' }} + + - name: Fail if changelog is missing + if: steps.check-folders.outputs.missing_changelogs != '' + run: | + echo "::error::Missing changelog updates in some folders" + exit 1 diff --git a/.github/workflows/pr-conflict-checker.yml b/.github/workflows/pr-conflict-checker.yml index 77280d5136..3761d252a3 100644 --- a/.github/workflows/pr-conflict-checker.yml +++ b/.github/workflows/pr-conflict-checker.yml @@ -1,42 +1,40 @@ -name: Prowler - PR Conflict Checker +name: 'Tools: PR Conflict Checker' on: - pull_request: + pull_request_target: types: - - opened - - synchronize - - reopened + - 'opened' + - 'synchronize' + - 'reopened' branches: - - "master" - - "v5.*" - # Leaving this commented until we find a way to run it for forks but in Prowler's context - # pull_request_target: - # types: - # - opened - # - synchronize - # - reopened - # branches: - # - "master" - # - "v5.*" + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: - conflict-checker: + check-conflicts: runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read pull-requests: write issues: write steps: - - name: Checkout repository + - name: Checkout PR head uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 - name: Get changed files id: changed-files uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: - files: | - ** + files: '**' - name: Check for conflict markers id: conflict-check @@ -51,10 +49,10 @@ jobs: if [ -f "$file" ]; then echo "Checking file: $file" - # Look for conflict markers - if grep -l "^<<<<<<<\|^=======\|^>>>>>>>" "$file" 2>/dev/null; then + # Look for conflict markers (more precise regex) + if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then echo "Conflict markers found in: $file" - CONFLICT_FILES="$CONFLICT_FILES$file " + CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n' HAS_CONFLICTS=true fi fi @@ -62,114 +60,64 @@ jobs: if [ "$HAS_CONFLICTS" = true ]; then echo "has_conflicts=true" >> $GITHUB_OUTPUT - echo "conflict_files=$CONFLICT_FILES" >> $GITHUB_OUTPUT - echo "Conflict markers detected in files: $CONFLICT_FILES" + { + echo "conflict_files<> $GITHUB_OUTPUT + echo "Conflict markers detected" else echo "has_conflicts=false" >> $GITHUB_OUTPUT echo "No conflict markers found in changed files" fi - - name: Add conflict label - if: steps.conflict-check.outputs.has_conflicts == 'true' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - with: - github-token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - script: | - const { data: labels } = await github.rest.issues.listLabelsOnIssue({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); + - name: Manage conflict label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }} + run: | + LABEL_NAME="has-conflicts" - const hasConflictLabel = labels.some(label => label.name === 'has-conflicts'); + # Add or remove label based on conflict status + if [ "$HAS_CONFLICTS" = "true" ]; then + echo "Adding conflict label to PR #${PR_NUMBER}..." + gh pr edit "$PR_NUMBER" --add-label "$LABEL_NAME" --repo ${{ github.repository }} || true + else + echo "Removing conflict label from PR #${PR_NUMBER}..." + gh pr edit "$PR_NUMBER" --remove-label "$LABEL_NAME" --repo ${{ github.repository }} || true + fi - if (!hasConflictLabel) { - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - labels: ['has-conflicts'] - }); - console.log('Added has-conflicts label'); - } else { - console.log('has-conflicts label already exists'); - } - - - name: Remove conflict label - if: steps.conflict-check.outputs.has_conflicts == 'false' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - with: - github-token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - script: | - try { - await github.rest.issues.removeLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - name: 'has-conflicts' - }); - console.log('Removed has-conflicts label'); - } catch (error) { - if (error.status === 404) { - console.log('has-conflicts label was not present'); - } else { - throw error; - } - } - - - name: Find existing conflict comment - if: steps.conflict-check.outputs.has_conflicts == 'true' + - name: Find existing comment uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 id: find-comment with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' - body-regex: '(⚠️ \*\*Conflict Markers Detected\*\*|✅ \*\*Conflict Markers Resolved\*\*)' + body-includes: '' - - name: Create or update conflict comment - if: steps.conflict-check.outputs.has_conflicts == 'true' + - name: Create or update comment uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 with: comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} edit-mode: replace body: | - ⚠️ **Conflict Markers Detected** + + ${{ steps.conflict-check.outputs.has_conflicts == 'true' && '⚠️ **Conflict Markers Detected**' || '✅ **Conflict Markers Resolved**' }} - This pull request contains unresolved conflict markers in the following files: - ``` - ${{ steps.conflict-check.outputs.conflict_files }} - ``` + ${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('This pull request contains unresolved conflict markers in the following files: + + {0} Please resolve these conflicts by: 1. Locating the conflict markers: `<<<<<<<`, `=======`, and `>>>>>>>` 2. Manually editing the files to resolve the conflicts 3. Removing all conflict markers - 4. Committing and pushing the changes - - - name: Find existing conflict comment when resolved - if: steps.conflict-check.outputs.has_conflicts == 'false' - uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 - id: find-resolved-comment - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-regex: '(⚠️ \*\*Conflict Markers Detected\*\*|✅ \*\*Conflict Markers Resolved\*\*)' - - - name: Update comment when conflicts resolved - if: steps.conflict-check.outputs.has_conflicts == 'false' && steps.find-resolved-comment.outputs.comment-id != '' - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 - with: - comment-id: ${{ steps.find-resolved-comment.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - edit-mode: replace - body: | - ✅ **Conflict Markers Resolved** - - All conflict markers have been successfully resolved in this pull request. + 4. Committing and pushing the changes', steps.conflict-check.outputs.conflict_files) || 'All conflict markers have been successfully resolved in this pull request.' }} - name: Fail workflow if conflicts detected if: steps.conflict-check.outputs.has_conflicts == 'true' run: | - echo "::error::Workflow failed due to conflict markers in files: ${{ steps.conflict-check.outputs.conflict_files }}" + echo "::error::Workflow failed due to conflict markers detected in the PR" exit 1 diff --git a/.github/workflows/pull-request-merged.yml b/.github/workflows/pr-merged.yml similarity index 55% rename from .github/workflows/pull-request-merged.yml rename to .github/workflows/pr-merged.yml index 4b5a93aabd..d8255026e6 100644 --- a/.github/workflows/pull-request-merged.yml +++ b/.github/workflows/pr-merged.yml @@ -1,27 +1,31 @@ -name: Prowler - Merged Pull Request +name: 'Tools: PR Merged' on: pull_request_target: - branches: ['master'] - types: ['closed'] + branches: + - 'master' + types: + - 'closed' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: false jobs: trigger-cloud-pull-request: - name: Trigger Cloud Pull Request if: github.event.pull_request.merged == true && github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.pull_request.merge_commit_sha }} - - - name: Set short git commit SHA + - name: Calculate short commit SHA id: vars run: | - shortSha=$(git rev-parse --short ${{ github.event.pull_request.merge_commit_sha }}) - echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV + SHORT_SHA="${{ github.event.pull_request.merge_commit_sha }}" + echo "SHORT_SHA=${SHORT_SHA::7}" >> $GITHUB_ENV - - name: Trigger pull request + - name: Trigger Cloud repository pull request uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} @@ -31,8 +35,12 @@ jobs: { "PROWLER_COMMIT_SHA": "${{ github.event.pull_request.merge_commit_sha }}", "PROWLER_COMMIT_SHORT_SHA": "${{ env.SHORT_SHA }}", + "PROWLER_PR_NUMBER": "${{ github.event.pull_request.number }}", "PROWLER_PR_TITLE": ${{ toJson(github.event.pull_request.title) }}, "PROWLER_PR_LABELS": ${{ toJson(github.event.pull_request.labels.*.name) }}, "PROWLER_PR_BODY": ${{ toJson(github.event.pull_request.body) }}, - "PROWLER_PR_URL": ${{ toJson(github.event.pull_request.html_url) }} + "PROWLER_PR_URL": ${{ toJson(github.event.pull_request.html_url) }}, + "PROWLER_PR_MERGED_BY": "${{ github.event.pull_request.merged_by.login }}", + "PROWLER_PR_BASE_BRANCH": "${{ github.event.pull_request.base.ref }}", + "PROWLER_PR_HEAD_BRANCH": "${{ github.event.pull_request.head.ref }}" } diff --git a/.github/workflows/prowler-release-preparation.yml b/.github/workflows/prepare-release.yml similarity index 89% rename from .github/workflows/prowler-release-preparation.yml rename to .github/workflows/prepare-release.yml index 7ecd937554..55b7d2e441 100644 --- a/.github/workflows/prowler-release-preparation.yml +++ b/.github/workflows/prepare-release.yml @@ -1,6 +1,6 @@ -name: Prowler - Release Preparation +name: 'Tools: Prepare Release' -run-name: Prowler Release Preparation for ${{ inputs.prowler_version }} +run-name: 'Prepare Release for Prowler ${{ inputs.prowler_version }}' on: workflow_dispatch: @@ -10,18 +10,23 @@ on: required: true type: string +concurrency: + group: ${{ github.workflow }}-${{ inputs.prowler_version }} + cancel-in-progress: false + env: - PROWLER_VERSION: ${{ github.event.inputs.prowler_version }} + PROWLER_VERSION: ${{ inputs.prowler_version }} jobs: prepare-release: - if: github.repository == 'prowler-cloud/prowler' + if: github.event_name == 'workflow_dispatch' && github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: write pull-requests: write steps: - - name: Checkout code + - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 @@ -34,15 +39,15 @@ jobs: - name: Install Poetry run: | - python3 -m pip install --user poetry + python3 -m pip install --user poetry==2.1.1 echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Configure Git run: | - git config --global user.name "prowler-bot" - git config --global user.email "179230569+prowler-bot@users.noreply.github.com" + git config --global user.name 'prowler-bot' + git config --global user.email '179230569+prowler-bot@users.noreply.github.com' - - name: Parse version and determine branch + - name: Parse version and read changelogs run: | # Validate version format (reusing pattern from sdk-bump-version.yml) if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then @@ -119,7 +124,7 @@ jobs: exit 1 fi - - name: Extract changelog entries + - name: Extract and combine changelog entries run: | set -e @@ -145,8 +150,8 @@ jobs: # Remove --- separators sed -i '/^---$/d' "$output_file" - # Remove trailing empty lines - sed -i '/^$/d' "$output_file" + # Remove only trailing empty lines (not all empty lines) + sed -i -e :a -e '/^\s*$/d;N;ba' "$output_file" } # Calculate expected versions for this release @@ -242,10 +247,15 @@ jobs: echo "" >> combined_changelog.md fi + # Add fallback message if no changelogs were added + if [ ! -s combined_changelog.md ]; then + echo "No component changes detected for this release." >> combined_changelog.md + fi + echo "Combined changelog preview:" cat combined_changelog.md - - name: Checkout existing branch for patch release + - name: Checkout release branch for patch release if: ${{ env.PATCH_VERSION != '0' }} run: | echo "Patch release detected, checking out existing branch $BRANCH_NAME..." @@ -260,7 +270,7 @@ jobs: exit 1 fi - - name: Verify version in pyproject.toml + - name: Verify SDK version in pyproject.toml run: | CURRENT_VERSION=$(grep '^version = ' pyproject.toml | sed -E 's/version = "([^"]+)"/\1/' | tr -d '[:space:]') PROWLER_VERSION_TRIMMED=$(echo "$PROWLER_VERSION" | tr -d '[:space:]') @@ -270,7 +280,7 @@ jobs: fi echo "✓ pyproject.toml version: $CURRENT_VERSION" - - name: Verify version in prowler/config/config.py + - name: Verify SDK version in prowler/config/config.py run: | CURRENT_VERSION=$(grep '^prowler_version = ' prowler/config/config.py | sed -E 's/prowler_version = "([^"]+)"/\1/' | tr -d '[:space:]') PROWLER_VERSION_TRIMMED=$(echo "$PROWLER_VERSION" | tr -d '[:space:]') @@ -280,7 +290,7 @@ jobs: fi echo "✓ prowler/config/config.py version: $CURRENT_VERSION" - - name: Verify version in api/pyproject.toml + - name: Verify API version in api/pyproject.toml if: ${{ env.HAS_API_CHANGES == 'true' }} run: | CURRENT_API_VERSION=$(grep '^version = ' api/pyproject.toml | sed -E 's/version = "([^"]+)"/\1/' | tr -d '[:space:]') @@ -291,7 +301,7 @@ jobs: fi echo "✓ api/pyproject.toml version: $CURRENT_API_VERSION" - - name: Verify prowler dependency in api/pyproject.toml + - name: Verify API prowler dependency in api/pyproject.toml if: ${{ env.PATCH_VERSION != '0' && env.HAS_API_CHANGES == 'true' }} run: | CURRENT_PROWLER_REF=$(grep 'prowler @ git+https://github.com/prowler-cloud/prowler.git@' api/pyproject.toml | sed -E 's/.*@([^"]+)".*/\1/' | tr -d '[:space:]') @@ -302,7 +312,7 @@ jobs: fi echo "✓ api/pyproject.toml prowler dependency: $CURRENT_PROWLER_REF" - - name: Verify version in api/src/backend/api/v1/views.py + - name: Verify API version in api/src/backend/api/v1/views.py if: ${{ env.HAS_API_CHANGES == 'true' }} run: | CURRENT_API_VERSION=$(grep 'spectacular_settings.VERSION = ' api/src/backend/api/v1/views.py | sed -E 's/.*spectacular_settings.VERSION = "([^"]+)".*/\1/' | tr -d '[:space:]') @@ -313,7 +323,7 @@ jobs: fi echo "✓ api/src/backend/api/v1/views.py version: $CURRENT_API_VERSION" - - name: Checkout existing release branch for minor release + - name: Checkout release branch for minor release if: ${{ env.PATCH_VERSION == '0' }} run: | echo "Minor release detected (patch = 0), checking out existing branch $BRANCH_NAME..." @@ -325,19 +335,12 @@ jobs: exit 1 fi - - name: Prepare prowler dependency update for minor release + - name: Update API prowler dependency for minor release if: ${{ env.PATCH_VERSION == '0' }} run: | CURRENT_PROWLER_REF=$(grep 'prowler @ git+https://github.com/prowler-cloud/prowler.git@' api/pyproject.toml | sed -E 's/.*@([^"]+)".*/\1/' | tr -d '[:space:]') BRANCH_NAME_TRIMMED=$(echo "$BRANCH_NAME" | tr -d '[:space:]') - # Create a temporary branch for the PR from the minor version branch - TEMP_BRANCH="update-api-dependency-$BRANCH_NAME_TRIMMED-$(date +%s)" - echo "TEMP_BRANCH=$TEMP_BRANCH" >> $GITHUB_ENV - - # Create temp branch from the current minor version branch - git checkout -b "$TEMP_BRANCH" - # Minor release: update the dependency to use the release branch echo "Updating prowler dependency from '$CURRENT_PROWLER_REF' to '$BRANCH_NAME_TRIMMED'" sed -i "s|prowler @ git+https://github.com/prowler-cloud/prowler.git@[^\"]*\"|prowler @ git+https://github.com/prowler-cloud/prowler.git@$BRANCH_NAME_TRIMMED\"|" api/pyproject.toml @@ -355,20 +358,19 @@ jobs: poetry lock cd .. - # Commit and push the temporary branch - git add api/pyproject.toml api/poetry.lock - git commit -m "chore(api): update prowler dependency to $BRANCH_NAME_TRIMMED for release $PROWLER_VERSION" - git push origin "$TEMP_BRANCH" - echo "✓ Prepared prowler dependency update to: $UPDATED_PROWLER_REF" - - name: Create Pull Request against release branch + - name: Create PR for API dependency update if: ${{ env.PATCH_VERSION == '0' }} uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - branch: ${{ env.TEMP_BRANCH }} + commit-message: 'chore(api): update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}' + branch: update-api-dependency-${{ env.BRANCH_NAME }}-${{ github.run_number }} base: ${{ env.BRANCH_NAME }} + add-paths: | + api/pyproject.toml + api/poetry.lock title: "chore(api): Update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}" body: | ### Description @@ -401,5 +403,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Clean up temporary files + if: always() run: | rm -f prowler_changelog.md api_changelog.md ui_changelog.md mcp_changelog.md combined_changelog.md diff --git a/.github/workflows/pull-request-check-changelog.yml b/.github/workflows/pull-request-check-changelog.yml deleted file mode 100644 index 3b96e6d499..0000000000 --- a/.github/workflows/pull-request-check-changelog.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Prowler - Check Changelog - -on: - pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] - -jobs: - check-changelog: - if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - pull-requests: write - env: - MONITORED_FOLDERS: "api ui prowler mcp_server" - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - fetch-depth: 0 - - - name: Get list of changed files - id: changed_files - run: | - git fetch origin ${{ github.base_ref }} - git diff --name-only origin/${{ github.base_ref }}...HEAD > changed_files.txt - cat changed_files.txt - - - name: Check for folder changes and changelog presence - id: check_folders - run: | - missing_changelogs="" - - for folder in $MONITORED_FOLDERS; do - if grep -q "^${folder}/" changed_files.txt; then - echo "Detected changes in ${folder}/" - if ! grep -q "^${folder}/CHANGELOG.md$" changed_files.txt; then - echo "No changelog update found for ${folder}/" - missing_changelogs="${missing_changelogs}- \`${folder}\`\n" - fi - fi - done - - echo "missing_changelogs<> $GITHUB_OUTPUT - echo -e "${missing_changelogs}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Find existing changelog comment - if: github.event.pull_request.head.repo.full_name == github.repository - id: find_comment - uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad #v4.0.0 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: '' - - - name: Update PR comment with changelog status - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-id: ${{ steps.find_comment.outputs.comment-id }} - edit-mode: replace - body: | - - ${{ steps.check_folders.outputs.missing_changelogs != '' && format('⚠️ **Changes detected in the following folders without a corresponding update to the `CHANGELOG.md`:** - - {0} - - Please add an entry to the corresponding `CHANGELOG.md` file to maintain a clear history of changes.', steps.check_folders.outputs.missing_changelogs) || '✅ All necessary `CHANGELOG.md` files have been updated. Great job! 🎉' }} - - - name: Fail if changelog is missing - if: steps.check_folders.outputs.missing_changelogs != '' - run: | - echo "ERROR: Missing changelog updates in some folders." - exit 1 diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index dea6f1f3e2..81f191765c 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -18,6 +18,7 @@ jobs: AUTH_TRUST_HOST: true NEXTAUTH_URL: 'http://localhost:3000' NEXT_PUBLIC_API_BASE_URL: 'http://localhost:8080/api/v1' + E2E_NEW_PASSWORD: ${{ secrets.E2E_NEW_PASSWORD }} steps: - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 diff --git a/.gitignore b/.gitignore index 4b39b18b83..bc8c66d39b 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,12 @@ secrets-*/ # JUnit Reports junit-reports/ +# Test and coverage artifacts +*_coverage.xml +pytest_*.xml +.coverage +htmlcov/ + # VSCode files .vscode/ @@ -83,3 +89,6 @@ CLAUDE.md # MCP Server mcp_server/prowler_mcp_server/prowler_app/server.py mcp_server/prowler_mcp_server/prowler_app/utils/schema.yaml + +# Compliance report +*.pdf diff --git a/Makefile b/Makefile index 368bb885bd..861c9cf7fe 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,14 @@ help: ## Show this help. @echo "Prowler Makefile" @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +##@ Build no cache +build-no-cache-dev: + docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat + ##@ Development Environment run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, and workers - docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat --build + docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat + +##@ Development Environment +build-and-run-api-dev: build-no-cache-dev run-api-dev + diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index b372edd33d..f01574421e 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -9,7 +9,7 @@ All notable changes to the **Prowler API** are documented in this file. --- -## [1.14.0] (Prowler UNRELEASED) +## [1.14.0] (Prowler 5.13.0) ### Added - Default JWT keys are generated and stored if they are missing from configuration [(#8655)](https://github.com/prowler-cloud/prowler/pull/8655) @@ -19,6 +19,7 @@ All notable changes to the **Prowler API** are documented in this file. - API Key support [(#8805)](https://github.com/prowler-cloud/prowler/pull/8805) - SAML role mapping protection for single-admin tenants to prevent accidental lockout [(#8882)](https://github.com/prowler-cloud/prowler/pull/8882) - Support for `passed_findings` and `total_findings` fields in compliance requirement overview for accurate Prowler ThreatScore calculation [(#8582)](https://github.com/prowler-cloud/prowler/pull/8582) +- PDF reporting for Prowler ThreatScore [(#8867)](https://github.com/prowler-cloud/prowler/pull/8867) - Database read replica support [(#8869)](https://github.com/prowler-cloud/prowler/pull/8869) - Support Common Cloud Controls for AWS, Azure and GCP [(#8000)](https://github.com/prowler-cloud/prowler/pull/8000) - Add `provider_id__in` filter support to findings and findings severity overview endpoints [(#8951)](https://github.com/prowler-cloud/prowler/pull/8951) diff --git a/api/poetry.lock b/api/poetry.lock index 61929f3bdc..40313d9fa5 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1256,6 +1256,98 @@ files = [ {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, ] +[[package]] +name = "contourpy" +version = "1.3.3" +description = "Python library for calculating contours of 2D quadrilateral grids" +optional = false +python-versions = ">=3.11" +groups = ["main"] +files = [ + {file = "contourpy-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1"}, + {file = "contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db"}, + {file = "contourpy-1.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620"}, + {file = "contourpy-1.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f"}, + {file = "contourpy-1.3.3-cp311-cp311-win32.whl", hash = "sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff"}, + {file = "contourpy-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42"}, + {file = "contourpy-1.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470"}, + {file = "contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb"}, + {file = "contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1"}, + {file = "contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7"}, + {file = "contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411"}, + {file = "contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69"}, + {file = "contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b"}, + {file = "contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc"}, + {file = "contourpy-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5"}, + {file = "contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9"}, + {file = "contourpy-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659"}, + {file = "contourpy-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7"}, + {file = "contourpy-1.3.3-cp313-cp313-win32.whl", hash = "sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d"}, + {file = "contourpy-1.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263"}, + {file = "contourpy-1.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9"}, + {file = "contourpy-1.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d"}, + {file = "contourpy-1.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b"}, + {file = "contourpy-1.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a"}, + {file = "contourpy-1.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e"}, + {file = "contourpy-1.3.3-cp313-cp313t-win32.whl", hash = "sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3"}, + {file = "contourpy-1.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8"}, + {file = "contourpy-1.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301"}, + {file = "contourpy-1.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a"}, + {file = "contourpy-1.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3"}, + {file = "contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b"}, + {file = "contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36"}, + {file = "contourpy-1.3.3-cp314-cp314-win32.whl", hash = "sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d"}, + {file = "contourpy-1.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd"}, + {file = "contourpy-1.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339"}, + {file = "contourpy-1.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772"}, + {file = "contourpy-1.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0"}, + {file = "contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4"}, + {file = "contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f"}, + {file = "contourpy-1.3.3-cp314-cp314t-win32.whl", hash = "sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae"}, + {file = "contourpy-1.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc"}, + {file = "contourpy-1.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77"}, + {file = "contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880"}, +] + +[package.dependencies] +numpy = ">=1.25" + +[package.extras] +bokeh = ["bokeh", "selenium"] +docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] +mypy = ["bokeh", "contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.17.0)", "types-Pillow"] +test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] +test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] + [[package]] name = "coverage" version = "7.5.4" @@ -1390,6 +1482,22 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["certifi (>=2024)", "cryptography-vectors (==44.0.1)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] +[[package]] +name = "cycler" +version = "0.12.1" +description = "Composable style cycles" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"}, + {file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"}, +] + +[package.extras] +docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] +tests = ["pytest", "pytest-cov", "pytest-xdist"] + [[package]] name = "dash" version = "3.1.1" @@ -2120,6 +2228,87 @@ werkzeug = ">=3.1.0" async = ["asgiref (>=3.2)"] dotenv = ["python-dotenv"] +[[package]] +name = "fonttools" +version = "4.60.1" +description = "Tools to manipulate font files" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "fonttools-4.60.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9a52f254ce051e196b8fe2af4634c2d2f02c981756c6464dc192f1b6050b4e28"}, + {file = "fonttools-4.60.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c7420a2696a44650120cdd269a5d2e56a477e2bfa9d95e86229059beb1c19e15"}, + {file = "fonttools-4.60.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee0c0b3b35b34f782afc673d503167157094a16f442ace7c6c5e0ca80b08f50c"}, + {file = "fonttools-4.60.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:282dafa55f9659e8999110bd8ed422ebe1c8aecd0dc396550b038e6c9a08b8ea"}, + {file = "fonttools-4.60.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4ba4bd646e86de16160f0fb72e31c3b9b7d0721c3e5b26b9fa2fc931dfdb2652"}, + {file = "fonttools-4.60.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0b0835ed15dd5b40d726bb61c846a688f5b4ce2208ec68779bc81860adb5851a"}, + {file = "fonttools-4.60.1-cp310-cp310-win32.whl", hash = "sha256:1525796c3ffe27bb6268ed2a1bb0dcf214d561dfaf04728abf01489eb5339dce"}, + {file = "fonttools-4.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:268ecda8ca6cb5c4f044b1fb9b3b376e8cd1b361cef275082429dc4174907038"}, + {file = "fonttools-4.60.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7b4c32e232a71f63a5d00259ca3d88345ce2a43295bb049d21061f338124246f"}, + {file = "fonttools-4.60.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3630e86c484263eaac71d117085d509cbcf7b18f677906824e4bace598fb70d2"}, + {file = "fonttools-4.60.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5c1015318e4fec75dd4943ad5f6a206d9727adf97410d58b7e32ab644a807914"}, + {file = "fonttools-4.60.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e6c58beb17380f7c2ea181ea11e7db8c0ceb474c9dd45f48e71e2cb577d146a1"}, + {file = "fonttools-4.60.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ec3681a0cb34c255d76dd9d865a55f260164adb9fa02628415cdc2d43ee2c05d"}, + {file = "fonttools-4.60.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f4b5c37a5f40e4d733d3bbaaef082149bee5a5ea3156a785ff64d949bd1353fa"}, + {file = "fonttools-4.60.1-cp311-cp311-win32.whl", hash = "sha256:398447f3d8c0c786cbf1209711e79080a40761eb44b27cdafffb48f52bcec258"}, + {file = "fonttools-4.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:d066ea419f719ed87bc2c99a4a4bfd77c2e5949cb724588b9dd58f3fd90b92bf"}, + {file = "fonttools-4.60.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7b0c6d57ab00dae9529f3faf187f2254ea0aa1e04215cf2f1a8ec277c96661bc"}, + {file = "fonttools-4.60.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:839565cbf14645952d933853e8ade66a463684ed6ed6c9345d0faf1f0e868877"}, + {file = "fonttools-4.60.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8177ec9676ea6e1793c8a084a90b65a9f778771998eb919d05db6d4b1c0b114c"}, + {file = "fonttools-4.60.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:996a4d1834524adbb423385d5a629b868ef9d774670856c63c9a0408a3063401"}, + {file = "fonttools-4.60.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a46b2f450bc79e06ef3b6394f0c68660529ed51692606ad7f953fc2e448bc903"}, + {file = "fonttools-4.60.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6ec722ee589e89a89f5b7574f5c45604030aa6ae24cb2c751e2707193b466fed"}, + {file = "fonttools-4.60.1-cp312-cp312-win32.whl", hash = "sha256:b2cf105cee600d2de04ca3cfa1f74f1127f8455b71dbad02b9da6ec266e116d6"}, + {file = "fonttools-4.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:992775c9fbe2cf794786fa0ffca7f09f564ba3499b8fe9f2f80bd7197db60383"}, + {file = "fonttools-4.60.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6f68576bb4bbf6060c7ab047b1574a1ebe5c50a17de62830079967b211059ebb"}, + {file = "fonttools-4.60.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:eedacb5c5d22b7097482fa834bda0dafa3d914a4e829ec83cdea2a01f8c813c4"}, + {file = "fonttools-4.60.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b33a7884fabd72bdf5f910d0cf46be50dce86a0362a65cfc746a4168c67eb96c"}, + {file = "fonttools-4.60.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2409d5fb7b55fd70f715e6d34e7a6e4f7511b8ad29a49d6df225ee76da76dd77"}, + {file = "fonttools-4.60.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c8651e0d4b3bdeda6602b85fdc2abbefc1b41e573ecb37b6779c4ca50753a199"}, + {file = "fonttools-4.60.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:145daa14bf24824b677b9357c5e44fd8895c2a8f53596e1b9ea3496081dc692c"}, + {file = "fonttools-4.60.1-cp313-cp313-win32.whl", hash = "sha256:2299df884c11162617a66b7c316957d74a18e3758c0274762d2cc87df7bc0272"}, + {file = "fonttools-4.60.1-cp313-cp313-win_amd64.whl", hash = "sha256:a3db56f153bd4c5c2b619ab02c5db5192e222150ce5a1bc10f16164714bc39ac"}, + {file = "fonttools-4.60.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a884aef09d45ba1206712c7dbda5829562d3fea7726935d3289d343232ecb0d3"}, + {file = "fonttools-4.60.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8a44788d9d91df72d1a5eac49b31aeb887a5f4aab761b4cffc4196c74907ea85"}, + {file = "fonttools-4.60.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e852d9dda9f93ad3651ae1e3bb770eac544ec93c3807888798eccddf84596537"}, + {file = "fonttools-4.60.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:154cb6ee417e417bf5f7c42fe25858c9140c26f647c7347c06f0cc2d47eff003"}, + {file = "fonttools-4.60.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5664fd1a9ea7f244487ac8f10340c4e37664675e8667d6fee420766e0fb3cf08"}, + {file = "fonttools-4.60.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:583b7f8e3c49486e4d489ad1deacfb8d5be54a8ef34d6df824f6a171f8511d99"}, + {file = "fonttools-4.60.1-cp314-cp314-win32.whl", hash = "sha256:66929e2ea2810c6533a5184f938502cfdaea4bc3efb7130d8cc02e1c1b4108d6"}, + {file = "fonttools-4.60.1-cp314-cp314-win_amd64.whl", hash = "sha256:f3d5be054c461d6a2268831f04091dc82753176f6ea06dc6047a5e168265a987"}, + {file = "fonttools-4.60.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:b6379e7546ba4ae4b18f8ae2b9bc5960936007a1c0e30b342f662577e8bc3299"}, + {file = "fonttools-4.60.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9d0ced62b59e0430b3690dbc5373df1c2aa7585e9a8ce38eff87f0fd993c5b01"}, + {file = "fonttools-4.60.1-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:875cb7764708b3132637f6c5fb385b16eeba0f7ac9fa45a69d35e09b47045801"}, + {file = "fonttools-4.60.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a184b2ea57b13680ab6d5fbde99ccef152c95c06746cb7718c583abd8f945ccc"}, + {file = "fonttools-4.60.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:026290e4ec76583881763fac284aca67365e0be9f13a7fb137257096114cb3bc"}, + {file = "fonttools-4.60.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f0e8817c7d1a0c2eedebf57ef9a9896f3ea23324769a9a2061a80fe8852705ed"}, + {file = "fonttools-4.60.1-cp314-cp314t-win32.whl", hash = "sha256:1410155d0e764a4615774e5c2c6fc516259fe3eca5882f034eb9bfdbee056259"}, + {file = "fonttools-4.60.1-cp314-cp314t-win_amd64.whl", hash = "sha256:022beaea4b73a70295b688f817ddc24ed3e3418b5036ffcd5658141184ef0d0c"}, + {file = "fonttools-4.60.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:122e1a8ada290423c493491d002f622b1992b1ab0b488c68e31c413390dc7eb2"}, + {file = "fonttools-4.60.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a140761c4ff63d0cb9256ac752f230460ee225ccef4ad8f68affc723c88e2036"}, + {file = "fonttools-4.60.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0eae96373e4b7c9e45d099d7a523444e3554360927225c1cdae221a58a45b856"}, + {file = "fonttools-4.60.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:596ecaca36367027d525b3b426d8a8208169d09edcf8c7506aceb3a38bfb55c7"}, + {file = "fonttools-4.60.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2ee06fc57512144d8b0445194c2da9f190f61ad51e230f14836286470c99f854"}, + {file = "fonttools-4.60.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b42d86938e8dda1cd9a1a87a6d82f1818eaf933348429653559a458d027446da"}, + {file = "fonttools-4.60.1-cp39-cp39-win32.whl", hash = "sha256:8b4eb332f9501cb1cd3d4d099374a1e1306783ff95489a1026bde9eb02ccc34a"}, + {file = "fonttools-4.60.1-cp39-cp39-win_amd64.whl", hash = "sha256:7473a8ed9ed09aeaa191301244a5a9dbe46fe0bf54f9d6cd21d83044c3321217"}, + {file = "fonttools-4.60.1-py3-none-any.whl", hash = "sha256:906306ac7afe2156fcf0042173d6ebbb05416af70f6b370967b47f8f00103bbb"}, + {file = "fonttools-4.60.1.tar.gz", hash = "sha256:ef00af0439ebfee806b25f24c8f92109157ff3fac5731dc7867957812e87b8d9"}, +] + +[package.extras] +all = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\"", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0) ; python_version <= \"3.12\"", "xattr ; sys_platform == \"darwin\"", "zopfli (>=0.1.4)"] +graphite = ["lz4 (>=1.7.4.2)"] +interpolatable = ["munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\""] +lxml = ["lxml (>=4.0)"] +pathops = ["skia-pathops (>=0.5.0)"] +plot = ["matplotlib"] +repacker = ["uharfbuzz (>=0.23.0)"] +symfont = ["sympy"] +type1 = ["xattr ; sys_platform == \"darwin\""] +unicode = ["unicodedata2 (>=15.1.0) ; python_version <= \"3.12\""] +woff = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "zopfli (>=0.1.4)"] + [[package]] name = "freezegun" version = "1.5.1" @@ -2787,6 +2976,117 @@ files = [ [package.dependencies] referencing = ">=0.31.0" +[[package]] +name = "kiwisolver" +version = "1.4.9" +description = "A fast implementation of the Cassowary constraint solver" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "kiwisolver-1.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b4b4d74bda2b8ebf4da5bd42af11d02d04428b2c32846e4c2c93219df8a7987b"}, + {file = "kiwisolver-1.4.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fb3b8132019ea572f4611d770991000d7f58127560c4889729248eb5852a102f"}, + {file = "kiwisolver-1.4.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:84fd60810829c27ae375114cd379da1fa65e6918e1da405f356a775d49a62bcf"}, + {file = "kiwisolver-1.4.9-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b78efa4c6e804ecdf727e580dbb9cba85624d2e1c6b5cb059c66290063bd99a9"}, + {file = "kiwisolver-1.4.9-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4efec7bcf21671db6a3294ff301d2fc861c31faa3c8740d1a94689234d1b415"}, + {file = "kiwisolver-1.4.9-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90f47e70293fc3688b71271100a1a5453aa9944a81d27ff779c108372cf5567b"}, + {file = "kiwisolver-1.4.9-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fdca1def57a2e88ef339de1737a1449d6dbf5fab184c54a1fca01d541317154"}, + {file = "kiwisolver-1.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:9cf554f21be770f5111a1690d42313e140355e687e05cf82cb23d0a721a64a48"}, + {file = "kiwisolver-1.4.9-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fc1795ac5cd0510207482c3d1d3ed781143383b8cfd36f5c645f3897ce066220"}, + {file = "kiwisolver-1.4.9-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:ccd09f20ccdbbd341b21a67ab50a119b64a403b09288c27481575105283c1586"}, + {file = "kiwisolver-1.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:540c7c72324d864406a009d72f5d6856f49693db95d1fbb46cf86febef873634"}, + {file = "kiwisolver-1.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:ede8c6d533bc6601a47ad4046080d36b8fc99f81e6f1c17b0ac3c2dc91ac7611"}, + {file = "kiwisolver-1.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:7b4da0d01ac866a57dd61ac258c5607b4cd677f63abaec7b148354d2b2cdd536"}, + {file = "kiwisolver-1.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eb14a5da6dc7642b0f3a18f13654847cd8b7a2550e2645a5bda677862b03ba16"}, + {file = "kiwisolver-1.4.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39a219e1c81ae3b103643d2aedb90f1ef22650deb266ff12a19e7773f3e5f089"}, + {file = "kiwisolver-1.4.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2405a7d98604b87f3fc28b1716783534b1b4b8510d8142adca34ee0bc3c87543"}, + {file = "kiwisolver-1.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dc1ae486f9abcef254b5618dfb4113dd49f94c68e3e027d03cf0143f3f772b61"}, + {file = "kiwisolver-1.4.9-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a1f570ce4d62d718dce3f179ee78dac3b545ac16c0c04bb363b7607a949c0d1"}, + {file = "kiwisolver-1.4.9-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb27e7b78d716c591e88e0a09a2139c6577865d7f2e152488c2cc6257f460872"}, + {file = "kiwisolver-1.4.9-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:15163165efc2f627eb9687ea5f3a28137217d217ac4024893d753f46bce9de26"}, + {file = "kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bdee92c56a71d2b24c33a7d4c2856bd6419d017e08caa7802d2963870e315028"}, + {file = "kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:412f287c55a6f54b0650bd9b6dce5aceddb95864a1a90c87af16979d37c89771"}, + {file = "kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2c93f00dcba2eea70af2be5f11a830a742fe6b579a1d4e00f47760ef13be247a"}, + {file = "kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f117e1a089d9411663a3207ba874f31be9ac8eaa5b533787024dc07aeb74f464"}, + {file = "kiwisolver-1.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:be6a04e6c79819c9a8c2373317d19a96048e5a3f90bec587787e86a1153883c2"}, + {file = "kiwisolver-1.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:0ae37737256ba2de764ddc12aed4956460277f00c4996d51a197e72f62f5eec7"}, + {file = "kiwisolver-1.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ac5a486ac389dddcc5bef4f365b6ae3ffff2c433324fb38dd35e3fab7c957999"}, + {file = "kiwisolver-1.4.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2ba92255faa7309d06fe44c3a4a97efe1c8d640c2a79a5ef728b685762a6fd2"}, + {file = "kiwisolver-1.4.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a2899935e724dd1074cb568ce7ac0dce28b2cd6ab539c8e001a8578eb106d14"}, + {file = "kiwisolver-1.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f6008a4919fdbc0b0097089f67a1eb55d950ed7e90ce2cc3e640abadd2757a04"}, + {file = "kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67bb8b474b4181770f926f7b7d2f8c0248cbcb78b660fdd41a47054b28d2a752"}, + {file = "kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2327a4a30d3ee07d2fbe2e7933e8a37c591663b96ce42a00bc67461a87d7df77"}, + {file = "kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a08b491ec91b1d5053ac177afe5290adacf1f0f6307d771ccac5de30592d198"}, + {file = "kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d8fc5c867c22b828001b6a38d2eaeb88160bf5783c6cb4a5e440efc981ce286d"}, + {file = "kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3b3115b2581ea35bb6d1f24a4c90af37e5d9b49dcff267eeed14c3893c5b86ab"}, + {file = "kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858e4c22fb075920b96a291928cb7dea5644e94c0ee4fcd5af7e865655e4ccf2"}, + {file = "kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ed0fecd28cc62c54b262e3736f8bb2512d8dcfdc2bcf08be5f47f96bf405b145"}, + {file = "kiwisolver-1.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:f68208a520c3d86ea51acf688a3e3002615a7f0238002cccc17affecc86a8a54"}, + {file = "kiwisolver-1.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:2c1a4f57df73965f3f14df20b80ee29e6a7930a57d2d9e8491a25f676e197c60"}, + {file = "kiwisolver-1.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5d0432ccf1c7ab14f9949eec60c5d1f924f17c037e9f8b33352fa05799359b8"}, + {file = "kiwisolver-1.4.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efb3a45b35622bb6c16dbfab491a8f5a391fe0e9d45ef32f4df85658232ca0e2"}, + {file = "kiwisolver-1.4.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a12cf6398e8a0a001a059747a1cbf24705e18fe413bc22de7b3d15c67cffe3f"}, + {file = "kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b67e6efbf68e077dd71d1a6b37e43e1a99d0bff1a3d51867d45ee8908b931098"}, + {file = "kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5656aa670507437af0207645273ccdfee4f14bacd7f7c67a4306d0dcaeaf6eed"}, + {file = "kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bfc08add558155345129c7803b3671cf195e6a56e7a12f3dde7c57d9b417f525"}, + {file = "kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:40092754720b174e6ccf9e845d0d8c7d8e12c3d71e7fc35f55f3813e96376f78"}, + {file = "kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:497d05f29a1300d14e02e6441cf0f5ee81c1ff5a304b0d9fb77423974684e08b"}, + {file = "kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bdd1a81a1860476eb41ac4bc1e07b3f07259e6d55bbf739b79c8aaedcf512799"}, + {file = "kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:e6b93f13371d341afee3be9f7c5964e3fe61d5fa30f6a30eb49856935dfe4fc3"}, + {file = "kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d75aa530ccfaa593da12834b86a0724f58bff12706659baa9227c2ccaa06264c"}, + {file = "kiwisolver-1.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:dd0a578400839256df88c16abddf9ba14813ec5f21362e1fe65022e00c883d4d"}, + {file = "kiwisolver-1.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:d4188e73af84ca82468f09cadc5ac4db578109e52acb4518d8154698d3a87ca2"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:5a0f2724dfd4e3b3ac5a82436a8e6fd16baa7d507117e4279b660fe8ca38a3a1"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1b11d6a633e4ed84fc0ddafd4ebfd8ea49b3f25082c04ad12b8315c11d504dc1"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61874cdb0a36016354853593cffc38e56fc9ca5aa97d2c05d3dcf6922cd55a11"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:60c439763a969a6af93b4881db0eed8fadf93ee98e18cbc35bc8da868d0c4f0c"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92a2f997387a1b79a75e7803aa7ded2cfbe2823852ccf1ba3bcf613b62ae3197"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a31d512c812daea6d8b3be3b2bfcbeb091dbb09177706569bcfc6240dcf8b41c"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:52a15b0f35dad39862d376df10c5230155243a2c1a436e39eb55623ccbd68185"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a30fd6fdef1430fd9e1ba7b3398b5ee4e2887783917a687d86ba69985fb08748"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cc9617b46837c6468197b5945e196ee9ca43057bb7d9d1ae688101e4e1dddf64"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:0ab74e19f6a2b027ea4f845a78827969af45ce790e6cb3e1ebab71bdf9f215ff"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dba5ee5d3981160c28d5490f0d1b7ed730c22470ff7f6cc26cfcfaacb9896a07"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-win_arm64.whl", hash = "sha256:0749fd8f4218ad2e851e11cc4dc05c7cbc0cbc4267bdfdb31782e65aace4ee9c"}, + {file = "kiwisolver-1.4.9-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:9928fe1eb816d11ae170885a74d074f57af3a0d65777ca47e9aeb854a1fba386"}, + {file = "kiwisolver-1.4.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d0005b053977e7b43388ddec89fa567f43d4f6d5c2c0affe57de5ebf290dc552"}, + {file = "kiwisolver-1.4.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2635d352d67458b66fd0667c14cb1d4145e9560d503219034a18a87e971ce4f3"}, + {file = "kiwisolver-1.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:767c23ad1c58c9e827b649a9ab7809fd5fd9db266a9cf02b0e926ddc2c680d58"}, + {file = "kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72d0eb9fba308b8311685c2268cf7d0a0639a6cd027d8128659f72bdd8a024b4"}, + {file = "kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f68e4f3eeca8fb22cc3d731f9715a13b652795ef657a13df1ad0c7dc0e9731df"}, + {file = "kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d84cd4061ae292d8ac367b2c3fa3aad11cb8625a95d135fe93f286f914f3f5a6"}, + {file = "kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a60ea74330b91bd22a29638940d115df9dc00af5035a9a2a6ad9399ffb4ceca5"}, + {file = "kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ce6a3a4e106cf35c2d9c4fa17c05ce0b180db622736845d4315519397a77beaf"}, + {file = "kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:77937e5e2a38a7b48eef0585114fe7930346993a88060d0bf886086d2aa49ef5"}, + {file = "kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:24c175051354f4a28c5d6a31c93906dc653e2bf234e8a4bbfb964892078898ce"}, + {file = "kiwisolver-1.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:0763515d4df10edf6d06a3c19734e2566368980d21ebec439f33f9eb936c07b7"}, + {file = "kiwisolver-1.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:0e4e2bf29574a6a7b7f6cb5fa69293b9f96c928949ac4a53ba3f525dffb87f9c"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d976bbb382b202f71c67f77b0ac11244021cfa3f7dfd9e562eefcea2df711548"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2489e4e5d7ef9a1c300a5e0196e43d9c739f066ef23270607d45aba368b91f2d"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e2ea9f7ab7fbf18fffb1b5434ce7c69a07582f7acc7717720f1d69f3e806f90c"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b34e51affded8faee0dfdb705416153819d8ea9250bbbf7ea1b249bdeb5f1122"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8aacd3d4b33b772542b2e01beb50187536967b514b00003bdda7589722d2a64"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7cf974dd4e35fa315563ac99d6287a1024e4dc2077b8a7d7cd3d2fb65d283134"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:85bd218b5ecfbee8c8a82e121802dcb519a86044c9c3b2e4aef02fa05c6da370"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0856e241c2d3df4efef7c04a1e46b1936b6120c9bcf36dd216e3acd84bc4fb21"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9af39d6551f97d31a4deebeac6f45b156f9755ddc59c07b402c148f5dbb6482a"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:bb4ae2b57fc1d8cbd1cf7b1d9913803681ffa903e7488012be5b76dedf49297f"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:aedff62918805fb62d43a4aa2ecd4482c380dc76cd31bd7c8878588a61bd0369"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:1fa333e8b2ce4d9660f2cda9c0e1b6bafcfb2457a9d259faa82289e73ec24891"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:4a48a2ce79d65d363597ef7b567ce3d14d68783d2b2263d98db3d9477805ba32"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4d1d9e582ad4d63062d34077a9a1e9f3c34088a2ec5135b1f7190c07cf366527"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:deed0c7258ceb4c44ad5ec7d9918f9f14fd05b2be86378d86cf50e63d1e7b771"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a590506f303f512dff6b7f75fd2fd18e16943efee932008fe7140e5fa91d80e"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e09c2279a4d01f099f52d5c4b3d9e208e91edcbd1a175c9662a8b16e000fece9"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c9e7cdf45d594ee04d5be1b24dd9d49f3d1590959b2271fb30b5ca2b262c00fb"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:720e05574713db64c356e86732c0f3c5252818d05f9df320f0ad8380641acea5"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:17680d737d5335b552994a2008fab4c851bcd7de33094a82067ef3a576ff02fa"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:85b5352f94e490c028926ea567fc569c52ec79ce131dadb968d3853e809518c2"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:464415881e4801295659462c49461a24fb107c140de781d55518c4b80cb6790f"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:fb940820c63a9590d31d88b815e7a3aa5915cad3ce735ab45f0c730b39547de1"}, + {file = "kiwisolver-1.4.9.tar.gz", hash = "sha256:c3b22c26c6fd6811b0ae8363b95ca8ce4ea3c202d3d0975b2914310ceb1bcc4d"}, +] + [[package]] name = "kombu" version = "5.5.4" @@ -3137,6 +3437,85 @@ dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"] docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"] tests = ["pytest", "simplejson"] +[[package]] +name = "matplotlib" +version = "3.10.6" +description = "Python plotting package" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "matplotlib-3.10.6-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:bc7316c306d97463a9866b89d5cc217824e799fa0de346c8f68f4f3d27c8693d"}, + {file = "matplotlib-3.10.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d00932b0d160ef03f59f9c0e16d1e3ac89646f7785165ce6ad40c842db16cc2e"}, + {file = "matplotlib-3.10.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8fa4c43d6bfdbfec09c733bca8667de11bfa4970e8324c471f3a3632a0301c15"}, + {file = "matplotlib-3.10.6-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ea117a9c1627acaa04dbf36265691921b999cbf515a015298e54e1a12c3af837"}, + {file = "matplotlib-3.10.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:08fc803293b4e1694ee325896030de97f74c141ccff0be886bb5915269247676"}, + {file = "matplotlib-3.10.6-cp310-cp310-win_amd64.whl", hash = "sha256:2adf92d9b7527fbfb8818e050260f0ebaa460f79d61546374ce73506c9421d09"}, + {file = "matplotlib-3.10.6-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:905b60d1cb0ee604ce65b297b61cf8be9f4e6cfecf95a3fe1c388b5266bc8f4f"}, + {file = "matplotlib-3.10.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7bac38d816637343e53d7185d0c66677ff30ffb131044a81898b5792c956ba76"}, + {file = "matplotlib-3.10.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:942a8de2b5bfff1de31d95722f702e2966b8a7e31f4e68f7cd963c7cd8861cf6"}, + {file = "matplotlib-3.10.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3276c85370bc0dfca051ec65c5817d1e0f8f5ce1b7787528ec8ed2d524bbc2f"}, + {file = "matplotlib-3.10.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9df5851b219225731f564e4b9e7f2ac1e13c9e6481f941b5631a0f8e2d9387ce"}, + {file = "matplotlib-3.10.6-cp311-cp311-win_amd64.whl", hash = "sha256:abb5d9478625dd9c9eb51a06d39aae71eda749ae9b3138afb23eb38824026c7e"}, + {file = "matplotlib-3.10.6-cp311-cp311-win_arm64.whl", hash = "sha256:886f989ccfae63659183173bb3fced7fd65e9eb793c3cc21c273add368536951"}, + {file = "matplotlib-3.10.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:31ca662df6a80bd426f871105fdd69db7543e28e73a9f2afe80de7e531eb2347"}, + {file = "matplotlib-3.10.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1678bb61d897bb4ac4757b5ecfb02bfb3fddf7f808000fb81e09c510712fda75"}, + {file = "matplotlib-3.10.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:56cd2d20842f58c03d2d6e6c1f1cf5548ad6f66b91e1e48f814e4fb5abd1cb95"}, + {file = "matplotlib-3.10.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:662df55604a2f9a45435566d6e2660e41efe83cd94f4288dfbf1e6d1eae4b0bb"}, + {file = "matplotlib-3.10.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:08f141d55148cd1fc870c3387d70ca4df16dee10e909b3b038782bd4bda6ea07"}, + {file = "matplotlib-3.10.6-cp312-cp312-win_amd64.whl", hash = "sha256:590f5925c2d650b5c9d813c5b3b5fc53f2929c3f8ef463e4ecfa7e052044fb2b"}, + {file = "matplotlib-3.10.6-cp312-cp312-win_arm64.whl", hash = "sha256:f44c8d264a71609c79a78d50349e724f5d5fc3684ead7c2a473665ee63d868aa"}, + {file = "matplotlib-3.10.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:819e409653c1106c8deaf62e6de6b8611449c2cd9939acb0d7d4e57a3d95cc7a"}, + {file = "matplotlib-3.10.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:59c8ac8382fefb9cb71308dde16a7c487432f5255d8f1fd32473523abecfecdf"}, + {file = "matplotlib-3.10.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:84e82d9e0fd70c70bc55739defbd8055c54300750cbacf4740c9673a24d6933a"}, + {file = "matplotlib-3.10.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25f7a3eb42d6c1c56e89eacd495661fc815ffc08d9da750bca766771c0fd9110"}, + {file = "matplotlib-3.10.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f9c862d91ec0b7842920a4cfdaaec29662195301914ea54c33e01f1a28d014b2"}, + {file = "matplotlib-3.10.6-cp313-cp313-win_amd64.whl", hash = "sha256:1b53bd6337eba483e2e7d29c5ab10eee644bc3a2491ec67cc55f7b44583ffb18"}, + {file = "matplotlib-3.10.6-cp313-cp313-win_arm64.whl", hash = "sha256:cbd5eb50b7058b2892ce45c2f4e92557f395c9991f5c886d1bb74a1582e70fd6"}, + {file = "matplotlib-3.10.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:acc86dd6e0e695c095001a7fccff158c49e45e0758fdf5dcdbb0103318b59c9f"}, + {file = "matplotlib-3.10.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e228cd2ffb8f88b7d0b29e37f68ca9aaf83e33821f24a5ccc4f082dd8396bc27"}, + {file = "matplotlib-3.10.6-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:658bc91894adeab669cf4bb4a186d049948262987e80f0857216387d7435d833"}, + {file = "matplotlib-3.10.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8913b7474f6dd83ac444c9459c91f7f0f2859e839f41d642691b104e0af056aa"}, + {file = "matplotlib-3.10.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:091cea22e059b89f6d7d1a18e2c33a7376c26eee60e401d92a4d6726c4e12706"}, + {file = "matplotlib-3.10.6-cp313-cp313t-win_amd64.whl", hash = "sha256:491e25e02a23d7207629d942c666924a6b61e007a48177fdd231a0097b7f507e"}, + {file = "matplotlib-3.10.6-cp313-cp313t-win_arm64.whl", hash = "sha256:3d80d60d4e54cda462e2cd9a086d85cd9f20943ead92f575ce86885a43a565d5"}, + {file = "matplotlib-3.10.6-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:70aaf890ce1d0efd482df969b28a5b30ea0b891224bb315810a3940f67182899"}, + {file = "matplotlib-3.10.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1565aae810ab79cb72e402b22facfa6501365e73ebab70a0fdfb98488d2c3c0c"}, + {file = "matplotlib-3.10.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3b23315a01981689aa4e1a179dbf6ef9fbd17143c3eea77548c2ecfb0499438"}, + {file = "matplotlib-3.10.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30fdd37edf41a4e6785f9b37969de57aea770696cb637d9946eb37470c94a453"}, + {file = "matplotlib-3.10.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bc31e693da1c08012c764b053e702c1855378e04102238e6a5ee6a7117c53a47"}, + {file = "matplotlib-3.10.6-cp314-cp314-win_amd64.whl", hash = "sha256:05be9bdaa8b242bc6ff96330d18c52f1fc59c6fb3a4dd411d953d67e7e1baf98"}, + {file = "matplotlib-3.10.6-cp314-cp314-win_arm64.whl", hash = "sha256:f56a0d1ab05d34c628592435781d185cd99630bdfd76822cd686fb5a0aecd43a"}, + {file = "matplotlib-3.10.6-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:94f0b4cacb23763b64b5dace50d5b7bfe98710fed5f0cef5c08135a03399d98b"}, + {file = "matplotlib-3.10.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cc332891306b9fb39462673d8225d1b824c89783fee82840a709f96714f17a5c"}, + {file = "matplotlib-3.10.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee1d607b3fb1590deb04b69f02ea1d53ed0b0bf75b2b1a5745f269afcbd3cdd3"}, + {file = "matplotlib-3.10.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:376a624a218116461696b27b2bbf7a8945053e6d799f6502fc03226d077807bf"}, + {file = "matplotlib-3.10.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:83847b47f6524c34b4f2d3ce726bb0541c48c8e7692729865c3df75bfa0f495a"}, + {file = "matplotlib-3.10.6-cp314-cp314t-win_amd64.whl", hash = "sha256:c7e0518e0d223683532a07f4b512e2e0729b62674f1b3a1a69869f98e6b1c7e3"}, + {file = "matplotlib-3.10.6-cp314-cp314t-win_arm64.whl", hash = "sha256:4dd83e029f5b4801eeb87c64efd80e732452781c16a9cf7415b7b63ec8f374d7"}, + {file = "matplotlib-3.10.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:13fcd07ccf17e354398358e0307a1f53f5325dca22982556ddb9c52837b5af41"}, + {file = "matplotlib-3.10.6-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:470fc846d59d1406e34fa4c32ba371039cd12c2fe86801159a965956f2575bd1"}, + {file = "matplotlib-3.10.6-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7173f8551b88f4ef810a94adae3128c2530e0d07529f7141be7f8d8c365f051"}, + {file = "matplotlib-3.10.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f2d684c3204fa62421bbf770ddfebc6b50130f9cad65531eeba19236d73bb488"}, + {file = "matplotlib-3.10.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:6f4a69196e663a41d12a728fab8751177215357906436804217d6d9cf0d4d6cf"}, + {file = "matplotlib-3.10.6-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d6ca6ef03dfd269f4ead566ec6f3fb9becf8dab146fb999022ed85ee9f6b3eb"}, + {file = "matplotlib-3.10.6.tar.gz", hash = "sha256:ec01b645840dd1996df21ee37f208cd8ba57644779fa20464010638013d3203c"}, +] + +[package.dependencies] +contourpy = ">=1.0.1" +cycler = ">=0.10" +fonttools = ">=4.22.0" +kiwisolver = ">=1.3.1" +numpy = ">=1.23" +packaging = ">=20.0" +pillow = ">=8" +pyparsing = ">=2.3.1" +python-dateutil = ">=2.7" + +[package.extras] +dev = ["meson-python (>=0.13.1,<0.17.0)", "pybind11 (>=2.13.2,!=2.13.3)", "setuptools (>=64)", "setuptools_scm (>=7)"] + [[package]] name = "mccabe" version = "0.7.0" @@ -3857,6 +4236,131 @@ files = [ [package.dependencies] setuptools = "*" +[[package]] +name = "pillow" +version = "11.3.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, + {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae"}, + {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9"}, + {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:040a5b691b0713e1f6cbe222e0f4f74cd233421e105850ae3b3c0ceda520f42e"}, + {file = "pillow-11.3.0-cp310-cp310-win32.whl", hash = "sha256:89bd777bc6624fe4115e9fac3352c79ed60f3bb18651420635f26e643e3dd1f6"}, + {file = "pillow-11.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:19d2ff547c75b8e3ff46f4d9ef969a06c30ab2d4263a9e287733aa8b2429ce8f"}, + {file = "pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f"}, + {file = "pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722"}, + {file = "pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f"}, + {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e"}, + {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:932c754c2d51ad2b2271fd01c3d121daaa35e27efae2a616f77bf164bc0b3e94"}, + {file = "pillow-11.3.0-cp311-cp311-win32.whl", hash = "sha256:b4b8f3efc8d530a1544e5962bd6b403d5f7fe8b9e08227c6b255f98ad82b4ba0"}, + {file = "pillow-11.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:1a992e86b0dd7aeb1f053cd506508c0999d710a8f07b4c791c63843fc6a807ac"}, + {file = "pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd"}, + {file = "pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4"}, + {file = "pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024"}, + {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809"}, + {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6be31e3fc9a621e071bc17bb7de63b85cbe0bfae91bb0363c893cbe67247780d"}, + {file = "pillow-11.3.0-cp312-cp312-win32.whl", hash = "sha256:7b161756381f0918e05e7cb8a371fff367e807770f8fe92ecb20d905d0e1c149"}, + {file = "pillow-11.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6444696fce635783440b7f7a9fc24b3ad10a9ea3f0ab66c5905be1c19ccf17d"}, + {file = "pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1c627742b539bba4309df89171356fcb3cc5a9178355b2727d1b74a6cf155fbd"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:30b7c02f3899d10f13d7a48163c8969e4e653f8b43416d23d13d1bbfdc93b9f8"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f"}, + {file = "pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c"}, + {file = "pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8"}, + {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2"}, + {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:45dfc51ac5975b938e9809451c51734124e73b04d0f0ac621649821a63852e7b"}, + {file = "pillow-11.3.0-cp313-cp313-win32.whl", hash = "sha256:a4d336baed65d50d37b88ca5b60c0fa9d81e3a87d4a7930d3880d1624d5b31f3"}, + {file = "pillow-11.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bce5c4fd0921f99d2e858dc4d4d64193407e1b99478bc5cacecba2311abde51"}, + {file = "pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580"}, + {file = "pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e"}, + {file = "pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59"}, + {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe"}, + {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83e1b0161c9d148125083a35c1c5a89db5b7054834fd4387499e06552035236c"}, + {file = "pillow-11.3.0-cp313-cp313t-win32.whl", hash = "sha256:2a3117c06b8fb646639dce83694f2f9eac405472713fcb1ae887469c0d4f6788"}, + {file = "pillow-11.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:857844335c95bea93fb39e0fa2726b4d9d758850b34075a7e3ff4f4fa3aa3b31"}, + {file = "pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e"}, + {file = "pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12"}, + {file = "pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77"}, + {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874"}, + {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:98a9afa7b9007c67ed84c57c9e0ad86a6000da96eaa638e4f8abe5b65ff83f0a"}, + {file = "pillow-11.3.0-cp314-cp314-win32.whl", hash = "sha256:02a723e6bf909e7cea0dac1b0e0310be9d7650cd66222a5f1c571455c0a45214"}, + {file = "pillow-11.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a418486160228f64dd9e9efcd132679b7a02a5f22c982c78b6fc7dab3fefb635"}, + {file = "pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6"}, + {file = "pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae"}, + {file = "pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477"}, + {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50"}, + {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a1bc6ba083b145187f648b667e05a2534ecc4b9f2784c2cbe3089e44868f2b9b"}, + {file = "pillow-11.3.0-cp314-cp314t-win32.whl", hash = "sha256:118ca10c0d60b06d006be10a501fd6bbdfef559251ed31b794668ed569c87e12"}, + {file = "pillow-11.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8924748b688aa210d79883357d102cd64690e56b923a186f35a82cbc10f997db"}, + {file = "pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa"}, + {file = "pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f"}, + {file = "pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:23cff760a9049c502721bdb743a7cb3e03365fafcdfc2ef9784610714166e5a4"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6359a3bc43f57d5b375d1ad54a0074318a0844d11b76abccf478c37c986d3cfc"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a"}, + {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978"}, + {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:97afb3a00b65cc0804d1c7abddbf090a81eaac02768af58cbdcaaa0a931e0b6d"}, + {file = "pillow-11.3.0-cp39-cp39-win32.whl", hash = "sha256:ea944117a7974ae78059fcc1800e5d3295172bb97035c0c1d9345fca1419da71"}, + {file = "pillow-11.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:e5c5858ad8ec655450a7c7df532e9842cf8df7cc349df7225c60d5d348c8aada"}, + {file = "pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8"}, + {file = "pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +test-arrow = ["pyarrow"] +tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "trove-classifiers (>=2024.10.12)"] +typing = ["typing-extensions ; python_version < \"3.10\""] +xmp = ["defusedxml"] + [[package]] name = "platformdirs" version = "4.3.8" @@ -5016,6 +5520,29 @@ attrs = ">=22.2.0" rpds-py = ">=0.7.0" typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} +[[package]] +name = "reportlab" +version = "4.4.4" +description = "The Reportlab Toolkit" +optional = false +python-versions = "<4,>=3.9" +groups = ["main"] +files = [ + {file = "reportlab-4.4.4-py3-none-any.whl", hash = "sha256:299b3b0534e7202bb94ed2ddcd7179b818dcda7de9d8518a57c85a58a1ebaadb"}, + {file = "reportlab-4.4.4.tar.gz", hash = "sha256:cb2f658b7f4a15be2cc68f7203aa67faef67213edd4f2d4bdd3eb20dab75a80d"}, +] + +[package.dependencies] +charset-normalizer = "*" +pillow = ">=9.0.0" + +[package.extras] +accel = ["rl_accel (>=0.9.0,<1.1)"] +bidi = ["rlbidi"] +pycairo = ["freetype-py (>=2.3.0,<2.4)", "rlPyCairo (>=0.2.0,<1)"] +renderpm = ["rl_renderPM (>=4.0.3,<4.1)"] +shaping = ["uharfbuzz"] + [[package]] name = "requests" version = "2.32.5" @@ -6259,4 +6786,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "03442fd4673006c5a74374f90f53621fd1c9d117279fe6cc0355ef833eb7f9bb" +content-hash = "3c9164d668d37d6373eb5200bbe768232ead934d9312b9c68046b1df922789f3" diff --git a/api/pyproject.toml b/api/pyproject.toml index 1573f851ce..b1cd4af120 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "drf-spectacular-jsonapi==0.5.1", "gunicorn==23.0.0", "lxml==5.3.2", - "prowler @ git+https://github.com/prowler-cloud/prowler.git@DEVREL-91-add-ia-c-to-the-app", + "prowler @ git+https://github.com/prowler-cloud/prowler.git@master", "psycopg2-binary==2.9.9", "pytest-celery[redis] (>=1.0.1,<2.0.0)", "sentry-sdk[django] (>=2.20.0,<3.0.0)", @@ -33,7 +33,9 @@ dependencies = [ "xmlsec==1.3.14", "h2 (==4.3.0)", "markdown (>=3.9,<4.0)", - "drf-simple-apikey (==2.2.1)" + "drf-simple-apikey (==2.2.1)", + "matplotlib (>=3.10.6,<4.0.0)", + "reportlab (>=4.4.4,<5.0.0)" ] description = "Prowler's API (Django/DRF)" license = "Apache-2.0" diff --git a/api/src/backend/api/migrations/0048_api_key.py b/api/src/backend/api/migrations/0048_api_key.py index 32b7fe144d..c3142ecda1 100644 --- a/api/src/backend/api/migrations/0048_api_key.py +++ b/api/src/backend/api/migrations/0048_api_key.py @@ -24,7 +24,7 @@ class Migration(migrations.Migration): ( "name", models.CharField( - max_length=255, + max_length=100, validators=[django.core.validators.MinLengthValidator(3)], ), ), diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 984710ccae..ab7cd31186 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -2726,6 +2726,55 @@ class TestScanViewSet: == "There is a problem with credentials." ) + @patch("api.v1.views.ScanViewSet._get_task_status") + @patch("api.v1.views.get_s3_client") + @patch("api.v1.views.env.str") + def test_threatscore_s3_wildcard( + self, + mock_env_str, + mock_get_s3_client, + mock_get_task_status, + authenticated_client, + scans_fixture, + ): + """ + When the threatscore endpoint is called with an S3 output_location, + the view should list objects in S3 using wildcard pattern matching, + retrieve the matching PDF file, and return it with HTTP 200 and proper headers. + """ + scan = scans_fixture[0] + scan.state = StateChoices.COMPLETED + bucket = "test-bucket" + zip_key = "tenant-id/scan-id/prowler-output-foo.zip" + scan.output_location = f"s3://{bucket}/{zip_key}" + scan.save() + + pdf_key = os.path.join( + os.path.dirname(zip_key), + "threatscore", + "prowler-output-123_threatscore_report.pdf", + ) + + mock_s3_client = Mock() + mock_s3_client.list_objects_v2.return_value = {"Contents": [{"Key": pdf_key}]} + mock_s3_client.get_object.return_value = {"Body": io.BytesIO(b"pdf-bytes")} + + mock_env_str.return_value = bucket + mock_get_s3_client.return_value = mock_s3_client + mock_get_task_status.return_value = None + + url = reverse("scan-threatscore", kwargs={"pk": scan.id}) + response = authenticated_client.get(url) + + assert response.status_code == status.HTTP_200_OK + assert response["Content-Type"] == "application/pdf" + assert response["Content-Disposition"].endswith( + '"prowler-output-123_threatscore_report.pdf"' + ) + assert response.content == b"pdf-bytes" + mock_s3_client.list_objects_v2.assert_called_once() + mock_s3_client.get_object.assert_called_once_with(Bucket=bucket, Key=pdf_key) + def test_report_s3_success(self, authenticated_client, scans_fixture, monkeypatch): """ When output_location is an S3 URL and the S3 client returns the file successfully, diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 6e561ff969..ab5331ecd7 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -1,3 +1,4 @@ +import fnmatch import glob import logging import os @@ -1593,6 +1594,25 @@ class ProviderViewSet(BaseRLSViewSet): }, request=None, ), + threatscore=extend_schema( + tags=["Scan"], + summary="Retrieve threatscore report", + description="Download a specific threatscore report (e.g., 'prowler_threatscore_aws') as a PDF file.", + request=None, + responses={ + 200: OpenApiResponse( + description="PDF file containing the threatscore report" + ), + 202: OpenApiResponse(description="The task is in progress"), + 401: OpenApiResponse( + description="API key missing or user not Authenticated" + ), + 403: OpenApiResponse(description="There is a problem with credentials"), + 404: OpenApiResponse( + description="The scan has no threatscore reports, or the threatscore report generation task has not started yet" + ), + }, + ), ) @method_decorator(CACHE_DECORATOR, name="list") @method_decorator(CACHE_DECORATOR, name="retrieve") @@ -1649,6 +1669,9 @@ class ScanViewSet(BaseRLSViewSet): if hasattr(self, "response_serializer_class"): return self.response_serializer_class return ScanComplianceReportSerializer + elif self.action == "threatscore": + if hasattr(self, "response_serializer_class"): + return self.response_serializer_class return super().get_serializer_class() def partial_update(self, request, *args, **kwargs): @@ -1753,7 +1776,18 @@ class ScanViewSet(BaseRLSViewSet): status=status.HTTP_502_BAD_GATEWAY, ) contents = resp.get("Contents", []) - keys = [obj["Key"] for obj in contents if obj["Key"].endswith(suffix)] + keys = [] + for obj in contents: + key = obj["Key"] + key_basename = os.path.basename(key) + if any(ch in suffix for ch in ("*", "?", "[")): + if fnmatch.fnmatch(key_basename, suffix): + keys.append(key) + elif key_basename == suffix: + keys.append(key) + elif key.endswith(suffix): + # Backward compatibility if suffix already includes directories + keys.append(key) if not keys: return Response( { @@ -1880,6 +1914,45 @@ class ScanViewSet(BaseRLSViewSet): content, filename = loader return self._serve_file(content, filename, "text/csv") + @action( + detail=True, + methods=["get"], + url_name="threatscore", + ) + def threatscore(self, request, pk=None): + scan = self.get_object() + running_resp = self._get_task_status(scan) + if running_resp: + return running_resp + + if not scan.output_location: + return Response( + { + "detail": "The scan has no reports, or the threatscore report generation task has not started yet." + }, + status=status.HTTP_404_NOT_FOUND, + ) + + if scan.output_location.startswith("s3://"): + bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "") + key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/") + prefix = os.path.join( + os.path.dirname(key_prefix), + "threatscore", + "*_threatscore_report.pdf", + ) + loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True) + else: + base = os.path.dirname(scan.output_location) + pattern = os.path.join(base, "threatscore", "*_threatscore_report.pdf") + loader = self._load_file(pattern, s3=False) + + if isinstance(loader, Response): + return loader + + content, filename = loader + return self._serve_file(content, filename, "application/pdf") + def create(self, request, *args, **kwargs): input_serializer = self.get_serializer(data=request.data) input_serializer.is_valid(raise_exception=True) diff --git a/api/src/backend/tasks/assets/fonts/FiraCode-Regular.ttf b/api/src/backend/tasks/assets/fonts/FiraCode-Regular.ttf new file mode 100644 index 0000000000..3a57209a97 Binary files /dev/null and b/api/src/backend/tasks/assets/fonts/FiraCode-Regular.ttf differ diff --git a/api/src/backend/tasks/assets/fonts/PlusJakartaSans-Regular.ttf b/api/src/backend/tasks/assets/fonts/PlusJakartaSans-Regular.ttf new file mode 100644 index 0000000000..f43b5f4356 Binary files /dev/null and b/api/src/backend/tasks/assets/fonts/PlusJakartaSans-Regular.ttf differ diff --git a/api/src/backend/tasks/assets/img/prowler_logo.png b/api/src/backend/tasks/assets/img/prowler_logo.png new file mode 100644 index 0000000000..b9bdc4c088 Binary files /dev/null and b/api/src/backend/tasks/assets/img/prowler_logo.png differ diff --git a/api/src/backend/tasks/jobs/export.py b/api/src/backend/tasks/jobs/export.py index 1444c30388..0c9c01f579 100644 --- a/api/src/backend/tasks/jobs/export.py +++ b/api/src/backend/tasks/jobs/export.py @@ -187,18 +187,21 @@ def get_s3_client(): return s3_client -def _upload_to_s3(tenant_id: str, zip_path: str, scan_id: str) -> str | None: +def _upload_to_s3( + tenant_id: str, scan_id: str, local_path: str, relative_key: str +) -> str | None: """ - Upload the specified ZIP file to an S3 bucket. - If the S3 bucket environment variables are not configured, - the function returns None without performing an upload. + Upload a local artifact to an S3 bucket under the tenant/scan prefix. + Args: - tenant_id (str): The tenant identifier, used as part of the S3 key prefix. - zip_path (str): The local file system path to the ZIP file to be uploaded. - scan_id (str): The scan identifier, used as part of the S3 key prefix. + tenant_id (str): The tenant identifier used as the first segment of the S3 key. + scan_id (str): The scan identifier used as the second segment of the S3 key. + local_path (str): Filesystem path to the artifact to upload. + relative_key (str): Object key relative to `//`. + Returns: - str: The S3 URI of the uploaded file (e.g., "s3:///") if successful. - None: If the required environment variables for the S3 bucket are not set. + str | None: S3 URI of the uploaded artifact, or None if the upload is skipped. + Raises: botocore.exceptions.ClientError: If the upload attempt to S3 fails for any reason. """ @@ -206,34 +209,26 @@ def _upload_to_s3(tenant_id: str, zip_path: str, scan_id: str) -> str | None: if not bucket: return + if not relative_key: + return + + if not os.path.isfile(local_path): + return + try: s3 = get_s3_client() - # Upload the ZIP file (outputs) to the S3 bucket - zip_key = f"{tenant_id}/{scan_id}/{os.path.basename(zip_path)}" - s3.upload_file( - Filename=zip_path, - Bucket=bucket, - Key=zip_key, - ) + s3_key = f"{tenant_id}/{scan_id}/{relative_key}" + s3.upload_file(Filename=local_path, Bucket=bucket, Key=s3_key) - # Upload the compliance directory to the S3 bucket - compliance_dir = os.path.join(os.path.dirname(zip_path), "compliance") - for filename in os.listdir(compliance_dir): - local_path = os.path.join(compliance_dir, filename) - if not os.path.isfile(local_path): - continue - file_key = f"{tenant_id}/{scan_id}/compliance/{filename}" - s3.upload_file(Filename=local_path, Bucket=bucket, Key=file_key) - - return f"s3://{base.DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET}/{zip_key}" + return f"s3://{base.DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET}/{s3_key}" except (ClientError, NoCredentialsError, ParamValidationError, ValueError) as e: logger.error(f"S3 upload failed: {str(e)}") def _generate_output_directory( output_directory, prowler_provider: object, tenant_id: str, scan_id: str -) -> tuple[str, str]: +) -> tuple[str, str, str]: """ Generate a file system path for the output directory of a prowler scan. @@ -260,6 +255,7 @@ def _generate_output_directory( >>> _generate_output_directory("/tmp", "aws", "tenant-1234", "scan-5678") '/tmp/tenant-1234/aws/scan-5678/prowler-output-2023-02-15T12:34:56', '/tmp/tenant-1234/aws/scan-5678/compliance/prowler-output-2023-02-15T12:34:56' + '/tmp/tenant-1234/aws/scan-5678/threatscore/prowler-output-2023-02-15T12:34:56' """ # Sanitize the prowler provider name to ensure it is a valid directory name prowler_provider_sanitized = re.sub(r"[^\w\-]", "-", prowler_provider) @@ -280,4 +276,10 @@ def _generate_output_directory( ) os.makedirs("/".join(compliance_path.split("/")[:-1]), exist_ok=True) - return path, compliance_path + threatscore_path = ( + f"{output_directory}/{tenant_id}/{scan_id}/threatscore/prowler-output-" + f"{prowler_provider_sanitized}-{timestamp}" + ) + os.makedirs("/".join(threatscore_path.split("/")[:-1]), exist_ok=True) + + return path, compliance_path, threatscore_path diff --git a/api/src/backend/tasks/jobs/report.py b/api/src/backend/tasks/jobs/report.py new file mode 100644 index 0000000000..641fa5757a --- /dev/null +++ b/api/src/backend/tasks/jobs/report.py @@ -0,0 +1,1337 @@ +import io +import os +from collections import defaultdict +from pathlib import Path +from shutil import rmtree + +import matplotlib.pyplot as plt +from celery.utils.log import get_task_logger +from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY +from django.db.models import Count, Q +from reportlab.lib import colors +from reportlab.lib.enums import TA_CENTER +from reportlab.lib.pagesizes import letter +from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet +from reportlab.lib.units import inch +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont +from reportlab.pdfgen import canvas +from reportlab.platypus import ( + Image, + PageBreak, + Paragraph, + SimpleDocTemplate, + Spacer, + Table, + TableStyle, +) +from tasks.jobs.export import _generate_output_directory, _upload_to_s3 +from tasks.utils import batched + +from api.db_router import READ_REPLICA_ALIAS +from api.db_utils import rls_transaction +from api.models import Finding, Provider, ScanSummary, StatusChoices +from api.utils import initialize_prowler_provider +from prowler.lib.check.compliance_models import Compliance +from prowler.lib.outputs.finding import Finding as FindingOutput + +pdfmetrics.registerFont( + TTFont( + "PlusJakartaSans", + os.path.join( + os.path.dirname(__file__), "../assets/fonts/PlusJakartaSans-Regular.ttf" + ), + ) +) + +pdfmetrics.registerFont( + TTFont( + "FiraCode", + os.path.join(os.path.dirname(__file__), "../assets/fonts/FiraCode-Regular.ttf"), + ) +) + +logger = get_task_logger(__name__) + + +def _create_pdf_styles() -> dict[str, ParagraphStyle]: + """ + Create and return PDF paragraph styles used throughout the report. + + Returns: + dict[str, ParagraphStyle]: A dictionary containing the following styles: + - 'title': Title style with prowler green color + - 'h1': Heading 1 style with blue color and background + - 'h2': Heading 2 style with light blue color + - 'h3': Heading 3 style for sub-headings + - 'normal': Normal text style with left indent + - 'normal_center': Normal text style without indent + """ + styles = getSampleStyleSheet() + prowler_dark_green = colors.Color(0.1, 0.5, 0.2) + + title_style = ParagraphStyle( + "CustomTitle", + parent=styles["Title"], + fontSize=24, + textColor=prowler_dark_green, + spaceAfter=20, + fontName="PlusJakartaSans", + alignment=TA_CENTER, + ) + + h1 = ParagraphStyle( + "CustomH1", + parent=styles["Heading1"], + fontSize=18, + textColor=colors.Color(0.2, 0.4, 0.6), + spaceBefore=20, + spaceAfter=12, + fontName="PlusJakartaSans", + leftIndent=0, + borderWidth=2, + borderColor=colors.Color(0.2, 0.4, 0.6), + borderPadding=8, + backColor=colors.Color(0.95, 0.97, 1.0), + ) + + h2 = ParagraphStyle( + "CustomH2", + parent=styles["Heading2"], + fontSize=14, + textColor=colors.Color(0.3, 0.5, 0.7), + spaceBefore=15, + spaceAfter=8, + fontName="PlusJakartaSans", + leftIndent=10, + borderWidth=1, + borderColor=colors.Color(0.7, 0.8, 0.9), + borderPadding=5, + backColor=colors.Color(0.98, 0.99, 1.0), + ) + + h3 = ParagraphStyle( + "CustomH3", + parent=styles["Heading3"], + fontSize=12, + textColor=colors.Color(0.4, 0.6, 0.8), + spaceBefore=10, + spaceAfter=6, + fontName="PlusJakartaSans", + leftIndent=20, + ) + + normal = ParagraphStyle( + "CustomNormal", + parent=styles["Normal"], + fontSize=10, + textColor=colors.Color(0.2, 0.2, 0.2), + spaceBefore=4, + spaceAfter=4, + leftIndent=30, + fontName="PlusJakartaSans", + ) + + normal_center = ParagraphStyle( + "CustomNormalCenter", + parent=styles["Normal"], + fontSize=10, + textColor=colors.Color(0.2, 0.2, 0.2), + fontName="PlusJakartaSans", + ) + + return { + "title": title_style, + "h1": h1, + "h2": h2, + "h3": h3, + "normal": normal, + "normal_center": normal_center, + } + + +def _create_risk_component(risk_level: int, weight: int, score: int = 0) -> Table: + """ + Create a visual risk component table for the PDF report. + + Args: + risk_level (int): The risk level (0-5), where higher values indicate higher risk. + weight (int): The weight of the risk component. + score (int): The calculated score. Defaults to 0. + + Returns: + Table: A ReportLab Table object with colored cells representing risk, weight, and score. + """ + if risk_level >= 4: + risk_color = colors.Color(0.8, 0.2, 0.2) + elif risk_level >= 3: + risk_color = colors.Color(0.9, 0.6, 0.2) + elif risk_level >= 2: + risk_color = colors.Color(0.9, 0.9, 0.2) + else: + risk_color = colors.Color(0.2, 0.8, 0.2) + + if weight <= 50: + weight_color = colors.Color(0.2, 0.8, 0.2) + elif weight <= 100: + weight_color = colors.Color(0.9, 0.9, 0.2) + else: + weight_color = colors.Color(0.8, 0.2, 0.2) + + score_color = colors.Color(0.4, 0.4, 0.4) + + data = [ + [ + "Risk Level:", + str(risk_level), + "Weight:", + str(weight), + "Score:", + str(score), + ] + ] + + table = Table( + data, + colWidths=[ + 0.8 * inch, + 0.4 * inch, + 0.6 * inch, + 0.4 * inch, + 0.5 * inch, + 0.4 * inch, + ], + ) + + table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (0, 0), colors.Color(0.9, 0.9, 0.9)), + ("BACKGROUND", (1, 0), (1, 0), risk_color), + ("TEXTCOLOR", (1, 0), (1, 0), colors.white), + ("FONTNAME", (1, 0), (1, 0), "FiraCode"), + ("BACKGROUND", (2, 0), (2, 0), colors.Color(0.9, 0.9, 0.9)), + ("BACKGROUND", (3, 0), (3, 0), weight_color), + ("TEXTCOLOR", (3, 0), (3, 0), colors.white), + ("FONTNAME", (3, 0), (3, 0), "FiraCode"), + ("BACKGROUND", (4, 0), (4, 0), colors.Color(0.9, 0.9, 0.9)), + ("BACKGROUND", (5, 0), (5, 0), score_color), + ("TEXTCOLOR", (5, 0), (5, 0), colors.white), + ("FONTNAME", (5, 0), (5, 0), "FiraCode"), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("FONTSIZE", (0, 0), (-1, -1), 10), + ("GRID", (0, 0), (-1, -1), 0.5, colors.black), + ("LEFTPADDING", (0, 0), (-1, -1), 6), + ("RIGHTPADDING", (0, 0), (-1, -1), 6), + ("TOPPADDING", (0, 0), (-1, -1), 8), + ("BOTTOMPADDING", (0, 0), (-1, -1), 8), + ] + ) + ) + + return table + + +def _create_status_component(status: str) -> Table: + """ + Create a visual status component with colored background. + + Args: + status (str): The status value (e.g., "PASS", "FAIL", "MANUAL"). + + Returns: + Table: A ReportLab Table object displaying the status with appropriate color coding. + """ + if status.upper() == "PASS": + status_color = colors.Color(0.2, 0.8, 0.2) + elif status.upper() == "FAIL": + status_color = colors.Color(0.8, 0.2, 0.2) + else: + status_color = colors.Color(0.4, 0.4, 0.4) + + data = [["State:", status.upper()]] + + table = Table(data, colWidths=[0.6 * inch, 0.8 * inch]) + + table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (0, 0), colors.Color(0.9, 0.9, 0.9)), + ("FONTNAME", (0, 0), (0, 0), "PlusJakartaSans"), + ("BACKGROUND", (1, 0), (1, 0), status_color), + ("TEXTCOLOR", (1, 0), (1, 0), colors.white), + ("FONTNAME", (1, 0), (1, 0), "FiraCode"), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("FONTSIZE", (0, 0), (-1, -1), 12), + ("GRID", (0, 0), (-1, -1), 0.5, colors.black), + ("LEFTPADDING", (0, 0), (-1, -1), 8), + ("RIGHTPADDING", (0, 0), (-1, -1), 8), + ("TOPPADDING", (0, 0), (-1, -1), 10), + ("BOTTOMPADDING", (0, 0), (-1, -1), 10), + ] + ) + ) + + return table + + +def _create_section_score_chart( + requirements_list: list[dict], attributes_by_requirement_id: dict +) -> io.BytesIO: + """ + Create a bar chart showing compliance score by section using ThreatScore formula. + + Args: + requirements_list (list[dict]): List of requirement dictionaries with status and findings data. + attributes_by_requirement_id (dict): Mapping of requirement IDs to their attributes including risk level and weight. + + Returns: + io.BytesIO: A BytesIO buffer containing the chart image in PNG format. + """ + # Define expected sections + expected_sections = [ + "1. IAM", + "2. Attack Surface", + "3. Logging and Monitoring", + "4. Encryption", + ] + + # Initialize all expected sections with default values + sections_data = { + section: { + "numerator": 0, + "denominator": 0, + "has_findings": False, + } + for section in expected_sections + } + + # Collect data from requirements + for requirement in requirements_list: + requirement_id = requirement["id"] + requirement_attributes = attributes_by_requirement_id.get(requirement_id, {}) + + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if metadata: + m = metadata[0] + section = getattr(m, "Section", "Unknown") + + # Add section if not in expected list (for flexibility) + if section not in sections_data: + sections_data[section] = { + "numerator": 0, + "denominator": 0, + "has_findings": False, + } + + # Get findings data + passed_findings = requirement["attributes"].get("passed_findings", 0) + total_findings = requirement["attributes"].get("total_findings", 0) + + if total_findings > 0: + sections_data[section]["has_findings"] = True + risk_level = getattr(m, "LevelOfRisk", 0) + weight = getattr(m, "Weight", 0) + + # Calculate using ThreatScore formula from UI + rate_i = passed_findings / total_findings + rfac_i = 1 + 0.25 * risk_level + + sections_data[section]["numerator"] += ( + rate_i * total_findings * weight * rfac_i + ) + sections_data[section]["denominator"] += ( + total_findings * weight * rfac_i + ) + + section_names = [] + compliance_percentages = [] + + for section, data in sections_data.items(): + if data["has_findings"] and data["denominator"] > 0: + compliance_percentage = (data["numerator"] / data["denominator"]) * 100 + else: + compliance_percentage = 100 # No findings = 100% (PASS) + + section_names.append(section) + compliance_percentages.append(compliance_percentage) + + # Sort alphabetically by section name + sorted_data = sorted( + zip(section_names, compliance_percentages), + key=lambda x: x[0], + ) + section_names, compliance_percentages = ( + zip(*sorted_data) if sorted_data else ([], []) + ) + + fig, ax = plt.subplots(figsize=(12, 8)) + + colors_list = [] + for percentage in compliance_percentages: + if percentage >= 80: + color = "#4CAF50" + elif percentage >= 60: + color = "#8BC34A" + elif percentage >= 40: + color = "#FFEB3B" + elif percentage >= 20: + color = "#FF9800" + else: + color = "#F44336" + colors_list.append(color) + + bars = ax.bar(section_names, compliance_percentages, color=colors_list) + + ax.set_ylabel("Compliance Score (%)", fontsize=12) + ax.set_xlabel("Section", fontsize=12) + ax.set_ylim(0, 100) + + for bar, percentage in zip(bars, compliance_percentages): + height = bar.get_height() + ax.text( + bar.get_x() + bar.get_width() / 2.0, + height + 1, + f"{percentage:.1f}%", + ha="center", + va="bottom", + fontweight="bold", + ) + + plt.xticks(rotation=45, ha="right") + + ax.grid(True, alpha=0.3, axis="y") + + plt.tight_layout() + + buffer = io.BytesIO() + plt.savefig(buffer, format="png", dpi=300, bbox_inches="tight") + buffer.seek(0) + plt.close() + + return buffer + + +def _add_pdf_footer(canvas_obj: canvas.Canvas, doc: SimpleDocTemplate) -> None: + """ + Add footer with page number and branding to each page of the PDF. + + Args: + canvas_obj (canvas.Canvas): The ReportLab canvas object for drawing. + doc (SimpleDocTemplate): The document template containing page information. + """ + width, height = doc.pagesize + page_num_text = f"Page {doc.page}" + canvas_obj.setFont("PlusJakartaSans", 9) + canvas_obj.setFillColorRGB(0.4, 0.4, 0.4) + canvas_obj.drawString(30, 20, page_num_text) + powered_text = "Powered by Prowler" + text_width = canvas_obj.stringWidth(powered_text, "PlusJakartaSans", 9) + canvas_obj.drawString(width - text_width - 30, 20, powered_text) + + +def _aggregate_requirement_statistics_from_database( + tenant_id: str, scan_id: str +) -> dict[str, dict[str, int]]: + """ + Aggregate finding statistics by check_id using database aggregation. + + This function uses Django ORM aggregation to calculate pass/fail statistics + entirely in the database, avoiding the need to load findings into memory. + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): The ID of the scan to retrieve findings for. + + Returns: + dict[str, dict[str, int]]: Dictionary mapping check_id to statistics: + - 'passed' (int): Number of passed findings for this check + - 'total' (int): Total number of findings for this check + + Example: + { + 'aws_iam_user_mfa_enabled': {'passed': 10, 'total': 15}, + 'aws_s3_bucket_public_access': {'passed': 0, 'total': 5} + } + """ + requirement_statistics_by_check_id = {} + + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + # Use database aggregation to calculate stats without loading findings into memory + aggregated_statistics_queryset = ( + Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id) + .values("check_id") + .annotate( + total_findings=Count("id"), + passed_findings=Count("id", filter=Q(status=StatusChoices.PASS)), + ) + ) + + for aggregated_stat in aggregated_statistics_queryset: + check_id = aggregated_stat["check_id"] + requirement_statistics_by_check_id[check_id] = { + "passed": aggregated_stat["passed_findings"], + "total": aggregated_stat["total_findings"], + } + + logger.info( + f"Aggregated statistics for {len(requirement_statistics_by_check_id)} unique checks" + ) + return requirement_statistics_by_check_id + + +def _load_findings_for_requirement_checks( + tenant_id: str, scan_id: str, check_ids: list[str], prowler_provider +) -> dict[str, list[FindingOutput]]: + """ + Load findings for specific check IDs on-demand. + + This function loads only the findings needed for a specific set of checks, + minimizing memory usage by avoiding loading all findings at once. This is used + when generating detailed findings tables for specific requirements in the PDF. + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): The ID of the scan to retrieve findings for. + check_ids (list[str]): List of check IDs to load findings for. + prowler_provider: The initialized Prowler provider instance. + + Returns: + dict[str, list[FindingOutput]]: Dictionary mapping check_id to list of FindingOutput objects. + + Example: + { + 'aws_iam_user_mfa_enabled': [FindingOutput(...), FindingOutput(...)], + 'aws_s3_bucket_public_access': [FindingOutput(...)] + } + """ + findings_by_check_id = defaultdict(list) + + if not check_ids: + return dict(findings_by_check_id) + + logger.info(f"Loading findings for {len(check_ids)} checks on-demand") + + findings_queryset = ( + Finding.all_objects.filter( + tenant_id=tenant_id, scan_id=scan_id, check_id__in=check_ids + ) + .order_by("uid") + .iterator() + ) + + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + for batch, is_last_batch in batched( + findings_queryset, DJANGO_FINDINGS_BATCH_SIZE + ): + for finding_model in batch: + finding_output = FindingOutput.transform_api_finding( + finding_model, prowler_provider + ) + findings_by_check_id[finding_output.check_id].append(finding_output) + + total_findings_loaded = sum( + len(findings) for findings in findings_by_check_id.values() + ) + logger.info( + f"Loaded {total_findings_loaded} findings for {len(findings_by_check_id)} checks" + ) + + return dict(findings_by_check_id) + + +def _calculate_requirements_data_from_statistics( + compliance_obj, requirement_statistics_by_check_id: dict[str, dict[str, int]] +) -> tuple[dict[str, dict], list[dict]]: + """ + Calculate requirement status and statistics using pre-aggregated database statistics. + + This function uses O(n) lookups with pre-aggregated statistics from the database, + avoiding the need to iterate over all findings for each requirement. + + Args: + compliance_obj: The compliance framework object containing requirements. + requirement_statistics_by_check_id (dict[str, dict[str, int]]): Pre-aggregated statistics + mapping check_id to {'passed': int, 'total': int} counts. + + Returns: + tuple[dict[str, dict], list[dict]]: A tuple containing: + - attributes_by_requirement_id: Dictionary mapping requirement IDs to their attributes. + - requirements_list: List of requirement dictionaries with status and statistics. + """ + attributes_by_requirement_id = {} + requirements_list = [] + + compliance_framework = getattr(compliance_obj, "Framework", "N/A") + compliance_version = getattr(compliance_obj, "Version", "N/A") + + for requirement in compliance_obj.Requirements: + requirement_id = requirement.Id + requirement_description = getattr(requirement, "Description", "") + requirement_checks = getattr(requirement, "Checks", []) + requirement_attributes = getattr(requirement, "Attributes", []) + + # Store requirement metadata for later use + attributes_by_requirement_id[requirement_id] = { + "attributes": { + "req_attributes": requirement_attributes, + "checks": requirement_checks, + }, + "description": requirement_description, + } + + # Calculate aggregated passed and total findings for this requirement + total_passed_findings = 0 + total_findings_count = 0 + + for check_id in requirement_checks: + if check_id in requirement_statistics_by_check_id: + check_statistics = requirement_statistics_by_check_id[check_id] + total_findings_count += check_statistics["total"] + total_passed_findings += check_statistics["passed"] + + # Determine overall requirement status based on findings + if total_findings_count > 0: + if total_passed_findings == total_findings_count: + requirement_status = StatusChoices.PASS + else: + # Partial pass or complete fail both count as FAIL + requirement_status = StatusChoices.FAIL + else: + # No findings means manual review required + requirement_status = StatusChoices.MANUAL + + requirements_list.append( + { + "id": requirement_id, + "attributes": { + "framework": compliance_framework, + "version": compliance_version, + "status": requirement_status, + "description": requirement_description, + "passed_findings": total_passed_findings, + "total_findings": total_findings_count, + }, + } + ) + + return attributes_by_requirement_id, requirements_list + + +def generate_threatscore_report( + tenant_id: str, + scan_id: str, + compliance_id: str, + output_path: str, + provider_id: str, + only_failed: bool = True, + min_risk_level: int = 4, +) -> None: + """ + Generate a PDF compliance report based on Prowler ThreatScore framework. + + This function creates a comprehensive PDF report containing: + - Compliance overview and metadata + - Section-by-section compliance scores with charts + - Overall ThreatScore calculation + - Critical failed requirements + - Detailed findings for each requirement + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): ID of the scan executed by Prowler. + compliance_id (str): ID of the compliance framework (e.g., "prowler_threatscore_aws"). + output_path (str): Output PDF file path (e.g., "/tmp/threatscore_report.pdf"). + provider_id (str): Provider ID for the scan. + only_failed (bool): If True, only requirements with status "FAIL" will be included + in the detailed requirements section. Defaults to True. + min_risk_level (int): Minimum risk level for critical failed requirements. Defaults to 4. + + Raises: + Exception: If any error occurs during PDF generation, it will be logged and re-raised. + """ + logger.info( + f"Generating the report for the scan {scan_id} with provider {provider_id}" + ) + try: + # Get PDF styles + pdf_styles = _create_pdf_styles() + title_style = pdf_styles["title"] + h1 = pdf_styles["h1"] + h2 = pdf_styles["h2"] + h3 = pdf_styles["h3"] + normal = pdf_styles["normal"] + normal_center = pdf_styles["normal_center"] + + # Get compliance and provider information + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + provider_obj = Provider.objects.get(id=provider_id) + prowler_provider = initialize_prowler_provider(provider_obj) + provider_type = provider_obj.provider + + frameworks_bulk = Compliance.get_bulk(provider_type) + compliance_obj = frameworks_bulk[compliance_id] + compliance_framework = getattr(compliance_obj, "Framework", "N/A") + compliance_version = getattr(compliance_obj, "Version", "N/A") + compliance_name = getattr(compliance_obj, "Name", "N/A") + compliance_description = getattr(compliance_obj, "Description", "") + + # Aggregate requirement statistics from database (memory-efficient) + logger.info(f"Aggregating requirement statistics for scan {scan_id}") + requirement_statistics_by_check_id = ( + _aggregate_requirement_statistics_from_database(tenant_id, scan_id) + ) + + # Calculate requirements data using aggregated statistics + attributes_by_requirement_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + compliance_obj, requirement_statistics_by_check_id + ) + ) + + # Initialize PDF document + doc = SimpleDocTemplate( + output_path, + pagesize=letter, + title=f"Prowler ThreatScore Report - {compliance_framework}", + author="Prowler", + subject=f"Compliance Report for {compliance_framework}", + creator="Prowler Engineering Team", + keywords=f"compliance,{compliance_framework},security,framework,prowler", + ) + + elements = [] + + # Add logo + img_path = os.path.join( + os.path.dirname(__file__), "../assets/img/prowler_logo.png" + ) + logo = Image( + img_path, + width=5 * inch, + height=1 * inch, + ) + elements.append(logo) + + elements.append(Spacer(1, 0.5 * inch)) + elements.append(Paragraph("Prowler ThreatScore Report", title_style)) + elements.append(Spacer(1, 0.5 * inch)) + + # Add compliance information table + info_data = [ + ["Framework:", compliance_framework], + ["ID:", compliance_id], + ["Name:", Paragraph(compliance_name, normal_center)], + ["Version:", compliance_version], + ["Scan ID:", scan_id], + ["Description:", Paragraph(compliance_description, normal_center)], + ] + info_table = Table(info_data, colWidths=[2 * inch, 4 * inch]) + info_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (0, 5), colors.Color(0.2, 0.4, 0.6)), + ("TEXTCOLOR", (0, 0), (0, 5), colors.white), + ("FONTNAME", (0, 0), (0, 5), "FiraCode"), + ("BACKGROUND", (1, 0), (1, 5), colors.Color(0.95, 0.97, 1.0)), + ("TEXTCOLOR", (1, 0), (1, 5), colors.Color(0.2, 0.2, 0.2)), + ("FONTNAME", (1, 0), (1, 5), "PlusJakartaSans"), + ("ALIGN", (0, 0), (-1, -1), "LEFT"), + ("VALIGN", (0, 0), (-1, -1), "TOP"), + ("FONTSIZE", (0, 0), (-1, -1), 11), + ("GRID", (0, 0), (-1, -1), 1, colors.Color(0.7, 0.8, 0.9)), + ("LEFTPADDING", (0, 0), (-1, -1), 10), + ("RIGHTPADDING", (0, 0), (-1, -1), 10), + ("TOPPADDING", (0, 0), (-1, -1), 8), + ("BOTTOMPADDING", (0, 0), (-1, -1), 8), + ] + ) + ) + + elements.append(info_table) + elements.append(PageBreak()) + + # Add compliance score chart + elements.append(Paragraph("Compliance Score by Sections", h1)) + elements.append(Spacer(1, 0.2 * inch)) + + chart_buffer = _create_section_score_chart( + requirements_list, attributes_by_requirement_id + ) + chart_image = Image(chart_buffer, width=7 * inch, height=5.5 * inch) + elements.append(chart_image) + + # Calculate overall ThreatScore using the same formula as the UI + numerator = 0 + denominator = 0 + has_findings = False + + for requirement in requirements_list: + requirement_id = requirement["id"] + requirement_attributes = attributes_by_requirement_id.get( + requirement_id, {} + ) + + # Get findings data + passed_findings = requirement["attributes"].get("passed_findings", 0) + total_findings = requirement["attributes"].get("total_findings", 0) + + # Skip if no findings (avoid division by zero) + if total_findings == 0: + continue + + has_findings = True + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if metadata and len(metadata) > 0: + m = metadata[0] + risk_level = getattr(m, "LevelOfRisk", 0) + weight = getattr(m, "Weight", 0) + + # Calculate using ThreatScore formula from UI + rate_i = passed_findings / total_findings + rfac_i = 1 + 0.25 * risk_level + + numerator += rate_i * total_findings * weight * rfac_i + denominator += total_findings * weight * rfac_i + + # Calculate ThreatScore (percentualScore) + # If no findings exist, consider it 100% (PASS) + if not has_findings: + overall_compliance = 100 + elif denominator > 0: + overall_compliance = (numerator / denominator) * 100 + else: + overall_compliance = 0 + + elements.append(Spacer(1, 0.3 * inch)) + + summary_data = [ + ["ThreatScore:", f"{overall_compliance:.2f}%"], + ] + + if overall_compliance >= 80: + compliance_color = colors.Color(0.2, 0.8, 0.2) + elif overall_compliance >= 60: + compliance_color = colors.Color(0.8, 0.8, 0.2) + else: + compliance_color = colors.Color(0.8, 0.2, 0.2) + + summary_table = Table(summary_data, colWidths=[2.5 * inch, 2 * inch]) + summary_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (0, 0), colors.Color(0.1, 0.3, 0.5)), + ("TEXTCOLOR", (0, 0), (0, 0), colors.white), + ("FONTNAME", (0, 0), (0, 0), "FiraCode"), + ("FONTSIZE", (0, 0), (0, 0), 12), + ("BACKGROUND", (1, 0), (1, 0), compliance_color), + ("TEXTCOLOR", (1, 0), (1, 0), colors.white), + ("FONTNAME", (1, 0), (1, 0), "FiraCode"), + ("FONTSIZE", (1, 0), (1, 0), 16), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("GRID", (0, 0), (-1, -1), 1.5, colors.Color(0.5, 0.6, 0.7)), + ("LEFTPADDING", (0, 0), (-1, -1), 12), + ("RIGHTPADDING", (0, 0), (-1, -1), 12), + ("TOPPADDING", (0, 0), (-1, -1), 10), + ("BOTTOMPADDING", (0, 0), (-1, -1), 10), + ] + ) + ) + + elements.append(summary_table) + elements.append(PageBreak()) + + # Add requirements index + elements.append(Paragraph("Requirements Index", h1)) + + sections = {} + for ( + requirement_id, + requirement_attributes, + ) in attributes_by_requirement_id.items(): + meta = requirement_attributes["attributes"]["req_attributes"][0] + section = getattr(meta, "Section", "N/A") + subsection = getattr(meta, "SubSection", "N/A") + title = getattr(meta, "Title", "N/A") + + if section not in sections: + sections[section] = {} + if subsection not in sections[section]: + sections[section][subsection] = [] + + sections[section][subsection].append({"id": requirement_id, "title": title}) + + section_num = 1 + for section_name, subsections in sections.items(): + elements.append(Paragraph(f"{section_num}. {section_name}", h2)) + + subsection_num = 1 + for subsection_name, requirements in subsections.items(): + elements.append(Paragraph(f"{subsection_name}", h3)) + + req_num = 1 + for req in requirements: + elements.append(Paragraph(f"{req['id']} - {req['title']}", normal)) + req_num += 1 + + subsection_num += 1 + + section_num += 1 + elements.append(Spacer(1, 0.1 * inch)) + + elements.append(PageBreak()) + + # Add critical failed requirements section + elements.append(Paragraph("Top Requirements by Level of Risk", h1)) + elements.append(Spacer(1, 0.1 * inch)) + elements.append( + Paragraph( + f"Critical Failed Requirements (Risk Level ≥ {min_risk_level})", h2 + ) + ) + elements.append(Spacer(1, 0.2 * inch)) + + critical_failed_requirements = [] + for requirement in requirements_list: + requirement_status = requirement["attributes"]["status"] + if requirement_status == StatusChoices.FAIL: + requirement_id = requirement["id"] + metadata = ( + attributes_by_requirement_id.get(requirement_id, {}) + .get("attributes", {}) + .get("req_attributes", [{}])[0] + ) + if metadata: + risk_level = getattr(metadata, "LevelOfRisk", 0) + weight = getattr(metadata, "Weight", 0) + + if risk_level >= min_risk_level: + critical_failed_requirements.append( + { + "requirement": requirement, + "attributes": attributes_by_requirement_id[ + requirement_id + ], + "risk_level": risk_level, + "weight": weight, + "metadata": metadata, + } + ) + + critical_failed_requirements.sort( + key=lambda x: (x["risk_level"], x["weight"]), reverse=True + ) + + if not critical_failed_requirements: + elements.append( + Paragraph( + "✅ No critical failed requirements found. Great job!", normal + ) + ) + else: + elements.append( + Paragraph( + f"Found {len(critical_failed_requirements)} critical failed requirements that require immediate attention:", + normal, + ) + ) + elements.append(Spacer(1, 0.5 * inch)) + + table_data = [["Risk", "Weight", "Requirement ID", "Title", "Section"]] + + for idx, critical_failed_requirement in enumerate( + critical_failed_requirements + ): + requirement_id = critical_failed_requirement["requirement"]["id"] + risk_level = critical_failed_requirement["risk_level"] + weight = critical_failed_requirement["weight"] + title = getattr(critical_failed_requirement["metadata"], "Title", "N/A") + section = getattr( + critical_failed_requirement["metadata"], "Section", "N/A" + ) + + if len(title) > 50: + title = title[:47] + "..." + + table_data.append( + [str(risk_level), str(weight), requirement_id, title, section] + ) + + critical_table = Table( + table_data, + colWidths=[0.7 * inch, 0.9 * inch, 1.3 * inch, 3.1 * inch, 1.5 * inch], + ) + + critical_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (-1, 0), colors.Color(0.8, 0.2, 0.2)), + ("TEXTCOLOR", (0, 0), (-1, 0), colors.white), + ("FONTNAME", (0, 0), (-1, 0), "FiraCode"), + ("FONTSIZE", (0, 0), (-1, 0), 10), + ("BACKGROUND", (0, 1), (0, -1), colors.Color(0.8, 0.2, 0.2)), + ("TEXTCOLOR", (0, 1), (0, -1), colors.white), + ("FONTNAME", (0, 1), (0, -1), "FiraCode"), + ("ALIGN", (0, 1), (0, -1), "CENTER"), + ("FONTSIZE", (0, 1), (0, -1), 12), + ("ALIGN", (1, 1), (1, -1), "CENTER"), + ("FONTNAME", (1, 1), (1, -1), "FiraCode"), + ("FONTNAME", (2, 1), (2, -1), "FiraCode"), + ("FONTSIZE", (2, 1), (2, -1), 9), + ("FONTNAME", (3, 1), (-1, -1), "PlusJakartaSans"), + ("FONTSIZE", (3, 1), (-1, -1), 8), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("GRID", (0, 0), (-1, -1), 1, colors.Color(0.7, 0.7, 0.7)), + ("LEFTPADDING", (0, 0), (-1, -1), 6), + ("RIGHTPADDING", (0, 0), (-1, -1), 6), + ("TOPPADDING", (0, 0), (-1, -1), 8), + ("BOTTOMPADDING", (0, 0), (-1, -1), 8), + ( + "BACKGROUND", + (1, 1), + (-1, -1), + colors.Color(0.98, 0.98, 0.98), + ), + ] + ) + ) + + for idx, critical_failed_requirement in enumerate( + critical_failed_requirements + ): + row_idx = idx + 1 + weight = critical_failed_requirement["weight"] + + if weight >= 150: + weight_color = colors.Color(0.8, 0.2, 0.2) + elif weight >= 100: + weight_color = colors.Color(0.9, 0.6, 0.2) + else: + weight_color = colors.Color(0.9, 0.9, 0.2) + + critical_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (1, row_idx), (1, row_idx), weight_color), + ("TEXTCOLOR", (1, row_idx), (1, row_idx), colors.white), + ] + ) + ) + + elements.append(critical_table) + elements.append(Spacer(1, 0.2 * inch)) + + # Get styles for warning + styles = getSampleStyleSheet() + warning_text = """ + IMMEDIATE ACTION REQUIRED:
+ These requirements have the highest risk levels and have failed compliance checks. + Please prioritize addressing these issues to improve your security posture. + """ + + warning_style = ParagraphStyle( + "Warning", + parent=styles["Normal"], + fontSize=11, + textColor=colors.Color(0.8, 0.2, 0.2), + spaceBefore=10, + spaceAfter=10, + leftIndent=20, + rightIndent=20, + fontName="PlusJakartaSans", + backColor=colors.Color(1.0, 0.95, 0.95), + borderWidth=2, + borderColor=colors.Color(0.8, 0.2, 0.2), + borderPadding=10, + ) + + elements.append(Paragraph(warning_text, warning_style)) + + elements.append(PageBreak()) + + # Add detailed requirements section + def get_weight_for_requirement(requirement_dict): + requirement_id = requirement_dict["id"] + requirement_attributes = attributes_by_requirement_id.get( + requirement_id, {} + ) + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if metadata: + return getattr(metadata[0], "Weight", 0) + return 0 + + sorted_requirements = sorted( + requirements_list, key=get_weight_for_requirement, reverse=True + ) + + if only_failed: + sorted_requirements = [ + requirement + for requirement in sorted_requirements + if requirement["attributes"]["status"] == StatusChoices.FAIL + ] + + # Collect all check IDs for requirements that will be displayed + # This allows us to load only the findings we actually need (memory optimization) + check_ids_to_load = [] + for requirement in sorted_requirements: + requirement_id = requirement["id"] + requirement_attributes = attributes_by_requirement_id.get( + requirement_id, {} + ) + check_ids = requirement_attributes.get("attributes", {}).get("checks", []) + check_ids_to_load.extend(check_ids) + + # Load findings on-demand only for the checks that will be displayed + logger.info( + f"Loading findings on-demand for {len(sorted_requirements)} requirements" + ) + findings_by_check_id = _load_findings_for_requirement_checks( + tenant_id, scan_id, check_ids_to_load, prowler_provider + ) + + for requirement in sorted_requirements: + requirement_id = requirement["id"] + requirement_attributes = attributes_by_requirement_id.get( + requirement_id, {} + ) + requirement_description = requirement["attributes"]["description"] + requirement_status = requirement["attributes"]["status"] + + elements.append( + Paragraph( + f"{requirement_id}: {requirement_attributes.get('description', requirement_description)}", + h1, + ) + ) + + status_component = _create_status_component(requirement_status) + elements.append(status_component) + elements.append(Spacer(1, 0.1 * inch)) + + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if metadata and len(metadata) > 0: + m = metadata[0] + elements.append(Paragraph("Title: ", h3)) + elements.append(Paragraph(f"{getattr(m, 'Title', 'N/A')}", normal)) + elements.append(Paragraph("Section: ", h3)) + elements.append(Paragraph(f"{getattr(m, 'Section', 'N/A')}", normal)) + elements.append(Paragraph("SubSection: ", h3)) + elements.append(Paragraph(f"{getattr(m, 'SubSection', 'N/A')}", normal)) + elements.append(Paragraph("Description: ", h3)) + elements.append( + Paragraph(f"{getattr(m, 'AttributeDescription', 'N/A')}", normal) + ) + elements.append(Paragraph("Additional Information: ", h3)) + elements.append( + Paragraph(f"{getattr(m, 'AdditionalInformation', 'N/A')}", normal) + ) + elements.append(Spacer(1, 0.1 * inch)) + + risk_level = getattr(m, "LevelOfRisk", 0) + weight = getattr(m, "Weight", 0) + + if requirement_status == StatusChoices.PASS: + score = risk_level * weight + else: + score = 0 + + risk_component = _create_risk_component(risk_level, weight, score) + elements.append(risk_component) + elements.append(Spacer(1, 0.1 * inch)) + + # Get findings for this requirement's checks (loaded on-demand earlier) + requirement_check_ids = requirement_attributes.get("attributes", {}).get( + "checks", [] + ) + for check_id in requirement_check_ids: + elements.append(Paragraph(f"Check: {check_id}", h2)) + elements.append(Spacer(1, 0.1 * inch)) + + # Get findings for this check (already loaded on-demand) + check_findings = findings_by_check_id.get(check_id, []) + + if not check_findings: + elements.append( + Paragraph("- No information for this finding currently", normal) + ) + else: + findings_table_data = [ + [ + "Finding", + "Resource name", + "Severity", + "Status", + "Region", + ] + ] + for finding_output in check_findings: + check_metadata = getattr(finding_output, "metadata", {}) + finding_title = getattr( + check_metadata, + "CheckTitle", + getattr(finding_output, "check_id", ""), + ) + resource_name = getattr(finding_output, "resource_name", "") + if not resource_name: + resource_name = getattr(finding_output, "resource_uid", "") + severity = getattr(check_metadata, "Severity", "").capitalize() + finding_status = getattr(finding_output, "status", "").upper() + region = getattr(finding_output, "region", "global") + + findings_table_data.append( + [ + Paragraph(finding_title, normal_center), + Paragraph(resource_name, normal_center), + Paragraph(severity, normal_center), + Paragraph(finding_status, normal_center), + Paragraph(region, normal_center), + ] + ) + findings_table = Table( + findings_table_data, + colWidths=[ + 2.5 * inch, + 3 * inch, + 0.9 * inch, + 0.9 * inch, + 0.9 * inch, + ], + ) + findings_table.setStyle( + TableStyle( + [ + ( + "BACKGROUND", + (0, 0), + (-1, 0), + colors.Color(0.2, 0.4, 0.6), + ), + ("TEXTCOLOR", (0, 0), (-1, 0), colors.white), + ("FONTNAME", (0, 0), (-1, 0), "FiraCode"), + ("ALIGN", (0, 0), (0, 0), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("FONTSIZE", (0, 0), (-1, -1), 9), + ( + "GRID", + (0, 0), + (-1, -1), + 0.1, + colors.Color(0.7, 0.8, 0.9), + ), + ("LEFTPADDING", (0, 0), (0, 0), 0), + ("RIGHTPADDING", (0, 0), (0, 0), 0), + ("TOPPADDING", (0, 0), (-1, -1), 4), + ("BOTTOMPADDING", (0, 0), (-1, -1), 4), + ] + ) + ) + elements.append(findings_table) + elements.append(Spacer(1, 0.1 * inch)) + + elements.append(PageBreak()) + + # Build the PDF + doc.build(elements, onFirstPage=_add_pdf_footer, onLaterPages=_add_pdf_footer) + except Exception as e: + logger.info( + f"Error building the document, line {e.__traceback__.tb_lineno} -- {e}" + ) + raise e + + +def generate_threatscore_report_job( + tenant_id: str, scan_id: str, provider_id: str +) -> dict[str, bool | str]: + """ + Job function to generate a threatscore report and upload it to S3. + + This function orchestrates the complete report generation workflow: + 1. Validates that the scan has findings + 2. Checks provider type compatibility + 3. Generates the output directory + 4. Calls generate_threatscore_report to create the PDF + 5. Uploads the PDF to S3 + 6. Cleans up temporary files + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): The ID of the scan to generate a report for. + provider_id (str): The ID of the provider used in the scan. + + Returns: + dict[str, bool | str]: A dictionary containing: + - 'upload' (bool): True if the report was successfully uploaded to S3, False otherwise. + - 'error' (str): Error message if an exception occurred (only present on error). + """ + # Check if the scan has findings and get provider info + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + if not ScanSummary.objects.filter(scan_id=scan_id).exists(): + logger.info(f"No findings found for scan {scan_id}") + return {"upload": False} + + provider_obj = Provider.objects.get(id=provider_id) + provider_uid = provider_obj.uid + provider_type = provider_obj.provider + + if provider_type not in ["aws", "azure", "gcp", "m365"]: + logger.info( + f"Provider {provider_id} is not supported for threatscore report" + ) + return {"upload": False} + + # This compliance is hardcoded because is the only one that is available for the threatscore report + compliance_id = f"prowler_threatscore_{provider_type}" + logger.info( + f"Generating threatscore report for scan {scan_id} with compliance {compliance_id} inside the job" + ) + try: + logger.info("Generating the output directory") + out_dir, _, threatscore_path = _generate_output_directory( + DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id + ) + except Exception as e: + logger.error(f"Error generating output directory: {e}") + return {"error": str(e)} + + pdf_path = f"{threatscore_path}_threatscore_report.pdf" + logger.info(f"The path for the threatscore report is {pdf_path}") + generate_threatscore_report( + tenant_id=tenant_id, + scan_id=scan_id, + compliance_id=compliance_id, + output_path=pdf_path, + provider_id=provider_id, + only_failed=True, + min_risk_level=4, + ) + + upload_uri = _upload_to_s3( + tenant_id, + scan_id, + pdf_path, + f"threatscore/{Path(pdf_path).name}", + ) + if upload_uri: + try: + rmtree(Path(pdf_path).parent, ignore_errors=True) + except Exception as e: + logger.error(f"Error deleting output files: {e}") + final_location, did_upload = upload_uri, True + else: + final_location, did_upload = out_dir, False + + logger.info(f"Threatscore report outputs at {final_location}") + + return {"upload": did_upload} diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index fd4874eb6c..5c2edbd339 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -1,3 +1,4 @@ +import os from datetime import datetime, timedelta, timezone from pathlib import Path from shutil import rmtree @@ -26,6 +27,7 @@ from tasks.jobs.integrations import ( upload_s3_integration, upload_security_hub_integration, ) +from tasks.jobs.report import generate_threatscore_report_job from tasks.jobs.scan import ( aggregate_findings, create_compliance_requirements, @@ -64,10 +66,15 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str) generate_outputs_task.si( scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id ), - check_integrations_task.si( - tenant_id=tenant_id, - provider_id=provider_id, - scan_id=scan_id, + group( + generate_threatscore_report_task.si( + tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id + ), + check_integrations_task.si( + tenant_id=tenant_id, + provider_id=provider_id, + scan_id=scan_id, + ), ), ).apply_async() @@ -304,7 +311,7 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str): frameworks_bulk = Compliance.get_bulk(provider_type) frameworks_avail = get_compliance_frameworks(provider_type) - out_dir, comp_dir = _generate_output_directory( + out_dir, comp_dir, _ = _generate_output_directory( DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id ) @@ -407,7 +414,24 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str): writer._data.clear() compressed = _compress_output_files(out_dir) - upload_uri = _upload_to_s3(tenant_id, compressed, scan_id) + + upload_uri = _upload_to_s3( + tenant_id, + scan_id, + compressed, + os.path.basename(compressed), + ) + + compliance_dir_path = Path(comp_dir).parent + if compliance_dir_path.exists(): + for artifact_path in sorted(compliance_dir_path.iterdir()): + if artifact_path.is_file(): + _upload_to_s3( + tenant_id, + scan_id, + str(artifact_path), + f"compliance/{artifact_path.name}", + ) # S3 integrations (need output_directory) with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): @@ -617,3 +641,21 @@ def jira_integration_task( return send_findings_to_jira( tenant_id, integration_id, project_key, issue_type, finding_ids ) + + +@shared_task( + base=RLSTask, + name="scan-threatscore-report", + queue="scan-reports", +) +def generate_threatscore_report_task(tenant_id: str, scan_id: str, provider_id: str): + """ + Task to generate a threatscore report for a given scan. + Args: + tenant_id (str): The tenant identifier. + scan_id (str): The scan identifier. + provider_id (str): The provider identifier. + """ + return generate_threatscore_report_job( + tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id + ) diff --git a/api/src/backend/tasks/tests/test_export.py b/api/src/backend/tasks/tests/test_export.py index c10f20774b..f1e7120989 100644 --- a/api/src/backend/tasks/tests/test_export.py +++ b/api/src/backend/tasks/tests/test_export.py @@ -72,17 +72,26 @@ class TestOutputs: client_mock = MagicMock() mock_get_client.return_value = client_mock - result = _upload_to_s3("tenant-id", str(zip_path), "scan-id") + result = _upload_to_s3( + "tenant-id", + "scan-id", + str(zip_path), + "outputs.zip", + ) expected_uri = "s3://test-bucket/tenant-id/scan-id/outputs.zip" assert result == expected_uri - assert client_mock.upload_file.call_count == 2 + client_mock.upload_file.assert_called_once_with( + Filename=str(zip_path), + Bucket="test-bucket", + Key="tenant-id/scan-id/outputs.zip", + ) @patch("tasks.jobs.export.get_s3_client") @patch("tasks.jobs.export.base") def test_upload_to_s3_missing_bucket(self, mock_base, mock_get_client): mock_base.DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET = "" - result = _upload_to_s3("tenant", "/tmp/fake.zip", "scan") + result = _upload_to_s3("tenant", "scan", "/tmp/fake.zip", "fake.zip") assert result is None @patch("tasks.jobs.export.get_s3_client") @@ -101,11 +110,15 @@ class TestOutputs: client_mock = MagicMock() mock_get_client.return_value = client_mock - result = _upload_to_s3("tenant", str(zip_path), "scan") + result = _upload_to_s3( + "tenant", + "scan", + str(compliance_dir / "subdir"), + "compliance/subdir", + ) - expected_uri = "s3://test-bucket/tenant/scan/results.zip" - assert result == expected_uri - client_mock.upload_file.assert_called_once() + assert result is None + client_mock.upload_file.assert_not_called() @patch( "tasks.jobs.export.get_s3_client", @@ -126,7 +139,12 @@ class TestOutputs: compliance_dir.mkdir() (compliance_dir / "report.csv").write_text("csv") - _upload_to_s3("tenant", str(zip_path), "scan") + _upload_to_s3( + "tenant", + "scan", + str(zip_path), + "zipfile.zip", + ) mock_logger.assert_called() @patch("tasks.jobs.export.rls_transaction") @@ -150,15 +168,17 @@ class TestOutputs: provider = "aws" expected_timestamp = "20230615103045" - path, compliance = _generate_output_directory( + path, compliance, threatscore = _generate_output_directory( base_dir, provider, tenant_id, scan_id ) assert os.path.isdir(os.path.dirname(path)) assert os.path.isdir(os.path.dirname(compliance)) + assert os.path.isdir(os.path.dirname(threatscore)) assert path.endswith(f"{provider}-{expected_timestamp}") assert compliance.endswith(f"{provider}-{expected_timestamp}") + assert threatscore.endswith(f"{provider}-{expected_timestamp}") @patch("tasks.jobs.export.rls_transaction") @patch("tasks.jobs.export.Scan") @@ -181,12 +201,14 @@ class TestOutputs: provider = "aws/test@check" expected_timestamp = "20230615103045" - path, compliance = _generate_output_directory( + path, compliance, threatscore = _generate_output_directory( base_dir, provider, tenant_id, scan_id ) assert os.path.isdir(os.path.dirname(path)) assert os.path.isdir(os.path.dirname(compliance)) + assert os.path.isdir(os.path.dirname(threatscore)) assert path.endswith(f"aws-test-check-{expected_timestamp}") assert compliance.endswith(f"aws-test-check-{expected_timestamp}") + assert threatscore.endswith(f"aws-test-check-{expected_timestamp}") diff --git a/api/src/backend/tasks/tests/test_report.py b/api/src/backend/tasks/tests/test_report.py new file mode 100644 index 0000000000..a472067b2d --- /dev/null +++ b/api/src/backend/tasks/tests/test_report.py @@ -0,0 +1,963 @@ +import uuid +from pathlib import Path +from unittest.mock import MagicMock, patch + +import matplotlib +import pytest +from tasks.jobs.report import ( + _aggregate_requirement_statistics_from_database, + _calculate_requirements_data_from_statistics, + _load_findings_for_requirement_checks, + generate_threatscore_report, + generate_threatscore_report_job, +) +from tasks.tasks import generate_threatscore_report_task + +from api.models import Finding, StatusChoices +from prowler.lib.check.models import Severity + +matplotlib.use("Agg") # Use non-interactive backend for tests + + +@pytest.mark.django_db +class TestGenerateThreatscoreReport: + def setup_method(self): + self.scan_id = str(uuid.uuid4()) + self.provider_id = str(uuid.uuid4()) + self.tenant_id = str(uuid.uuid4()) + + def test_no_findings_returns_early(self): + with patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter: + mock_filter.return_value.exists.return_value = False + + result = generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + assert result == {"upload": False} + mock_filter.assert_called_once_with(scan_id=self.scan_id) + + @patch("tasks.jobs.report.rmtree") + @patch("tasks.jobs.report._upload_to_s3") + @patch("tasks.jobs.report.generate_threatscore_report") + @patch("tasks.jobs.report._generate_output_directory") + @patch("tasks.jobs.report.Provider.objects.get") + @patch("tasks.jobs.report.ScanSummary.objects.filter") + def test_generate_threatscore_report_happy_path( + self, + mock_scan_summary_filter, + mock_provider_get, + mock_generate_output_directory, + mock_generate_report, + mock_upload, + mock_rmtree, + ): + mock_scan_summary_filter.return_value.exists.return_value = True + + mock_provider = MagicMock() + mock_provider.uid = "provider-uid" + mock_provider.provider = "aws" + mock_provider_get.return_value = mock_provider + + mock_generate_output_directory.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + mock_upload.return_value = "s3://bucket/threatscore_report.pdf" + + result = generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + assert result == {"upload": True} + mock_generate_report.assert_called_once_with( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + compliance_id="prowler_threatscore_aws", + output_path="/tmp/threatscore_path_threatscore_report.pdf", + provider_id=self.provider_id, + only_failed=True, + min_risk_level=4, + ) + mock_upload.assert_called_once_with( + self.tenant_id, + self.scan_id, + "/tmp/threatscore_path_threatscore_report.pdf", + "threatscore/threatscore_path_threatscore_report.pdf", + ) + mock_rmtree.assert_called_once_with( + Path("/tmp/threatscore_path_threatscore_report.pdf").parent, + ignore_errors=True, + ) + + def test_generate_threatscore_report_fails_upload(self): + with ( + patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter, + patch("tasks.jobs.report.Provider.objects.get") as mock_provider_get, + patch("tasks.jobs.report._generate_output_directory") as mock_gen_dir, + patch("tasks.jobs.report.generate_threatscore_report"), + patch("tasks.jobs.report._upload_to_s3", return_value=None), + ): + mock_filter.return_value.exists.return_value = True + + # Mock provider + mock_provider = MagicMock() + mock_provider.uid = "aws-provider-uid" + mock_provider.provider = "aws" + mock_provider_get.return_value = mock_provider + + mock_gen_dir.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + result = generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + assert result == {"upload": False} + + def test_generate_threatscore_report_logs_rmtree_exception(self, caplog): + with ( + patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter, + patch("tasks.jobs.report.Provider.objects.get") as mock_provider_get, + patch("tasks.jobs.report._generate_output_directory") as mock_gen_dir, + patch("tasks.jobs.report.generate_threatscore_report"), + patch( + "tasks.jobs.report._upload_to_s3", return_value="s3://bucket/report.pdf" + ), + patch( + "tasks.jobs.report.rmtree", side_effect=Exception("Test deletion error") + ), + ): + mock_filter.return_value.exists.return_value = True + + # Mock provider + mock_provider = MagicMock() + mock_provider.uid = "aws-provider-uid" + mock_provider.provider = "aws" + mock_provider_get.return_value = mock_provider + + mock_gen_dir.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + with caplog.at_level("ERROR"): + generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + assert "Error deleting output files" in caplog.text + + def test_generate_threatscore_report_azure_provider(self): + with ( + patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter, + patch("tasks.jobs.report.Provider.objects.get") as mock_provider_get, + patch("tasks.jobs.report._generate_output_directory") as mock_gen_dir, + patch("tasks.jobs.report.generate_threatscore_report") as mock_generate, + patch( + "tasks.jobs.report._upload_to_s3", return_value="s3://bucket/report.pdf" + ), + patch("tasks.jobs.report.rmtree"), + ): + mock_filter.return_value.exists.return_value = True + + mock_provider = MagicMock() + mock_provider.uid = "azure-provider-uid" + mock_provider.provider = "azure" + mock_provider_get.return_value = mock_provider + + mock_gen_dir.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + mock_generate.assert_called_once_with( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + compliance_id="prowler_threatscore_azure", + output_path="/tmp/threatscore_path_threatscore_report.pdf", + provider_id=self.provider_id, + only_failed=True, + min_risk_level=4, + ) + + +@pytest.mark.django_db +class TestAggregateRequirementStatistics: + """Test suite for _aggregate_requirement_statistics_from_database function.""" + + def test_aggregates_findings_correctly(self, tenants_fixture, scans_fixture): + """Verify correct pass/total counts per check are aggregated from database.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + # Create findings with different check_ids and statuses + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-1", + check_id="check_1", + status=StatusChoices.PASS, + severity=Severity.high, + impact=Severity.high, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-2", + check_id="check_1", + status=StatusChoices.FAIL, + severity=Severity.high, + impact=Severity.high, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-3", + check_id="check_2", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + assert result == { + "check_1": {"passed": 1, "total": 2}, + "check_2": {"passed": 1, "total": 1}, + } + + def test_handles_empty_scan(self, tenants_fixture, scans_fixture): + """Return empty dict when no findings exist for the scan.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + assert result == {} + + def test_multiple_findings_same_check(self, tenants_fixture, scans_fixture): + """Aggregate multiple findings for same check_id correctly.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + # Create 5 findings for same check, 3 passed + for i in range(3): + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid=f"finding-pass-{i}", + check_id="check_same", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + for i in range(2): + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid=f"finding-fail-{i}", + check_id="check_same", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + assert result == {"check_same": {"passed": 3, "total": 5}} + + def test_only_failed_findings(self, tenants_fixture, scans_fixture): + """Correctly count when all findings are FAIL status.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-fail-1", + check_id="check_fail", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-fail-2", + check_id="check_fail", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + assert result == {"check_fail": {"passed": 0, "total": 2}} + + def test_mixed_statuses(self, tenants_fixture, scans_fixture): + """Test with PASS, FAIL, and MANUAL statuses mixed.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-pass", + check_id="check_mixed", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-fail", + check_id="check_mixed", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-manual", + check_id="check_mixed", + status=StatusChoices.MANUAL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + # Only PASS status is counted as passed + assert result == {"check_mixed": {"passed": 1, "total": 3}} + + +@pytest.mark.django_db +class TestLoadFindingsForChecks: + """Test suite for _load_findings_for_requirement_checks function.""" + + def test_loads_only_requested_checks( + self, tenants_fixture, scans_fixture, providers_fixture + ): + """Verify only findings for specified check_ids are loaded.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + providers_fixture[0] + + # Create findings with different check_ids + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-1", + check_id="check_requested", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-2", + check_id="check_not_requested", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + mock_provider = MagicMock() + + with patch( + "tasks.jobs.report.FindingOutput.transform_api_finding" + ) as mock_transform: + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_requested" + mock_transform.return_value = mock_finding_output + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), ["check_requested"], mock_provider + ) + + # Only one finding should be loaded + assert "check_requested" in result + assert "check_not_requested" not in result + assert len(result["check_requested"]) == 1 + assert mock_transform.call_count == 1 + + def test_empty_check_ids_returns_empty( + self, tenants_fixture, scans_fixture, providers_fixture + ): + """Return empty dict when check_ids list is empty.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + mock_provider = MagicMock() + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), [], mock_provider + ) + + assert result == {} + + def test_groups_by_check_id( + self, tenants_fixture, scans_fixture, providers_fixture + ): + """Multiple findings for same check are grouped correctly.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + # Create multiple findings for same check + for i in range(3): + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid=f"finding-{i}", + check_id="check_group", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + mock_provider = MagicMock() + + with patch( + "tasks.jobs.report.FindingOutput.transform_api_finding" + ) as mock_transform: + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_group" + mock_transform.return_value = mock_finding_output + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), ["check_group"], mock_provider + ) + + assert len(result["check_group"]) == 3 + + def test_transforms_to_finding_output( + self, tenants_fixture, scans_fixture, providers_fixture + ): + """Findings are transformed using FindingOutput.transform_api_finding.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-transform", + check_id="check_transform", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + mock_provider = MagicMock() + + with patch( + "tasks.jobs.report.FindingOutput.transform_api_finding" + ) as mock_transform: + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_transform" + mock_transform.return_value = mock_finding_output + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), ["check_transform"], mock_provider + ) + + # Verify transform was called + mock_transform.assert_called_once() + # Verify the transformed output is in the result + assert result["check_transform"][0] == mock_finding_output + + def test_batched_iteration(self, tenants_fixture, scans_fixture, providers_fixture): + """Works correctly with multiple batches of findings.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + # Create enough findings to ensure batching (assuming batch size > 1) + for i in range(10): + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid=f"finding-batch-{i}", + check_id="check_batch", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + mock_provider = MagicMock() + + with patch( + "tasks.jobs.report.FindingOutput.transform_api_finding" + ) as mock_transform: + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_batch" + mock_transform.return_value = mock_finding_output + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), ["check_batch"], mock_provider + ) + + # All 10 findings should be loaded regardless of batching + assert len(result["check_batch"]) == 10 + assert mock_transform.call_count == 10 + + +@pytest.mark.django_db +class TestCalculateRequirementsData: + """Test suite for _calculate_requirements_data_from_statistics function.""" + + def test_requirement_status_all_pass(self): + """Status is PASS when all findings for requirement checks pass.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_requirement = MagicMock() + mock_requirement.Id = "req_1" + mock_requirement.Description = "Test requirement" + mock_requirement.Checks = ["check_1", "check_2"] + mock_requirement.Attributes = [MagicMock()] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = { + "check_1": {"passed": 5, "total": 5}, + "check_2": {"passed": 3, "total": 3}, + } + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + assert len(requirements_list) == 1 + assert requirements_list[0]["attributes"]["status"] == StatusChoices.PASS + assert requirements_list[0]["attributes"]["passed_findings"] == 8 + assert requirements_list[0]["attributes"]["total_findings"] == 8 + + def test_requirement_status_some_fail(self): + """Status is FAIL when some findings fail.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_requirement = MagicMock() + mock_requirement.Id = "req_2" + mock_requirement.Description = "Test requirement with failures" + mock_requirement.Checks = ["check_3"] + mock_requirement.Attributes = [MagicMock()] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = { + "check_3": {"passed": 2, "total": 5}, + } + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + assert len(requirements_list) == 1 + assert requirements_list[0]["attributes"]["status"] == StatusChoices.FAIL + assert requirements_list[0]["attributes"]["passed_findings"] == 2 + assert requirements_list[0]["attributes"]["total_findings"] == 5 + + def test_requirement_status_no_findings(self): + """Status is MANUAL when no findings exist for requirement.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_requirement = MagicMock() + mock_requirement.Id = "req_3" + mock_requirement.Description = "Manual requirement" + mock_requirement.Checks = ["check_nonexistent"] + mock_requirement.Attributes = [MagicMock()] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = {} + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + assert len(requirements_list) == 1 + assert requirements_list[0]["attributes"]["status"] == StatusChoices.MANUAL + assert requirements_list[0]["attributes"]["passed_findings"] == 0 + assert requirements_list[0]["attributes"]["total_findings"] == 0 + + def test_aggregates_multiple_checks(self): + """Correctly sum stats across multiple checks in requirement.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_requirement = MagicMock() + mock_requirement.Id = "req_4" + mock_requirement.Description = "Multi-check requirement" + mock_requirement.Checks = ["check_a", "check_b", "check_c"] + mock_requirement.Attributes = [MagicMock()] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = { + "check_a": {"passed": 10, "total": 15}, + "check_b": {"passed": 5, "total": 10}, + "check_c": {"passed": 0, "total": 5}, + } + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + assert len(requirements_list) == 1 + # 10 + 5 + 0 = 15 passed + assert requirements_list[0]["attributes"]["passed_findings"] == 15 + # 15 + 10 + 5 = 30 total + assert requirements_list[0]["attributes"]["total_findings"] == 30 + # Not all passed, so should be FAIL + assert requirements_list[0]["attributes"]["status"] == StatusChoices.FAIL + + def test_returns_correct_structure(self): + """Verify tuple structure and dict keys are correct.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_attribute = MagicMock() + mock_requirement = MagicMock() + mock_requirement.Id = "req_5" + mock_requirement.Description = "Structure test" + mock_requirement.Checks = ["check_struct"] + mock_requirement.Attributes = [mock_attribute] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = {"check_struct": {"passed": 1, "total": 1}} + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + # Verify attributes_by_id structure + assert "req_5" in attributes_by_id + assert "attributes" in attributes_by_id["req_5"] + assert "description" in attributes_by_id["req_5"] + assert "req_attributes" in attributes_by_id["req_5"]["attributes"] + assert "checks" in attributes_by_id["req_5"]["attributes"] + + # Verify requirements_list structure + assert len(requirements_list) == 1 + req = requirements_list[0] + assert "id" in req + assert "attributes" in req + assert "framework" in req["attributes"] + assert "version" in req["attributes"] + assert "status" in req["attributes"] + assert "description" in req["attributes"] + assert "passed_findings" in req["attributes"] + assert "total_findings" in req["attributes"] + + +@pytest.mark.django_db +class TestGenerateThreatscoreReportFunction: + def setup_method(self): + self.scan_id = str(uuid.uuid4()) + self.provider_id = str(uuid.uuid4()) + self.tenant_id = str(uuid.uuid4()) + self.compliance_id = "prowler_threatscore_aws" + self.output_path = "/tmp/test_threatscore_report.pdf" + + @patch("tasks.jobs.report.initialize_prowler_provider") + @patch("tasks.jobs.report.Provider.objects.get") + @patch("tasks.jobs.report.Compliance.get_bulk") + @patch("tasks.jobs.report._aggregate_requirement_statistics_from_database") + @patch("tasks.jobs.report._calculate_requirements_data_from_statistics") + @patch("tasks.jobs.report._load_findings_for_requirement_checks") + @patch("tasks.jobs.report.SimpleDocTemplate") + @patch("tasks.jobs.report.Image") + @patch("tasks.jobs.report.Spacer") + @patch("tasks.jobs.report.Paragraph") + @patch("tasks.jobs.report.PageBreak") + @patch("tasks.jobs.report.Table") + @patch("tasks.jobs.report.TableStyle") + @patch("tasks.jobs.report.plt.subplots") + @patch("tasks.jobs.report.plt.savefig") + @patch("tasks.jobs.report.io.BytesIO") + def test_generate_threatscore_report_success( + self, + mock_bytesio, + mock_savefig, + mock_subplots, + mock_table_style, + mock_table, + mock_page_break, + mock_paragraph, + mock_spacer, + mock_image, + mock_doc_template, + mock_load_findings, + mock_calculate_requirements, + mock_aggregate_statistics, + mock_compliance_get_bulk, + mock_provider_get, + mock_initialize_provider, + ): + """Test the updated generate_threatscore_report using new memory-efficient architecture.""" + mock_provider = MagicMock() + mock_provider.provider = "aws" + mock_provider_get.return_value = mock_provider + + prowler_provider = MagicMock() + mock_initialize_provider.return_value = prowler_provider + + # Mock compliance object with requirements + mock_compliance_obj = MagicMock() + mock_compliance_obj.Framework = "ProwlerThreatScore" + mock_compliance_obj.Version = "1.0" + mock_compliance_obj.Description = "Test Description" + + # Configure requirement with properly set numeric attributes for chart generation + mock_requirement = MagicMock() + mock_requirement.Id = "req_1" + mock_requirement.Description = "Test requirement" + mock_requirement.Checks = ["check_1"] + + # Create a properly configured attribute mock with numeric values + mock_requirement_attr = MagicMock() + mock_requirement_attr.Section = "1. IAM" + mock_requirement_attr.SubSection = "1.1 Identity" + mock_requirement_attr.Title = "Test Requirement Title" + mock_requirement_attr.LevelOfRisk = 3 + mock_requirement_attr.Weight = 100 + mock_requirement_attr.AttributeDescription = "Test requirement description" + mock_requirement_attr.AdditionalInformation = "Additional test information" + + mock_requirement.Attributes = [mock_requirement_attr] + mock_compliance_obj.Requirements = [mock_requirement] + + mock_compliance_get_bulk.return_value = { + self.compliance_id: mock_compliance_obj + } + + # Mock the aggregated statistics from database + mock_aggregate_statistics.return_value = {"check_1": {"passed": 5, "total": 10}} + + # Mock the calculated requirements data with properly configured attributes + mock_attributes_by_id = { + "req_1": { + "attributes": { + "req_attributes": [mock_requirement_attr], + "checks": ["check_1"], + }, + "description": "Test requirement", + } + } + mock_requirements_list = [ + { + "id": "req_1", + "attributes": { + "framework": "ProwlerThreatScore", + "version": "1.0", + "status": StatusChoices.FAIL, + "description": "Test requirement", + "passed_findings": 5, + "total_findings": 10, + }, + } + ] + mock_calculate_requirements.return_value = ( + mock_attributes_by_id, + mock_requirements_list, + ) + + # Mock the on-demand loaded findings + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_1" + mock_finding_output.status = "FAIL" + mock_finding_output.metadata = MagicMock() + mock_finding_output.metadata.CheckTitle = "Test Check" + mock_finding_output.metadata.Severity = "HIGH" + mock_finding_output.resource_name = "test-resource" + mock_finding_output.region = "us-east-1" + + mock_load_findings.return_value = {"check_1": [mock_finding_output]} + + # Mock PDF generation components + mock_doc = MagicMock() + mock_doc_template.return_value = mock_doc + + mock_fig, mock_ax = MagicMock(), MagicMock() + mock_subplots.return_value = (mock_fig, mock_ax) + mock_buffer = MagicMock() + mock_bytesio.return_value = mock_buffer + + mock_image.return_value = MagicMock() + mock_spacer.return_value = MagicMock() + mock_paragraph.return_value = MagicMock() + mock_page_break.return_value = MagicMock() + mock_table.return_value = MagicMock() + mock_table_style.return_value = MagicMock() + + # Execute the function + generate_threatscore_report( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + compliance_id=self.compliance_id, + output_path=self.output_path, + provider_id=self.provider_id, + only_failed=True, + min_risk_level=4, + ) + + # Verify the new workflow was followed + mock_provider_get.assert_called_once_with(id=self.provider_id) + mock_initialize_provider.assert_called_once_with(mock_provider) + mock_compliance_get_bulk.assert_called_once_with("aws") + + # Verify the new functions were called in correct order with correct parameters + mock_aggregate_statistics.assert_called_once_with(self.tenant_id, self.scan_id) + mock_calculate_requirements.assert_called_once_with( + mock_compliance_obj, {"check_1": {"passed": 5, "total": 10}} + ) + mock_load_findings.assert_called_once_with( + self.tenant_id, self.scan_id, ["check_1"], prowler_provider + ) + + # Verify PDF was built + mock_doc_template.assert_called_once() + mock_doc.build.assert_called_once() + + @patch("tasks.jobs.report.initialize_prowler_provider") + @patch("tasks.jobs.report.Provider.objects.get") + @patch("tasks.jobs.report.Compliance.get_bulk") + @patch("tasks.jobs.report.Finding.all_objects.filter") + def test_generate_threatscore_report_exception_handling( + self, + mock_finding_filter, + mock_compliance_get_bulk, + mock_provider_get, + mock_initialize_provider, + ): + mock_provider_get.side_effect = Exception("Provider not found") + + with pytest.raises(Exception, match="Provider not found"): + generate_threatscore_report( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + compliance_id=self.compliance_id, + output_path=self.output_path, + provider_id=self.provider_id, + only_failed=True, + min_risk_level=4, + ) + + +@pytest.mark.django_db +class TestGenerateThreatscoreReportTask: + def setup_method(self): + self.scan_id = str(uuid.uuid4()) + self.provider_id = str(uuid.uuid4()) + self.tenant_id = str(uuid.uuid4()) + + @patch("tasks.tasks.generate_threatscore_report_job") + def test_generate_threatscore_report_task_calls_job(self, mock_generate_job): + mock_generate_job.return_value = {"upload": True} + + result = generate_threatscore_report_task( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + assert result == {"upload": True} + mock_generate_job.assert_called_once_with( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + @patch("tasks.tasks.generate_threatscore_report_job") + def test_generate_threatscore_report_task_handles_job_exception( + self, mock_generate_job + ): + mock_generate_job.side_effect = Exception("Job failed") + + with pytest.raises(Exception, match="Job failed"): + generate_threatscore_report_task( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) diff --git a/api/src/backend/tasks/tests/test_tasks.py b/api/src/backend/tasks/tests/test_tasks.py index b4262027f6..a98341ef35 100644 --- a/api/src/backend/tasks/tests/test_tasks.py +++ b/api/src/backend/tasks/tests/test_tasks.py @@ -98,7 +98,11 @@ class TestGenerateOutputs: ), patch( "tasks.tasks._generate_output_directory", - return_value=("out-dir", "comp-dir"), + return_value=( + "/tmp/test/out-dir", + "/tmp/test/comp-dir", + "/tmp/test/threat-dir", + ), ), patch("tasks.tasks.Scan.all_objects.filter") as mock_scan_update, patch("tasks.tasks.rmtree"), @@ -126,7 +130,8 @@ class TestGenerateOutputs: patch("tasks.tasks.get_compliance_frameworks"), patch("tasks.tasks.Finding.all_objects.filter") as mock_findings, patch( - "tasks.tasks._generate_output_directory", return_value=("out", "comp") + "tasks.tasks._generate_output_directory", + return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"), ), patch("tasks.tasks.FindingOutput._transform_findings_stats"), patch("tasks.tasks.FindingOutput.transform_api_finding"), @@ -168,15 +173,35 @@ class TestGenerateOutputs: mock_finding_output = MagicMock() mock_finding_output.compliance = {"cis": ["requirement-1", "requirement-2"]} + html_writer_mock = MagicMock() + html_writer_mock._data = [] + html_writer_mock.close_file = False + html_writer_mock.transform = MagicMock() + html_writer_mock.batch_write_data_to_file = MagicMock() + + compliance_writer_mock = MagicMock() + compliance_writer_mock._data = [] + compliance_writer_mock.close_file = False + compliance_writer_mock.transform = MagicMock() + compliance_writer_mock.batch_write_data_to_file = MagicMock() + + # Create a mock class that returns our mock instance when called + mock_compliance_class = MagicMock(return_value=compliance_writer_mock) + + mock_provider = MagicMock() + mock_provider.provider = "aws" + mock_provider.uid = "test-provider-uid" + with ( patch("tasks.tasks.ScanSummary.objects.filter") as mock_filter, - patch("tasks.tasks.Provider.objects.get"), + patch("tasks.tasks.Provider.objects.get", return_value=mock_provider), patch("tasks.tasks.initialize_prowler_provider"), patch("tasks.tasks.Compliance.get_bulk", return_value={"cis": MagicMock()}), patch("tasks.tasks.get_compliance_frameworks", return_value=["cis"]), patch("tasks.tasks.Finding.all_objects.filter") as mock_findings, patch( - "tasks.tasks._generate_output_directory", return_value=("out", "comp") + "tasks.tasks._generate_output_directory", + return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"), ), patch( "tasks.tasks.FindingOutput._transform_findings_stats", @@ -190,6 +215,20 @@ class TestGenerateOutputs: patch("tasks.tasks._upload_to_s3", return_value="s3://bucket/f.zip"), patch("tasks.tasks.Scan.all_objects.filter"), patch("tasks.tasks.rmtree"), + patch( + "tasks.tasks.OUTPUT_FORMATS_MAPPING", + { + "html": { + "class": lambda *args, **kwargs: html_writer_mock, + "suffix": ".html", + "kwargs": {}, + } + }, + ), + patch( + "tasks.tasks.COMPLIANCE_CLASS_MAP", + {"aws": [(lambda x: True, mock_compliance_class)]}, + ), ): mock_filter.return_value.exists.return_value = True mock_findings.return_value.order_by.return_value.iterator.return_value = [ @@ -197,29 +236,12 @@ class TestGenerateOutputs: True, ] - html_writer_mock = MagicMock() - with ( - patch( - "tasks.tasks.OUTPUT_FORMATS_MAPPING", - { - "html": { - "class": lambda *args, **kwargs: html_writer_mock, - "suffix": ".html", - "kwargs": {}, - } - }, - ), - patch( - "tasks.tasks.COMPLIANCE_CLASS_MAP", - {"aws": [(lambda x: True, MagicMock())]}, - ), - ): - generate_outputs_task( - scan_id=self.scan_id, - provider_id=self.provider_id, - tenant_id=self.tenant_id, - ) - html_writer_mock.batch_write_data_to_file.assert_called_once() + generate_outputs_task( + scan_id=self.scan_id, + provider_id=self.provider_id, + tenant_id=self.tenant_id, + ) + html_writer_mock.batch_write_data_to_file.assert_called_once() def test_transform_called_only_on_second_batch(self): raw1 = MagicMock() @@ -256,7 +278,11 @@ class TestGenerateOutputs: ), patch( "tasks.tasks._generate_output_directory", - return_value=("outdir", "compdir"), + return_value=( + "/tmp/test/outdir", + "/tmp/test/compdir", + "/tmp/test/threatdir", + ), ), patch("tasks.tasks._compress_output_files", return_value="outdir.zip"), patch("tasks.tasks._upload_to_s3", return_value="s3://bucket/outdir.zip"), @@ -303,12 +329,14 @@ class TestGenerateOutputs: def __init__(self, *args, **kwargs): self.transform_calls = [] self._data = [] + self.close_file = False writer_instances.append(self) def transform(self, fos, comp_obj, name): self.transform_calls.append((fos, comp_obj, name)) def batch_write_data_to_file(self): + # Mock implementation - do nothing pass two_batches = [ @@ -329,7 +357,11 @@ class TestGenerateOutputs: patch("tasks.tasks.get_compliance_frameworks", return_value=["cis"]), patch( "tasks.tasks._generate_output_directory", - return_value=("outdir", "compdir"), + return_value=( + "/tmp/test/outdir", + "/tmp/test/compdir", + "/tmp/test/threatdir", + ), ), patch("tasks.tasks.FindingOutput._transform_findings_stats"), patch( @@ -368,15 +400,35 @@ class TestGenerateOutputs: mock_finding_output = MagicMock() mock_finding_output.compliance = {"cis": ["requirement-1", "requirement-2"]} + json_writer_mock = MagicMock() + json_writer_mock._data = [] + json_writer_mock.close_file = False + json_writer_mock.transform = MagicMock() + json_writer_mock.batch_write_data_to_file = MagicMock() + + compliance_writer_mock = MagicMock() + compliance_writer_mock._data = [] + compliance_writer_mock.close_file = False + compliance_writer_mock.transform = MagicMock() + compliance_writer_mock.batch_write_data_to_file = MagicMock() + + # Create a mock class that returns our mock instance when called + mock_compliance_class = MagicMock(return_value=compliance_writer_mock) + + mock_provider = MagicMock() + mock_provider.provider = "aws" + mock_provider.uid = "test-provider-uid" + with ( patch("tasks.tasks.ScanSummary.objects.filter") as mock_filter, - patch("tasks.tasks.Provider.objects.get"), + patch("tasks.tasks.Provider.objects.get", return_value=mock_provider), patch("tasks.tasks.initialize_prowler_provider"), patch("tasks.tasks.Compliance.get_bulk", return_value={"cis": MagicMock()}), patch("tasks.tasks.get_compliance_frameworks", return_value=["cis"]), patch("tasks.tasks.Finding.all_objects.filter") as mock_findings, patch( - "tasks.tasks._generate_output_directory", return_value=("out", "comp") + "tasks.tasks._generate_output_directory", + return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"), ), patch( "tasks.tasks.FindingOutput._transform_findings_stats", @@ -390,6 +442,20 @@ class TestGenerateOutputs: patch("tasks.tasks._upload_to_s3", return_value="s3://bucket/file.zip"), patch("tasks.tasks.Scan.all_objects.filter"), patch("tasks.tasks.rmtree", side_effect=Exception("Test deletion error")), + patch( + "tasks.tasks.OUTPUT_FORMATS_MAPPING", + { + "json": { + "class": lambda *args, **kwargs: json_writer_mock, + "suffix": ".json", + "kwargs": {}, + } + }, + ), + patch( + "tasks.tasks.COMPLIANCE_CLASS_MAP", + {"aws": [(lambda x: True, mock_compliance_class)]}, + ), ): mock_filter.return_value.exists.return_value = True mock_findings.return_value.order_by.return_value.iterator.return_value = [ @@ -397,29 +463,13 @@ class TestGenerateOutputs: True, ] - with ( - patch( - "tasks.tasks.OUTPUT_FORMATS_MAPPING", - { - "json": { - "class": lambda *args, **kwargs: MagicMock(), - "suffix": ".json", - "kwargs": {}, - } - }, - ), - patch( - "tasks.tasks.COMPLIANCE_CLASS_MAP", - {"aws": [(lambda x: True, MagicMock())]}, - ), - ): - with caplog.at_level("ERROR"): - generate_outputs_task( - scan_id=self.scan_id, - provider_id=self.provider_id, - tenant_id=self.tenant_id, - ) - assert "Error deleting output files" in caplog.text + with caplog.at_level("ERROR"): + generate_outputs_task( + scan_id=self.scan_id, + provider_id=self.provider_id, + tenant_id=self.tenant_id, + ) + assert "Error deleting output files" in caplog.text @patch("tasks.tasks.rls_transaction") @patch("tasks.tasks.Integration.objects.filter") @@ -435,7 +485,8 @@ class TestGenerateOutputs: patch("tasks.tasks.get_compliance_frameworks", return_value=[]), patch("tasks.tasks.Finding.all_objects.filter") as mock_findings, patch( - "tasks.tasks._generate_output_directory", return_value=("out", "comp") + "tasks.tasks._generate_output_directory", + return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"), ), patch("tasks.tasks.FindingOutput._transform_findings_stats"), patch("tasks.tasks.FindingOutput.transform_api_finding"), @@ -476,8 +527,15 @@ class TestScanCompleteTasks: @patch("tasks.tasks.create_compliance_requirements_task.apply_async") @patch("tasks.tasks.perform_scan_summary_task.si") @patch("tasks.tasks.generate_outputs_task.si") + @patch("tasks.tasks.generate_threatscore_report_task.si") + @patch("tasks.tasks.check_integrations_task.si") def test_scan_complete_tasks( - self, mock_outputs_task, mock_scan_summary_task, mock_compliance_tasks + self, + mock_check_integrations_task, + mock_threatscore_task, + mock_outputs_task, + mock_scan_summary_task, + mock_compliance_tasks, ): _perform_scan_complete_tasks("tenant-id", "scan-id", "provider-id") mock_compliance_tasks.assert_called_once_with( @@ -492,6 +550,16 @@ class TestScanCompleteTasks: provider_id="provider-id", tenant_id="tenant-id", ) + mock_threatscore_task.assert_called_once_with( + tenant_id="tenant-id", + scan_id="scan-id", + provider_id="provider-id", + ) + mock_check_integrations_task.assert_called_once_with( + tenant_id="tenant-id", + provider_id="provider-id", + scan_id="scan-id", + ) @pytest.mark.django_db @@ -662,7 +730,7 @@ class TestCheckIntegrationsTask: mock_initialize_provider.return_value = MagicMock() mock_compliance_bulk.return_value = {} mock_get_frameworks.return_value = [] - mock_generate_dir.return_value = ("out-dir", "comp-dir") + mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir") mock_transform_stats.return_value = {"stats": "data"} # Mock findings @@ -787,7 +855,7 @@ class TestCheckIntegrationsTask: mock_initialize_provider.return_value = MagicMock() mock_compliance_bulk.return_value = {} mock_get_frameworks.return_value = [] - mock_generate_dir.return_value = ("out-dir", "comp-dir") + mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir") mock_transform_stats.return_value = {"stats": "data"} # Mock findings @@ -903,7 +971,7 @@ class TestCheckIntegrationsTask: mock_initialize_provider.return_value = MagicMock() mock_compliance_bulk.return_value = {} mock_get_frameworks.return_value = [] - mock_generate_dir.return_value = ("out-dir", "comp-dir") + mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir") mock_transform_stats.return_value = {"stats": "data"} # Mock findings diff --git a/docs/developer-guide/provider.mdx b/docs/developer-guide/provider.mdx index e0b7c62284..85dafc6300 100644 --- a/docs/developer-guide/provider.mdx +++ b/docs/developer-guide/provider.mdx @@ -17,160 +17,3442 @@ A provider is any platform or service that offers resources, data, or functional For providers supported by Prowler, refer to [Prowler Hub](https://hub.prowler.com/). -There are some custom providers added by the community, like [NHN Cloud](https://www.nhncloud.com/), that are not maintained by the Prowler team, but can be used in the Prowler CLI. They can be checked directly at the [Prowler GitHub repository](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). - + There are some custom providers added by the community, like [NHN Cloud](https://www.nhncloud.com/), that are not maintained by the Prowler team, but can be used in the Prowler CLI. The main purpose of this documentation is to guide you through creating a new provider and integrating it not only in the CLI, but also in the API and UI. Non official providers can be checked directly at the [Prowler GitHub repository](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). -## Adding a New Provider +--- -To integrate an unsupported Prowler provider and implement its security checks, create a dedicated folder for all related files (e.g., services, checks)." +## Provider Types in Prowler -This folder must be placed within [`prowler/providers//`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). +Prowler supports several types of providers, each with its own implementation pattern and use case. Understanding these differences is key to designing your provider correctly. -Within this folder the following folders are also to be created: +### Classifying your Provider -- `lib` – Stores additional utility functions and core files required by every provider. The following files and subfolders are commonly found in every provider's `lib` folder: +Before implementing a new provider, you need to determine which type it belongs to. This classification will guide your implementation approach and help you choose the right patterns and libraries. - - `service/service.py` – Provides a generic service class to be inherited by all services. - - `arguments/arguments.py` – Handles provider-specific argument parsing. - - `mutelist/mutelist.py` – Manages the mutelist functionality for the provider. +#### Decision Criteria -- `services` – Stores all [services](/developer-guide/services) that the provider offers and want to be audited by [Prowler checks](/developer-guide/checks). +Once you have decided the provider you want or need to add to Prowler, the next step is to study how to retrieve data from it. Based on that, the provider will fall into one of the following types: SDK, API or Tool/Wrapper (maybe in the future there will be new types but for now this are the only ones). -- `__init__.py` (empty) – Ensures Python recognizes this folder as a package. +**Choose SDK Provider if:** -- `_provider.py` – Defines authentication logic, configurations, and other provider-specific data. +- The target platform/service has an **official Python SDK** available +- The target platform/service has a **non-official Python SDK** available but it's been updated and maintained +- You need to support **multiple authentication methods** (profiles, service principals, IAM roles, etc.) +- The SDK provides **built-in session management**, retry logic, and error handling +- You want to leverage **SDK-specific features** like credential chaining, role assumption, etc +- The platform is a **major cloud provider** (AWS, Azure, GCP, etc.) or has mature SDK support -- `models.py` – Contains necessary models for the new provider. +**Choose API Provider if:** -By adhering to this structure, Prowler can effectively support services and security checks for additional providers. +- The target platform has a **REST API** but **no official Python SDK** +- The target platform has a **non-official Python SDK** available but it's not updated and maintained +- You need to implement **custom authentication flows** (OAuth, token-based, etc.) +- The platform is a **custom or community service** without official SDK support +- You want to use **standard HTTP libraries** like `requests` for API calls +- The platform exposes **well-documented REST endpoints** but lacks SDK tooling - -If your new provider requires a Python library (such as an official SDK or API client) to connect to its services, make sure to add it as a dependency in the `pyproject.toml` file. This ensures that all contributors and users have the necessary packages installed when working with your provider. +**Choose Tool/Wrapper Provider if:** - -## Provider Structure in Prowler +- You're integrating a **third-party security tool** or library +- The tool provides **scanning capabilities** that need to be adapted to Prowler's interface +- You don't need **authentication or session management** (the tool handles this) +- You need to **map tool arguments** and **convert outputs** to Prowler's format -Prowler's provider architecture is designed to facilitate security audits through a generic service tailored to each provider. This is accomplished by passing the necessary parameters to the constructor, which initializes all required session values. +**Special Case - Hybrid Providers:** -### Base Class +- Some providers may **combine multiple approaches** (e.g., SDK + Tool wrapper, SDK + API, etc.) +- Example: M365 uses **msgraph SDK** for authentication and some checks, and **PowerShell wrapper** for other checks that the SDK doesn't support +- These require **custom implementation patterns** that blend different provider types -All Prowler providers inherit from the same base class located in [`prowler/providers/common/provider.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/common/provider.py). It is an [abstract base class](https://docs.python.org/3/library/abc.html) that defines the interface for all provider classes. +#### Classification Examples -### Provider Class +| Provider | Type | Reasoning | +| ----------- | ------ | ----------------------------------------------------------------- | +| AWS | SDK | Official boto3 SDK, multiple auth methods, mature ecosystem | +| Azure | SDK | Official azure-identity SDK, service principals, managed identity | +| GCP | SDK | Official google-auth SDK, service accounts, ADC support | +| Kubernetes | SDK | Official kubernetes SDK, service accounts, ADC support | +| NHN Cloud | API | Custom REST API, no official SDK, community provider | +| MongoDB Atlas| API | Custom REST API, no official SDK | +| IAC | Tool | Third-party security tool that uses trivy, no auth needed, output conversion| +| M365 | Hybrid | Combines msgraph SDK for auth + PowerShell wrapper for operations | +| GitHub | Hybrid | Non-Official PyGithub SDK but it's been updated and maintained + Official graphql API requests| -#### Provider Implementation Guidance +#### Questions to Ask Yourself -Given the complexity and variability of providers, use existing provider implementations as templates when developing new integrations. +**1. Does the platform have an official Python SDK?** -- [AWS](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/aws/aws_provider.py) -- [GCP](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/gcp/gcp_provider.py) -- [Azure](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/azure/azure_provider.py) -- [Kubernetes](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/kubernetes/kubernetes_provider.py) -- [Microsoft365](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/microsoft365/microsoft365_provider.py) -- [GitHub](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/github/github_provider.py) -- [MongoDB Atlas](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/mongodbatlas/mongodbatlas_provider.py) +- Yes → Consider SDK Provider +- No → Continue to question 2 -### Basic Provider Implementation: Pseudocode Example +**2. Does the platform have a non-official Python SDK?** -To simplify understanding, the following pseudocode outlines the fundamental structure of a provider, including library imports necessary for authentication. +- Yes → Then if the SDK is updated and maintained, consider SDK Provider, otherwise continue to question 3. +- No → Continue to question 3 -```python title="Provider Example Class" +**3. Is this a third-party security tool or library?** -# Library Imports for Authentication +- Yes → Consider Tool/Wrapper Provider +- No → Continue to question 4 -# When implementing authentication for a provider, import the required libraries. +**4. Does the platform expose a REST API?** -from prowler.config.config import load_and_validate_config_file -from prowler.lib.logger import logger -from prowler.lib.mutelist.mutelist import parse_mutelist_file -from prowler.lib.utils.utils import print_boxes -from prowler.providers.common.models import Audit_Metadata +- Yes → Consider API Provider +- No → You may need a custom approach + +![FlowChart Decision](../img/provider-decision-tree.png) + +#### Implementation Complexity + +- **SDK Providers**: Low complexity. You have mature examples like AWS, Azure, GCP, Kubernetes, etc. that you can leverage to implement your provider. +- **API Providers**: Medium complexity. You need to implement the authentication and session management, and the API calls to the provider. You now have NHN and MongoDB Atlas as example to follow. +- **Tool/Wrapper Providers**: High complexity. You need to implement the argument/output mapping to the provider and handle problems that the tool/wrapper may have. You now have IAC and the PowerShell wrapper as example to follow. +- **Hybrid Providers**: High complexity. You need to "customize" your provider, mixing the other types of providers in order to achieve the desired result. You have M365 (msgraph SDK + PowerShell wrapper) and Github (PyGithub SDK + graphql API requests) as examples. + +### Determining Regional vs Non-Regional Architecture + +After classifying your provider type, the next critical decision is determining whether your provider operates with **regional concepts** or is **global/non-regional**. This decision fundamentally affects how your provider and services are structured and executed. + +#### Regional Providers + +Regional providers operate across multiple geographic locations and require region-specific resource discovery and iteration. + +**Examples:** + +- **AWS**: Has regions like `us-east-1`, `eu-west-1`, `ap-southeast-2` +- **Azure**: Has regions like `East US`, `West Europe`, `Australia East` +- **GCP**: Has regions like `us-central1`, `europe-west1`, `asia-southeast1` + +**Implementation Requirements:** + +- Must implement region discovery and iteration +- Services must be instantiated per region or handle multi-region data +- Checks must execute across all available/specified regions +- Resource ARNs/IDs must include region information +- Region-specific client initialization + +**Execution Pattern:** + +```python +# Regional provider execution pattern +for region in provider.get_regions(): + regional_client = service.get_regional_client(region) + regional_resources = regional_client.discover_resources() + # Process regional resources +``` + +#### Non-Regional (Global) Providers + +Non-regional providers operate globally without geographic partitioning. + +**Examples:** + +- **GitHub**: Repositories, organizations are global concepts +- **M365**: Tenants operate globally across Microsoft datacenters +- **Kubernetes**: Clusters are independent units without regional concepts + +**Implementation Requirements:** + +- Single global client/session +- No region iteration required +- Global resource discovery +- Simpler resource identification (no region in ARNs/IDs) +- Single audit execution + +**Execution Pattern:** + +```python +# Non-regional provider execution pattern +global_client = service.get_client() +global_resources = global_client.discover_resources() +# Process all resources in single iteration +``` + +#### Decision Matrix + +| Aspect | Regional Provider | Non-Regional Provider | +| ------------------------ | ------------------------ | ---------------------- | +| **Client Init** | Per-region clients | Single global client | +| **Resource Discovery** | Iterate through regions | Single discovery call | +| **Resource ARN/ID** | Include region | Global identifier/None | +| **Audit Execution** | Multi-region loops | Single execution | +| **Service Architecture** | Region-aware services | Global services | +| **Performance** | Parallelizable by region | Linear execution | + +#### Region Discovery + +Region discovery is the process of getting the list of regions that are available for the account. This is done by the provider and is stored in the `prowler/providers//lib/regions/_regions.py` file. + +```python +# File: prowler/providers/aws/aws_provider.py +def get_aws_enabled_regions(self, current_session: Session) -> set: + """get_aws_enabled_regions returns a set of enabled AWS regions""" + try: + # EC2 Client to check enabled regions + service = "ec2" + default_region = self.get_default_region(service) + ec2_client = current_session.client(service, region_name=default_region) + + enabled_regions = set() + # With AllRegions=False we only get the enabled regions for the account + for region in ec2_client.describe_regions(AllRegions=False).get("Regions", []): + enabled_regions.add(region.get("RegionName")) + + return enabled_regions + except Exception as error: + logger.error(f"{error.__class__.__name__}: {error}") + return set() +``` + +The function returns a JSON file containing the list of regions for the provider. It is used to retrieve the provider’s regions and to validate the region specified by the user. + +```json +# File: prowler/providers/aws/aws_regions_by_service.json (extract) +{ + "services": { + "ec2": { + "regions": { + "aws": [ + "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", + "ap-northeast-3", "ap-south-1", "ap-southeast-1", "ap-southeast-2", + "ca-central-1", "eu-central-1", "eu-north-1", "eu-south-1", + "eu-west-1", "eu-west-2", "eu-west-3", "me-south-1", + "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2" + ], + "aws-cn": ["cn-north-1", "cn-northwest-1"], + "aws-us-gov": ["us-gov-east-1", "us-gov-west-1"] + } + } + } +} +``` + +### Regional Service Implementation + +For detailed guidance on implementing services for regional services, including code examples, service architecture, and check execution patterns, see the [Regional Service Implementation](./services#regional-service-implementation) section in the Services documentation. + +**Key concepts covered:** + +- Threading and parallel processing across regions +- Service implementation patterns for regional providers +- Cross-region resource attribution and ARN handling +- Best practices for performance and error isolation + +## Step 1: Create the Provider Backend (CLI Integration) + +Once the type of provider and its regional architecture are determined, the next step is to start creating the code of the provider. + +### SDK Providers + +General aspects to consider when implementing a new SDK provider: + +**Definition:** + +- Use the official SDK of the provider to interact with its resources and APIs. +- Examples: AWS (boto3), Azure (azure-identity), GCP (google-auth), Kubernetes (kubernetes), M365 (msal/msgraph), GitHub (PyGithub). + +**Typical Use Cases:** + +- Cloud platforms and services with mature Python SDKs. +- Need to support multiple authentication methods (profiles, service principals, etc). +- Providers that offer comprehensive Python libraries for resource management. + +**Key Characteristics:** + +- Authentication and session management handled by the SDK. +- Arguments: Depends on the provider, but for example we can have `profile`, `region`, `tenant_id`, `client_id`, `client_secret`, etc. +- Outputs: Standardized via SDK models and responses. +- Session objects that can be reused across multiple API calls. +- Built-in retry logic and error handling. + +**Implementation Details:** + +- SDK providers typically use credential objects or session objects provided by the official SDK. +- They often support multiple authentication methods (several types of credentials, configuration files, IAM roles, etc.). +- Session management includes token refresh, connection pooling, and retry mechanisms. +- Resource discovery and enumeration is usually straightforward through SDK methods. + +--- + +### Implementation Guide for SDK Providers + +Now it's time to start creating the code needed to implement the provider. + +#### Step 1: Create the Provider Structure + +**Explanation:** +SDK providers require a specific folder structure to organize authentication, configuration, and service management. This structure follows Prowler's conventions and ensures proper integration with the CLI and API. + +**Required Structure:** + +``` +prowler/providers// +├── __init__.py +├── _provider.py +├── models.py +├── exceptions/ +│ ├── __init__.py +│ └── exceptions.py +├── services/ +│ ├── service_name1/ +│ └── service_name2/ +└── lib/ + ├── __init__.py + ├── arguments/ + │ ├── __init__.py + │ └── arguments.py + ├── mutelist/ + │ ├── __init__.py + │ └── mutelist.py + ├── regions/ + │ ├── __init__.py + │ └── _regions.py + └── service/ + ├── __init__.py + └── service.py +``` + +**Key Components:** + +- **`_provider.py`**: Main provider class with authentication and session management +- **`models.py`**: Data structures for identity, session, and provider-specific information +- **`exceptions/`**: Custom exception classes for error handling +- **`services/`**: Folder that contains all the provider services, how to make a new service is explained in another section. +- **`lib/arguments/`**: CLI argument validation and parsing +- **`lib/mutelist/`**: Resource exclusion and muting functionality +- **`lib/regions/`**: Region management and validation. If the provider is NOT regional, this folder will not be created. +- **`lib/service/`**: Base service class for provider-specific services + +#### Step 2: Implement the Provider Class + +**Explanation:** +The provider class is the core component that handles authentication, session management, and identity information. It inherits from Prowler's base Provider class and implements SDK-specific authentication flows. All providers must share, as far as possible, common patterns for session setup, identity management, and credential validation. + +Nevertheless, you may encounter changes and must adapt the implementation logic accordingly. A basic example of a common provider implementation is the following: + +**File:** `prowler/providers//_provider.py` + +```python +import os +from typing import Optional, Union from prowler.providers.common.provider import Provider -from prowler.providers..models import ( - # All provider models needed. - ProviderSessionModel, - ProviderIdentityModel, - ProviderOutputOptionsModel -) +from prowler.providers.common.models import Audit_Metadata, Connection +from prowler.config.config import load_and_validate_config_file, get_default_mute_file_path +from prowler.lib.logger import logger +from prowler.lib.utils.utils import print_boxes -class NewProvider(Provider): - # All properties from the class, some of which are properties in the base class. - _type: str = "" - _session: - _identity: +# Import your SDK and all the needed libraries for the provider. +import your_sdk_library +from your_sdk_library.auth_methods import ClientSecretCredential, ProfileCredential, DefaultCredential + +# Import the needed exceptions, mutelist and models for the provider. +from prowler.providers..exceptions.exceptions import Exceptions +from prowler.providers..mutelist.mutelist import Mutelist +from prowler.providers..models import NeededModels + +class YourProvider(Provider): + """ + YourProvider class is the main class for the Your Provider. + + This class is responsible for initializing the provider, setting up the session, + validating credentials, and managing identity information. + + Attributes: + _type (str): The provider type. + _session (YourSDKSession): The provider session. + _identity (YourProviderIdentityInfo): The provider identity information. + _audit_config (dict): The audit configuration. + _mutelist (YourProviderMutelist): The provider mutelist. + audit_metadata (Audit_Metadata): The audit metadata. + """ + + _type: str = "your_provider" + _session: your_sdk_library.Session + _identity: YourProviderIdentityInfo _audit_config: dict - _output_options: ProviderOutputOptionsModel - _mutelist: dict + _mutelist: YourProviderMutelist audit_metadata: Audit_Metadata - def __init__(self, arguments): + def __init__( + self, + # Authentication parameters + client_id: str = None, + client_secret: str = None, + tenant_id: str = None, + # Configuration + config_path: str = None, + config_content: dict = None, + mutelist_path: str = None, + mutelist_content: dict = None, + # Additional provider-specific parameters + region: str = None, + profile: str = None, + ): """ - Initializes the NewProvider instance. + Initializes the YourProvider instance. + Args: - arguments (dict): A dictionary containing configuration arguments. + client_id: The client ID for authentication + client_secret: The client secret for authentication + tenant_id: The tenant ID for authentication + config_path: Path to the configuration file + config_content: Configuration content as dictionary + mutelist_path: Path to the mutelist file + mutelist_content: Mutelist content as dictionary + region: The region to use + profile: The profile to use + + Raises: + YourProviderSetUpSessionError: If session setup fails + YourProviderInvalidCredentialsError: If credentials are invalid """ - logger.info("Setting provider ...") + logger.info("Initializing YourProvider ...") - # Initializing the Provider Session - - # Steps: - - # - Retrieve Account Information - # - Extract relevant account identifiers (subscriptions, projects, or other service references) from the provided arguments. - - # Establish a Session - - # Use the method enforced by the parent class to set up the session: - self._session = self.setup_session(credentials_file) - - # Define Provider Identity - # Assign the identity class, typically provided by the Python provider library: - self._identity = () - - # Configure the Provider - # Set the provider-specific configuration. - self._audit_config = load_and_validate_config_file( - self._type, arguments.config_file + # Setup session using SDK + self._session = self.setup_session( + client_id, client_secret, tenant_id, region, profile ) - # All the enforced properties by the parent class. + # Get identity information + self._identity = self.setup_identity(self._session) + + # Load configuration + if config_content: + self._audit_config = config_content + else: + if not config_path: + config_path = default_config_file_path + self._audit_config = load_and_validate_config_file(self._type, config_path) + + # Setup mutelist + if mutelist_content: + self._mutelist = YourProviderMutelist(mutelist_content=mutelist_content) + else: + if not mutelist_path: + mutelist_path = get_default_mute_file_path(self._type) + self._mutelist = YourProviderMutelist(mutelist_path=mutelist_path) + + Provider.set_global_provider(self) + + @staticmethod + def setup_session( + client_id: str = None, + client_secret: str = None, + tenant_id: str = None, + region: str = None, + profile: str = None, + ) -> your_sdk_library.Session: + """ + Sets up the provider session using the provided credentials. + + This method handles the authentication flow and creates a session object + that can be used to interact with the provider's services. + + Args: + client_id: The client ID for authentication + client_secret: The client secret for authentication + tenant_id: The tenant ID for authentication + region: The region to use + profile: The profile to use + + Returns: + YourSDKSession: The authenticated session object + + Raises: + YourProviderSetUpSessionError: If session setup fails + """ + try: + logger.debug("Creating session ...") + + # Determine authentication method based on provided parameters + if client_id and client_secret and tenant_id: + # Use client credentials authentication + credentials = your_sdk_library.ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret + ) + auth_method = "Client Credentials" + elif profile: + # Use profile-based authentication + credentials = your_sdk_library.ProfileCredential(profile=profile) + auth_method = "Profile" + else: + # Use default authentication (environment variables, etc.) + credentials = your_sdk_library.DefaultCredential() + auth_method = "Default" + + # Create session with credentials + session = your_sdk_library.Session( + credentials=credentials, + region=region + ) + + logger.debug(f"Session created using {auth_method} authentication") + return session + + except Exception as error: + logger.critical(f"Failed to setup session: {error}") + raise YourProviderSetUpSessionError( + original_exception=error, + file=os.path.basename(__file__), + ) + + def setup_identity(self, session: your_sdk_library.Session) -> YourProviderIdentityInfo: + """ + Gets identity information from the provider session. + + This method retrieves account information, user details, and other + identity-related data from the provider. + + Args: + session: The authenticated session object + + Returns: + YourProviderIdentityInfo: The identity information + + Raises: + YourProviderSetUpIdentityError: If identity setup fails + """ + try: + # Use SDK to get account/identity information + identity_info = session.get_identity() + + return YourProviderIdentityInfo( + account_id=identity_info.account_id, + account_name=identity_info.account_name, + region=identity_info.region, + user_id=identity_info.user_id, + # Add other identity fields as needed + ) + except Exception as e: + logger.error(f"Failed to get identity information: {e}") + raise YourProviderSetUpIdentityError( + original_exception=e, + file=os.path.basename(__file__), + ) + @property def identity(self): + """Returns the provider identity information.""" return self._identity @property def session(self): + """Returns the provider session object.""" return self._session @property def type(self): + """Returns the provider type.""" return self._type @property def audit_config(self): + """Returns the audit configuration.""" return self._audit_config @property - def output_options(self): - return self._output_options + def mutelist(self): + """Returns the provider mutelist.""" + return self._mutelist - def setup_session(self, ): + def print_credentials(self): """ - Sets up the Provider session. + Display account information with color formatting. + + This method prints the provider credentials and account information + in a formatted way using colorama for better readability. + """ + from colorama import Fore, Style + from prowler.lib.utils.utils import print_boxes + + report_lines = [ + f" Account ID: {Fore.YELLOW}{self._identity.account_id}{Style.RESET_ALL}", + f" Account Name: {Fore.YELLOW}{self._identity.account_name}{Style.RESET_ALL}", + f" Region: {Fore.YELLOW}{self._identity.region}{Style.RESET_ALL}", + f" User ID: {Fore.YELLOW}{self._identity.user_id}{Style.RESET_ALL}", + ] + report_title = f"{Style.BRIGHT}Using the {self._type.upper()} credentials below:{Style.RESET_ALL}" + print_boxes(report_lines, report_title) + + @staticmethod + def test_connection( + client_id: str = None, + client_secret: str = None, + tenant_id: str = None, + region: str = None, + profile: str = None, + raise_on_exception: bool = True, + provider_id: str = None, + ) -> Connection: + """ + Test connection to the provider. + + This method validates the provided credentials and tests the connection + to the provider's services. Args: - Can include all necessary arguments to set up the session + client_id: The client ID for authentication + client_secret: The client secret for authentication + tenant_id: The tenant ID for authentication + region: The region to test + profile: The profile to use + raise_on_exception: Whether to raise exceptions or return Connection object + provider_id: The provider ID to validate against Returns: - Credentials necessary to communicate with the provider. - """ - pass + Connection: Connection test result - """ - This method is enforced by parent class and is used to print all relevant - information during the prowler execution as a header of execution. - Displaying Account Information with Color Formatting. In Prowler, Account IDs, usernames, and other identifiers are typically displayed using color formatting provided by the colorama module (Fore). - """ - def print_credentials(self): - pass + Raises: + YourProviderSetUpSessionError: If session setup fails + YourProviderInvalidCredentialsError: If credentials are invalid + """ + try: + # Create temporary session for testing + test_session = YourProvider.setup_session( + client_id, client_secret, tenant_id, region, profile + ) + + # Test the connection by getting identity + identity = YourProvider.setup_identity(test_session) + + # Validate provider ID if provided + if provider_id and identity.account_id != provider_id: + raise YourProviderInvalidProviderIdError( + file=os.path.basename(__file__), + ) + + return Connection( + status=True, + message=f"Successfully connected to {provider_id or 'provider'}", + error=None, + ) + except Exception as e: + if raise_on_exception: + raise e + return Connection( + status=False, + message="Failed to connect", + error=str(e), + ) + + def get_regions(self) -> set: + """ + Get available regions for the provider. + + Returns: + set: Set of available region names + """ + # Implementation depends on your provider + # Example for cloud providers that support regions + return {"region1", "region2", "region3"} + + def get_services(self) -> list: + """ + Get available services for the provider. + + Returns: + list: List of available service names + """ + # Implementation depends on your provider + return ["service1", "service2", "service3"] ``` + +#### Step 3: Create Models + +**Explanation:** +Models define the data structures used by your provider. They include identity information, session details, and provider-specific configurations. These models ensure type safety and consistent data handling across the provider. + +**File:** `prowler/providers//models.py` + +```python +# Import the needed generic libraries for the provider. +from pydantic import BaseModel +from dataclasses import dataclass +from typing import Optional, List + +# Import the needed Prowler libraries for the provider. +from prowler.providers.common.models import ProviderOutputOptions +from prowler.config.config import output_file_timestamp + +class YourProviderIdentityInfo: + """ + Identity information for the provider. + + This class holds all the identity-related information retrieved + from the provider, including account details and user information. + """ + account_id: str + account_name: str + region: str + user_id: str + # Add other identity fields as needed + +class YourProviderSession: + """ + Session object that contains the credentials and authentication details for the provider. + + This class holds the actual credentials and authentication information needed + to establish a connection with the provider's services. + """ + # Authentication credentials + access_key: str + secret_key: str + # Or for other providers: + # client_id: str + # client_secret: str + # tenant_id: str + + # Connection details + region: str +``` + +#### Step 4: Implement Arguments + +**Explanation:** +Argument validation ensures that the provider receives valid configuration parameters. This step is crucial for preventing runtime errors and providing clear error messages to users. The validation should check for required parameters and validate their format. + +**File:** `prowler/providers//lib/arguments/arguments.py` + +```python +def init_parser(self): + """Init the Provider CLI parser""" + _parser = self.subparsers.add_parser( + "", parents=[self.common_providers_parser], help=" Provider" + ) + # Authentication Modes + _auth_subparser = _parser.add_argument_group("Authentication Modes") + _auth_modes_group = _auth_subparser.add_mutually_exclusive_group() + _auth_modes_group.add_argument( + "--credentials-file", + nargs="?", + metavar="FILE_PATH", + help="Authenticate using a Service Account Application Credentials JSON file", + ) + _auth_modes_group.add_argument( + "--impersonate-service-account", + nargs="?", + metavar="SERVICE_ACCOUNT", + help="Impersonate a Service Account", + ) + _parser.add_argument( + "--your-provider-region", + help="Your Provider Region", + type=str, + ) + _parser.add_argument( + "--env-auth", + action="store_true", + help="Use User and Password environment variables authentication to log in against ", + ) + # More arguments for the provider. +``` + +#### Step 5: Implement Mutelist + +**Explanation:** +The mutelist functionality allows users to exclude specific resources or checks from the audit. This is useful for handling false positives or excluding resources that are intentionally configured differently. + +**File:** `prowler/providers//lib/mutelist/mutelist.py` + +```python +from prowler.lib.mutelist.mutelist import Mutelist +from prowler.lib.check.models import CheckReportYourProvider + +class YourProviderMutelist(Mutelist): + """ + Mutelist implementation for YourProvider. + + This class handles the muting functionality for the provider, + allowing users to exclude specific checks or resources from audits. + """ + + def is_finding_muted(self, finding: CheckReportYourProvider) -> bool: + """ + Check if a specific finding is muted. + + Args: + finding: The finding to check + """ + return self.is_muted(finding.check_id, finding.resource_id) +``` + +#### Step 6: Implement Regions + +**Explanation:** +Region management is essential for cloud providers that operate across multiple geographic locations. This component handles region validation and provides region-specific functionality. + + +Regions are optional, only if the provider has regions, for example Github does not have regions, but AWS does. + + +**File:** `prowler/providers//lib/regions/_regions.py` + +```python +from typing import List, Set + +def get_regions() -> List[str]: + """ + Get list of available regions for the provider. + + Returns: + List[str]: List of available region names + """ + return [ + "region1", + "region2", + "region3", + # ... other regions + ] + +def validate_region(region: str) -> bool: + """ + Validate if a region is supported. + + Args: + region: The region to validate + + Returns: + bool: True if the region is valid, False otherwise + """ + return region in get_regions() + +def get_default_region() -> str: + """ + Get the default region for the provider. + + Returns: + str: The default region name + """ + return "region1" + +def get_global_region() -> str: + """ + Get the global region for the provider. + + Returns: + str: The global region name + """ + return "global" +``` + +#### Step 7: Create Custom Exceptions + +**Explanation:** +Custom exceptions are needed to be able to handle the errors in a more specific way. Prowler uses a structured exception system with error codes, messages, and remediation steps. + +**File:** `prowler/providers//exceptions.py` + +```python +from prowler.exceptions.exceptions import ProwlerException + + +# Exceptions codes from 7000 to 7999 are reserved for YourProvider exceptions (Numbers as example) +class YourProviderBaseException(ProwlerException): + """Base class for YourProvider Errors.""" + + YOUR_PROVIDER_ERROR_CODES = { + (7001, "YourProviderCredentialsError"): { + "message": "Error loading credentials for YourProvider", + "remediation": "Check the credentials and ensure they are properly set up. API_KEY and API_SECRET are required.", + }, + (7002, "YourProviderAuthenticationError"): { + "message": "Authentication failed with YourProvider", + "remediation": "Check the API credentials and ensure they are valid and have proper permissions.", + }, + (7003, "YourProviderInvalidRegionError"): { + "message": "Invalid region provided for YourProvider", + "remediation": "Check the region and ensure it is a valid region for YourProvider.", + }, + (7004, "YourProviderSetUpSessionError"): { + "message": "Error setting up session", + "remediation": "Check the session setup and ensure it is properly configured.", + }, + (7005, "YourProviderInvalidProviderIdError"): { + "message": "Provider does not match with the expected account_id", + "remediation": "Check the provider and ensure it matches the expected account_id.", + }, + } + + def __init__(self, code, file=None, original_exception=None, message=None): + provider = "YourProvider" + error_info = self.YOUR_PROVIDER_ERROR_CODES.get((code, self.__class__.__name__)) + if message: + error_info["message"] = message + super().__init__( + code=code, + source=provider, + file=file, + original_exception=original_exception, + error_info=error_info, + ) + + +class YourProviderCredentialsError(YourProviderBaseException): + """Base class for YourProvider credentials errors.""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7001, file=file, original_exception=original_exception, message=message + ) + + +class YourProviderAuthenticationError(YourProviderCredentialsError): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7002, file=file, original_exception=original_exception, message=message + ) + + +class YourProviderInvalidRegionError(YourProviderBaseException): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7003, file=file, original_exception=original_exception, message=message + ) + + +class YourProviderSetUpSessionError(YourProviderCredentialsError): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7004, file=file, original_exception=original_exception, message=message + ) + + +class YourProviderInvalidProviderIdError(YourProviderBaseException): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7005, file=file, original_exception=original_exception, message=message + ) +``` + +#### Step 8: Implement Service Base Class + +**Explanation:** +The service base class defines a common interface for all services in your provider, since they will inherit from it. It defines the client to make requests to, the audit configuration and the fixer configuration. + +**File:** `prowler/providers//lib/service/service.py` + +```python +from prowler.providers.._provider import Provider + +class YourProviderService(BaseService): + """ + Base service class for YourProvider services. + + This class provides common functionality for all services + within the provider, including session management and error handling. + """ + + def __init__(self, provider: Provider): + """ + Initialize the service. + + Args: + provider: The provider instance + """ + self.client = provider.session.get_client(self.service_name) + self.audit_config = provider.audit_config + self.fixer_config = provider.fixer_config +``` + +#### Step 9: Register in CLI + +**Explanation:** +Add your provider to the available providers in the CLI. + +**File:** `prowler/lib/cli/parser.py` + +```python +class ProwlerArgumentParser: + # Set the default parser + def __init__(self): + # CLI Arguments + self.parser = argparse.ArgumentParser( + prog="prowler", + formatter_class=RawTextHelpFormatter, + usage="prowler [-h] [--version] {aws,azure,gcp,kubernetes,m365,github,nhn,dashboard,iac,your_provider} ...", + epilog=""" +Available Cloud Providers: + {aws,azure,gcp,kubernetes,m365,github,iac,nhn,your_provider} + aws AWS Provider + azure Azure Provider + gcp GCP Provider + kubernetes Kubernetes Provider + m365 Microsoft 365 Provider + github GitHub Provider + iac IaC Provider (Preview) + nhn NHN Provider (Unofficial) + your_provider Your Provider + +Available components: + dashboard Local dashboard + +To see the different available options on a specific component, run: + prowler {provider|dashboard} -h|--help + +Detailed documentation at https://docs.prowler.com +""", +``` + +#### Step 10: Register in Main + +**Explanation:** +Main registration makes your provider discoverable by Prowler's core system. It's needed to add your provider to the output options and to the compliance evaluation. + +**File:** `prowler/__main__.py` + +```python +# In the prowler setup output options section + if provider == "aws": + output_options = AWSOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "azure": + output_options = AzureOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "gcp": + output_options = GCPOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "kubernetes": + output_options = KubernetesOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "github": + output_options = GithubOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "m365": + output_options = M365OutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "nhn": + output_options = NHNOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "iac": + output_options = IACOutputOptions( + args, bulk_checks_metadata + ) + elif provider == "your_provider": + output_options = YourProviderOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + + + # Setup Compliance Options + elif provider == "your_provider": + for compliance_name in input_compliance_frameworks: + if compliance_name.startswith("cis_"): + # Generate CIS Finding Object (example of compliance with CIS framework) + filename = ( + f"{output_options.output_directory}/compliance/" + f"{output_options.output_filename}_{compliance_name}.csv" + ) + cis = YourProviderCIS( + findings=finding_outputs, + compliance=bulk_compliance_frameworks[compliance_name], + file_path=filename, + ) + generated_outputs["compliance"].append(cis) + cis.batch_write_data_to_file() +``` + +#### Step 11: Register in the list of providers + +**Explanation:** +This is needed to be able to use the provider in the generic checks. The provider must be registered in the `init_global_provider` method to handle CLI arguments and initialization. + +**File:** `prowler/providers/common/provider.py` + +```python +elif "your_provider" in provider_class_name.lower(): + provider_class( + username=arguments.your_provider_username, + password=arguments.your_provider_password, + tenant_id=arguments.your_provider_tenant_id, + config_path=arguments.config_file, + mutelist_path=arguments.mutelist_file, + fixer_config=fixer_config, + ) +``` + +#### Step 12: Add to Config + +**Explanation:** +Configuration registration ensures your provider is recognized by Prowler's configuration system. This enables proper handling of provider-specific settings and defaults. + +**File:** `prowler/config/config.py` + +```python +class Provider(str, Enum): + AWS = "aws" + AZURE = "azure" + GCP = "gcp" + KUBERNETES = "kubernetes" + M365 = "m365" + GITHUB = "github" + YOUR_PROVIDER = "your_provider" # Add your provider here +``` + +In some cases, you may need to create a new configuration file for your provider, for example, the AWS one that is inside `prowler/providers/aws/config.py`. + +#### Step 13: Create Compliance Files + +**Explanation:** +Compliance files define the security checks and standards that your provider supports. These JSON files map security controls to specific checks and provide remediation guidance. It's needed to create the folder with an init file to ensure the provider will work, however, adding different compliance files is optional. + +**Folder:** `prowler/compliance//` + +```json +{ + "Framework": "CIS", + "Version": "1.0", + "Provider": "your_provider", + "Description": "Description of the compliance framework", + # The requirements depends on the framework, for example, CIS has a requirements section with the checks and attributes. + "Requirements": [ + { + "Id": "1.1.1", + "Description": "Description of the requirement", + "Checks": ["your_provider_check_1", "your_provider_check_2"], + "Attributes": [] + } + ] +} +``` + +#### Step 14: Add Output Support + +**Explanation:** +Output support ensures that your provider's results are properly formatted in Prowler's various output formats (CSV, JSON, HTML, etc.). This step integrates your provider with Prowler's reporting system. + +**File:** `prowler/lib/outputs/summary_table.py` + +```python +# Add your provider case in the display_summary_table function +elif provider.type == "your_provider": + entity_type = "Your Entity Type" + audited_entities = provider.identity.your_entity_field +``` + +**File:** `prowler/lib/outputs/finding.py` + +```python +# Add your provider case in the fill_common_finding_data function +elif provider.type == "your_provider": + output_data["auth_method"] = f"Your Auth Method: {get_nested_attribute(provider, 'identity.auth_type')}" + output_data["account_uid"] = get_nested_attribute(provider, "identity.account_id") + output_data["account_name"] = get_nested_attribute(provider, "identity.account_name") + output_data["resource_name"] = check_output.resource_name + output_data["resource_uid"] = check_output.resource_id + output_data["region"] = check_output.location # or your location field +``` + +**File:** `prowler/lib/outputs/outputs.py` + +```python +# Add your provider case in the stdout_report function +if finding.check_metadata.Provider == "your_provider": + details = finding.your_location_field # e.g., finding.location, finding.namespace, etc. +``` + +#### Step 15: Generate the HTML Report + +**Explanation:** +The HTML file is needed to be able to generate the HTML report. This step involves adding support for your provider in the HTML output generation system to ensure proper display of assessment summaries and findings. + +**File:** `prowler/lib/outputs/html/html.py` + +```python +@staticmethod +def get_your_provider_assessment_summary(provider: Provider) -> str: + """ + get_your_provider_assessment_summary gets the HTML assessment summary for your provider + + Args: + provider (Provider): the provider object + + Returns: + str: the HTML assessment summary + """ + try: + return f""" +
+
+
+ Your Provider Assessment Summary +
+
    +
  • + Your Entity Type: {provider.identity.your_entity_field} +
  • +
  • + Your Location Field: {provider.identity.your_location_field} +
  • +
+
+
+
+
+
+ Your Provider Credentials +
+
    +
  • + Authentication Method: {provider.auth_method} +
  • +
  • + Identity ID: {provider.identity.identity_id} +
  • +
+
+
""" + except Exception as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}] -- {error}" + ) + return "" +``` + +#### Step 16: Add the Check Report Model + +**Explanation:** +Add the provider to the generic models, this is needed to be able to use the provider in the generic checks. + +**File:** `prowler/providers/check/models.py` + +```python +@dataclass +class CheckReportYourProvider(CheckReport): + """ + Check report for YourProvider. + """ + resource_name: str + resource_id: str + + def _init_(self, metadata: Dict, resource: Any) -> None: + super()._init_(metadata, resource) + self.resource_name = resource.name + self.resource_id = resource.id +``` + +#### Step 17: Add Dependencies + +**Explanation:** +Dependencies ensure that your provider's required libraries are available when Prowler is installed. This step adds the necessary SDK or API client to Prowler's dependency management. + +**File:** `pyproject.toml` + +```toml +[tool.poetry.dependencies] +python = "^3.9" +# ... other dependencies +your-sdk-library = "^1.0.0" # Add your SDK dependency +``` + +#### Step 18: Create Tests + +**Explanation:** +Testing ensures that your provider works correctly and maintains compatibility as Prowler evolves. Comprehensive tests cover authentication, session management, and provider-specific functionality. + +**Folder:** `tests/providers//` + +```python +import pytest +from prowler.providers.your_provider.your_provider import YourProvider + +class TestYourProvider: + """Test cases for YourProvider.""" + + def test_provider_initialization_with_client_credentials(self): + """Test provider initialization with client credentials.""" + provider = YourProvider( + client_id="test_client_id", + client_secret="test_client_secret", + tenant_id="test_tenant_id" + ) + assert provider.type == "your_provider" + assert provider.identity is not None + assert provider.session is not None + + def test_provider_initialization_with_profile(self): + """Test provider initialization with profile.""" + provider = YourProvider( + profile="test_profile" + ) + assert provider.type == "your_provider" + assert provider.identity is not None + + def test_connection_test(self): + """Test connection functionality.""" + result = YourProvider.test_connection( + client_id="test_client_id", + client_secret="test_client_secret", + tenant_id="test_tenant_id" + ) + # Add assertions based on expected behavior + + def test_identity_retrieval(self): + """Test identity information retrieval.""" + provider = YourProvider( + client_id="test_client_id", + client_secret="test_client_secret", + tenant_id="test_tenant_id" + ) + assert provider.identity.account_id is not None + assert provider.identity.account_name is not None + + def test_argument_validation(self): + """Test argument validation.""" + from prowler.providers.your_provider.lib.arguments.arguments import ( + validate_your_provider_arguments + ) + + # Valid arguments + validate_your_provider_arguments( + client_id="test_client_id", + client_secret="test_client_secret", + tenant_id="test_tenant_id" + ) + + # Invalid arguments + with pytest.raises(ValueError, match="at least one authentication method"): + validate_your_provider_arguments() +``` + +#### Step 19: Update Documentation + +**Explanation:** +Documentation updates ensure that users can find information about your provider in Prowler's documentation. This includes examples, configuration guides, and troubleshooting information. + +Update the provider documentation to include your new provider in the examples and implementation guidance. + +--- + +### API Providers + +**Definition:** + +- Interact directly with the provider's REST API using HTTP requests (e.g., via `requests`). +- Examples: NHN Cloud. + +**Typical Use Cases:** + +- Providers without an official Python SDK. +- Providers with a non-official Python SDK that is not updated and maintained. +- Providers that expose REST APIs and meet above requirements. + +**Key Characteristics:** + +- Manual management of authentication (tokens, username/password, etc). +- Arguments: Depends on the provider, for example, `username`, `password`, `tenant_id`, etc. +- Outputs: Dicts or custom models based on API responses. +- Custom HTTP session management with headers and authentication. +- Manual handling of pagination, rate limiting, and error responses. + +**Implementation Details:** + +- API providers require manual HTTP request management using libraries like `requests`. +- Authentication typically involves obtaining tokens via login endpoints or OAuth flows. +- Session management includes setting appropriate headers (Authorization, Content-Type, etc.). +- Resource discovery often requires multiple API calls to different endpoints. +- Error handling and retry logic must be implemented manually. + +--- + +### Implementation Guide for API Providers + +#### Step 1: Create the Provider Structure + +**Explanation:** +API providers require the same structure as the SDK providers, the main difference would be that due to the lack of an official Python SDK, some methods could be implemented differently or not implemented at all. + +**Required Structure:** + +``` +prowler/providers// +├── __init__.py +├── _provider.py +├── models.py +├── exceptions/ +│ ├── __init__.py +│ └── exceptions.py +├── services/ +│ ├── service_name1/ +│ └── service_name2/ +└── lib/ + ├── __init__.py + ├── arguments/ + │ ├── __init__.py + │ └── arguments.py + ├── mutelist/ + │ ├── __init__.py + │ └── mutelist.py + ├── regions/ + │ ├── __init__.py + │ └── regions.py + └── service/ + ├── __init__.py + └── service.py +``` + +**Key Components:** + +- **`_provider.py`**: Main provider class with HTTP session management +- **`models.py`**: Data structures for identity and API responses +- **`exceptions/`**: Custom exception classes for API errors +- **`services/`**: Folder that contains all the provider services +- **`lib/arguments/`**: CLI argument validation and parsing +- **`lib/mutelist/`**: Resource exclusion and muting functionality +- **`lib/regions/`**: Region management and validation. If the provider is NOT regional, this folder will not be created. +- **`lib/service/`**: Base service class for provider-specific services + +#### Step 2: Implement the Provider Class + +**Explanation:** +The provider class is the core component that handles HTTP session management, authentication, and identity information. It inherits from Prowler's base Provider class and implements API-specific authentication flows using direct HTTP requests. + +**File:** `prowler/providers//_provider.py` + +```python +import os +from typing import Optional +import requests +from prowler.providers.common.provider import Provider +from prowler.providers.common.models import Audit_Metadata, Connection +from prowler.config.config import load_and_validate_config_file, get_default_mute_file_path +from prowler.lib.logger import logger +from prowler.lib.utils.utils import print_boxes + +# Import the needed exceptions, mutelist and models for the provider. +from prowler.providers..exceptions.exceptions import Exceptions +from prowler.providers..lib.mutelist.mutelist import Mutelist +from prowler.providers..models import NeededModels + +class APIProvider(Provider): + """ + APIProvider class is the main class for the API Provider. + + This class is responsible for initializing the provider, setting up the HTTP session, + validating credentials, and managing identity information through direct API calls. + + Attributes: + _type (str): The provider type. + _session (requests.Session): The HTTP session for API calls. + _identity (APIIdentityInfo): The provider identity information. + _audit_config (dict): The audit configuration. + _mutelist (APIMutelist): The provider mutelist. + audit_metadata (Audit_Metadata): The audit metadata. + """ + + _type: str = "api_provider" + _session: Optional[requests.Session] + _identity: APIIdentityInfo + _audit_config: dict + _mutelist: APIMutelist + audit_metadata: Audit_Metadata + + def __init__( + self, + # Authentication parameters + username: str = None, + password: str = None, + tenant_id: str = None, + # Configuration + config_path: str = None, + config_content: dict = None, + mutelist_path: str = None, + mutelist_content: dict = None, + fixer_config: dict = None, + ): + """ + Initializes the APIProvider instance. + + Args: + username: The API username for authentication + password: The API password for authentication + tenant_id: The tenant ID for authentication + config_path: Path to the configuration file + config_content: Configuration content as dictionary + mutelist_path: Path to the mutelist file + mutelist_content: Mutelist content as dictionary + fixer_config: Fixer configuration dictionary + + Raises: + ValueError: If required authentication parameters are missing + """ + logger.info("Initializing APIProvider ...") + + # 1) Store argument values with environment variable fallback + self._username = username or os.getenv("YOUR_PROVIDER_USERNAME") + self._password = password or os.getenv("YOUR_PROVIDER_PASSWORD") + self._tenant_id = tenant_id or os.getenv("YOUR_PROVIDER_TENANT_ID") + + # Validate required parameters + if not all([self._username, self._password, self._tenant_id]): + raise ValueError("APIProvider requires username, password and tenant_id") + + # 2) Load audit_config, fixer_config, mutelist + self._fixer_config = fixer_config if fixer_config else {} + + if config_content: + self._audit_config = config_content + else: + if not config_path: + config_path = default_config_file_path + self._audit_config = load_and_validate_config_file(self._type, config_path) + + if mutelist_content: + self._mutelist = APIMutelist(mutelist_content=mutelist_content) + else: + if not mutelist_path: + mutelist_path = get_default_mute_file_path(self._type) + self._mutelist = APIMutelist(mutelist_path=mutelist_path) + + # 3) Initialize session/token + self._token = None + self._session = None + self.setup_session() + + # 4) Create identity object + self._identity = APIIdentityInfo( + tenant_id=self._tenant_id, + username=self._username, + ) + + Provider.set_global_provider(self) + + @property + def type(self) -> str: + """Returns the type of the provider.""" + return self._type + + @property + def identity(self) -> APIIdentityInfo: + """Returns the provider identity information.""" + return self._identity + + @property + def session(self) -> requests.Session: + """Returns the HTTP session for API calls.""" + return self._session + + @property + def audit_config(self) -> dict: + """Returns the audit configuration.""" + return self._audit_config + + @property + def fixer_config(self) -> dict: + """Returns the fixer configuration.""" + return self._fixer_config + + @property + def mutelist(self) -> APIMutelist: + """Returns the provider mutelist.""" + return self._mutelist + + def print_credentials(self) -> None: + """ + Display account information with color formatting. + + This method prints the provider credentials and account information + in a formatted way using colorama for better readability. + """ + from colorama import Style + + report_lines = [ + f" Username: {self._username}", + f" TenantID: {self._tenant_id}", + ] + report_title = f"{Style.BRIGHT}Using the {self._type.upper()} credentials below:{Style.RESET_ALL}" + print_boxes(report_lines, report_title) + + def setup_session(self) -> None: + """ + Implement API authentication method by calling the provider's authentication endpoint. + + This method performs the authentication flow to obtain an access token + and creates a requests.Session with the appropriate headers for API calls. + """ + # Example for a Keystone-like authentication + url = "https://api.your-provider.com/v2.0/tokens" + data = { + "auth": { + "tenantId": self._tenant_id, + "passwordCredentials": { + "username": self._username, + "password": self._password, + }, + } + } + + try: + response = requests.post(url, json=data, timeout=10) + if response.status_code == 200: + resp_json = response.json() + self._token = resp_json["access"]["token"]["id"] + + # Create session with authentication headers + sess = requests.Session() + sess.headers.update({ + "X-Auth-Token": self._token, + "Content-Type": "application/json" + }) + self._session = sess + logger.info("API token acquired successfully and session is set up.") + else: + logger.critical( + f"Failed to get token. Status: {response.status_code}, Body: {response.text}" + ) + raise ValueError("Failed to get API token") + except Exception as e: + logger.critical(f"[setup_session] Error: {e}") + raise e + + @staticmethod + def test_connection( + username: str, + password: str, + tenant_id: str, + raise_on_exception: bool = True, + ) -> Connection: + """ + Test connection to the API provider by performing: + 1) Authentication token request + 2) (Optional) a small test API call to confirm credentials are valid + + Args: + username: The API username + password: The API password + tenant_id: The tenant ID + raise_on_exception: If True, raise the caught exception; + if False, return Connection(error=exception). + + Returns: + Connection: Connection test result + """ + try: + # 1) Validate arguments + if not username or not password or not tenant_id: + error_msg = "API test_connection error: missing username/password/tenant_id" + logger.error(error_msg) + raise ValueError(error_msg) + + # 2) Request authentication token + token_url = "https://api.your-provider.com/v2.0/tokens" + data = { + "auth": { + "tenantId": tenant_id, + "passwordCredentials": { + "username": username, + "password": password, + }, + } + } + + resp = requests.post(token_url, json=data, timeout=10) + if resp.status_code != 200: + error_msg = f"Failed to get token. Status: {resp.status_code}, Body: {resp.text}" + logger.error(error_msg) + if raise_on_exception: + raise Exception(error_msg) + return Connection(error=Exception(error_msg)) + + # Success + token_json = resp.json() + api_token = token_json["access"]["token"]["id"] + logger.info("API test_connection: Successfully acquired token.") + + # 3) (Optional) Test API call to confirm credentials are valid + test_endpoint = f"https://api.your-provider.com/v2/{tenant_id}/test" + headers = { + "X-Auth-Token": api_token, + "Content-Type": "application/json", + } + + test_resp = requests.get(test_endpoint, headers=headers, timeout=10) + if test_resp.status_code == 200: + logger.info("API test_connection: Test call success. Credentials valid.") + return Connection(is_connected=True) + else: + error_msg = f"Test call failed. Status: {test_resp.status_code}, Body: {test_resp.text}" + logger.error(error_msg) + if raise_on_exception: + raise Exception(error_msg) + return Connection(error=Exception(error_msg)) + + except Exception as e: + logger.critical(f"{e.__class__.__name__}[{e.__traceback__.tb_lineno}]: {e}") + if raise_on_exception: + raise e + return Connection(error=e) + + @staticmethod + def validate_arguments(username: str, password: str, tenant_id: str) -> None: + """ + Ensures that username, password, and tenant_id are not empty. + + Args: + username: The username to validate + password: The password to validate + tenant_id: The tenant ID to validate + + Raises: + ValueError: If any required parameter is missing + """ + if not username or not password or not tenant_id: + raise ValueError("API Provider requires username, password and tenant_id.") +``` + +#### Step 3: Create Models + +**Explanation:** +Models define the data structures used by your API provider. They include identity information and API response structures. These models ensure type safety and consistent data handling across the provider. + +**File:** `prowler/providers//models.py` + +This step is common with SDK providers so you can follow the same pattern as [there](#step-3-create-models). + +#### Step 4: Implement Arguments + +**Explanation:** +Argument validation ensures that the API provider receives valid configuration parameters. This step is crucial for preventing runtime errors and providing clear error messages to users. + +**File:** `prowler/providers//lib/arguments/arguments.py` + +Arguments depends on the provider and not the type, so the pattern for this step is the same as the [SDK providers](#step-4-implement-arguments). + +#### Step 5: Implement Mutelist + +**Explanation:** +The mutelist functionality allows users to exclude specific resources or checks from the audit. This is useful for handling false positives or excluding resources that are intentionally configured differently. + +**File:** `prowler/providers//lib/mutelist/mutelist.py` + +The implementation of the mutelist is the same as the [SDK providers](#step-5-implement-mutelist). + +#### Step 6: Implement Regions + +**Explanation:** +Region management is essential for cloud providers that operate across multiple geographic locations. This component handles region validation and provides region-specific functionality. + + +Regions are optional, only if the provider has regions, for example Github does not have regions, but AWS does. + + +**File:** `prowler/providers//lib/regions/_regions.py` + +The implementation of the regions is the same as the [SDK providers](#step-6-implement-regions). + +#### Step 7: Create Custom Exceptions + +**Explanation:** +Custom exceptions provide specific error handling for API-related issues, making debugging and error reporting more effective. Prowler uses a structured exception system with error codes, messages, and remediation steps. + +**File:** `prowler/providers//exceptions/exceptions.py` + +```python +from prowler.exceptions.exceptions import ProwlerException + + +# Exceptions codes from 8000 to 8999 are reserved for API Provider exceptions (example numbers) +class APIProviderBaseException(ProwlerException): + """Base class for API Provider Errors.""" + + APIProvider_ERROR_CODES = { + (8000, "APIProviderCredentialsError"): { + "message": "API Provider credentials not found or invalid", + "remediation": "Check the API Provider API credentials and ensure they are properly set.", + }, + (8001, "APIProviderAuthenticationError"): { + "message": "API Provider authentication failed", + "remediation": "Check the API Provider API credentials and ensure they are valid.", + }, + (8002, "APIProviderSessionError"): { + "message": "API Provider session setup failed", + "remediation": "Check the session setup and ensure it is properly configured.", + }, + (8003, "APIProviderIdentityError"): { + "message": "API Provider identity setup failed", + "remediation": "Check credentials and ensure they are properly set up for API Provider.", + }, + (8004, "APIProviderAPIError"): { + "message": "API Provider API call failed", + "remediation": "Check the API request and ensure it is properly formatted.", + }, + (8005, "APIProviderRateLimitError"): { + "message": "API Provider API rate limit exceeded", + "remediation": "Reduce the number of API requests or wait before making more requests.", + }, + } + + def __init__(self, code, file=None, original_exception=None, message=None): + provider = "API Provider" + error_info = self.APIProvider_ERROR_CODES.get((code, self.__class__.__name__)) + if message: + error_info["message"] = message + super().__init__( + code=code, + source=provider, + file=file, + original_exception=original_exception, + error_info=error_info, + ) + + +class APIProviderCredentialsError(APIProviderBaseException): + """Exception for API Provider credentials errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8000, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderAuthenticationError(APIProviderBaseException): + """Exception for API Provider authentication errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8001, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderSessionError(APIProviderBaseException): + """Exception for API Provider session setup errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8002, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderIdentityError(APIProviderBaseException): + """Exception for API Provider identity setup errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8003, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderAPIError(APIProviderBaseException): + """Exception for API Provider API errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8004, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderRateLimitError(APIProviderBaseException): + """Exception for API Provider rate limit errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8005, + file=file, + original_exception=original_exception, + message=message, + ) +``` + +#### Step 8: Implement Service Base Class + +**Explanation:** +The service base class defines a common interface for all services in your provider, since they will inherit from it. It defines the client to make requests to, the audit configuration and the fixer configuration. + +**File:** `prowler/providers//lib/service/service.py` + +```python +from prowler.providers.._provider import Provider + +class APIProviderService(BaseService): + """ + Base service class for API Provider services. + + This class provides common functionality for all services + within the provider, including session management and error handling. + """ + + def __init__(self, provider: Provider): + """ + Initialize the service. + + Args: + provider: The provider instance + """ + self.client = provider.session.get_client(self.service_name) + self.audit_config = provider.audit_config + self.fixer_config = provider.fixer_config + self.session = provider.session + self.base_url = provider.session.base_url + self.auth = HTTPDigestAuth( + provider.session.public_key, + provider.session.private_key, + ) + self.headers = { + "Authorization": self.auth.encode(), + "Content-Type": "application/json", + } +``` + +#### Step 9: Register in CLI + +**Explanation:** +Add your provider to the available providers in the CLI. + +**File:** `prowler/lib/cli/parser.py` + +This step is the same as the [SDK providers](#step-9-register-in-cli). + +#### Step 10: Register in Main + +**Explanation:** +Main registration makes your provider discoverable by Prowler's core system. It's needed to add your provider to the output options and to the compliance evaluation. + +**File:** `prowler/__main__.py` + +This step is the same as the [SDK providers](#step-10-register-in-main). + +#### Step 11: Register in the list of providers + +**Explanation:** +This is needed to be able to use the provider in the generic checks. The provider must be registered in the `init_global_provider` method to handle CLI arguments and initialization. + +**File:** `prowler/providers/common/provider.py` + +This step is the same as the [SDK providers](#step-11-register-in-the-list-of-providers). + +#### Step 12: Add to Config + +**Explanation:** +Configuration registration ensures your API provider is recognized by Prowler's configuration system. This enables proper handling of provider-specific settings and defaults. + +**File:** `prowler/config/config.py` + +This step is the same as the [SDK providers](#step-12-add-to-config). + +#### Step 13: Create Compliance Files + +**Explanation:** +Compliance files define the security checks and standards that your provider supports. These JSON files map security controls to specific checks and provide remediation guidance. It's needed to create the folder with an init file to ensure the provider will work, however, adding different compliance files is optional. + +**Folder:** `prowler/compliance//` + +This step is the same as the [SDK providers](#step-13-create-compliance-files). + +#### Step 14: Add Output Support + +**Explanation:** +Output support ensures that your provider's results are properly formatted in Prowler's various output formats (CSV, JSON, HTML, etc.). This step integrates your provider with Prowler's reporting system. + +**File:** `prowler/lib/outputs/summary_table.py` + +This step is the same as the [SDK providers](#step-14-add-output-support). + +#### Step 15: Generate the HTML Report + +**Explanation:** +The HTML file is needed to be able to generate the HTML report. This step involves adding support for your provider in the HTML output generation system to ensure proper display of assessment summaries and findings. + +**File:** `prowler/lib/outputs/html/html.py` + +This step is the same as the [SDK providers](#step-15-generate-the-html-report). + +#### Step 16: Add the Check Report Model + +**Explanation:** +Add the provider to the generic models, this is needed to be able to use the provider in the generic checks. + +**File:** `prowler/providers/check/models.py` + +This step is the same as the [SDK providers](#step-16-add-the-check-report-model). + +#### Step 17: Create Tests + +**Explanation:** +Testing ensures that your API provider works correctly and maintains compatibility as Prowler evolves. Comprehensive tests cover authentication, session management, and API-specific functionality. + +**Folder:** `tests/providers//` + +```python +import pytest +from prowler.providers.api_provider.api_provider import APIProvider + +class TestAPIProvider: + """Test cases for APIProvider.""" + + def test_provider_initialization(self): + """Test provider initialization with valid credentials.""" + provider = APIProvider( + username="test_user", + password="test_password", + tenant_id="test_tenant" + ) + assert provider.type == "your_api_provider" + assert provider.identity is not None + assert provider.session is not None + + def test_connection_test(self): + """Test connection functionality.""" + result = APIProvider.test_connection( + username="test_user", + password="test_password", + tenant_id="test_tenant" + ) + # Add assertions based on expected behavior + + def test_argument_validation(self): + """Test argument validation.""" + from prowler.providers.api_provider.api_provider import ( + APIProvider + ) + + # Valid arguments + APIProvider.validate_arguments( + username="test_user", + password="test_password", + tenant_id="test_tenant" + ) + + # Invalid arguments + with pytest.raises(ValueError, match="requires username, password and tenant_id"): + APIProvider.validate_arguments("", "", "") + + def test_session_setup(self): + """Test session setup.""" + provider = APIProvider( + username="test_user", + password="test_password", + tenant_id="test_tenant" + ) + assert provider.session is not None + assert "X-Auth-Token" in provider.session.headers +``` + +#### Step 18: Update Documentation + +**Explanation:** +Documentation updates ensure that users can find information about your API provider in Prowler's documentation. This includes examples, configuration guides, and troubleshooting information. + +Update the provider documentation to include your new API provider in the examples and implementation guidance. + +--- + +### Tool/Wrapper Providers + +**Definition:** + +- Integrate third-party tools as libraries or subprocesses (e.g., Trivy for IaC). +- Examples: IaC (Trivy). + +**Typical Use Cases:** + +- Providers that require integration with external security tools. +- Tools that need to be executed as subprocesses or imported as libraries. +- Providers that require specific tool configurations and argument mapping. +- Legacy systems or tools that don't have direct API access. + +**Key Characteristics:** + +- No session/identity management required (tool handles this internally). +- Arguments: specific to the tool, but for example: `scan_path`, `frameworks`, `exclude_path`, `scan_repository_url`, etc. +- Outputs: Tool-specific output formats that need to be parsed and converted. +- Tool execution and output parsing. +- Configuration file mapping and argument translation. + +**Implementation Details:** + +- Tool providers typically execute external tools as subprocesses (e.g., `pwsh` or `trivy` command). +- They require mapping between Prowler's interface and the tool's arguments. +- Output parsing and conversion to Prowler's standard format is crucial. +- Tool-specific configuration files and validation. +- Repository cloning and temporary file management for remote scans (if needed). + +**Note:** This guide provides a general framework for integrating any external tool. The specific implementation details (like repository cloning, authentication tokens, etc.) will depend on your particular tool's requirements. The core pattern is: integrate with Prowler's CLI, execute your tool via subprocess, and parse the output into Prowler's format. + +--- + +### Implementation Guide for Tool/Wrapper Providers + +#### Step 1: Create the Provider Structure + +**Explanation:** +Tool/Wrapper providers require a specific folder structure to organize tool integration, configuration, and service management. This structure follows Prowler's conventions and ensures proper integration with the CLI and API. + +**Required Structure:** + +``` +prowler/providers// +├── __init__.py +├── _provider.py +├── models.py +└── lib/ + ├── __init__.py + └── arguments/ + ├── __init__.py + └── arguments.py +``` + +**Key Components:** + +- **`_provider.py`**: Main provider class with tool integration +- **`models.py`**: Data structures for tool output and configuration +- **`lib/arguments/`**: CLI argument validation and parsing + +#### Step 2: Implement the Provider Class + +**Explanation:** +The provider class is the core component that handles tool integration, execution, and output parsing. It inherits from Prowler's base Provider class and implements tool-specific execution flows using subprocesses or library calls. + +**File:** `prowler/providers//_provider.py` + +```python +import json +import subprocess +import sys +from typing import List + +from colorama import Fore, Style + +from prowler.config.config import ( + default_config_file_path, + load_and_validate_config_file, +) +from prowler.lib.check.models import CheckReportYourTool +from prowler.lib.logger import logger +from prowler.lib.utils.utils import print_boxes +from prowler.providers.common.models import Audit_Metadata +from prowler.providers.common.provider import Provider + +class ToolProvider(Provider): + """ + ToolProvider class is the main class for the Your Tool Provider. + + This class is responsible for initializing the provider, executing the external tool, + parsing tool output, and converting results to Prowler's standard format. + + Attributes: + _type (str): The provider type. + _session: Not used for tool providers. + _identity (str): Simple identity for tool providers. + _audit_config (dict): The audit configuration. + audit_metadata (Audit_Metadata): The audit metadata. + """ + + _type: str = "your_tool_provider" + audit_metadata: Audit_Metadata + + def __init__( + self, + # Tool-specific parameters + scan_path: str = ".", + tool_specific_arg: str = "default_value", + exclude_path: list[str] = [], + # Configuration + config_path: str = None, + config_content: dict = None, + fixer_config: dict = {}, + # Authentication (if needed for your tool) + auth_token: str = None, + auth_username: str = None, + ): + """ + Initializes the ToolProvider instance. + + Args: + scan_path: Path to the folder containing files to scan + tool_specific_arg: Tool-specific argument for your external tool + exclude_path: List of paths to exclude from scan + config_path: Path to the configuration file + config_content: Configuration content as dictionary + fixer_config: Fixer configuration dictionary + auth_token: Authentication token for your tool (if needed) + auth_username: Username for your tool (if needed) + + Raises: + ValueError: If required parameters are missing + """ + logger.info("Instantiating YourTool Provider...") + + # Store tool-specific parameters + self.scan_path = scan_path + self.tool_specific_arg = tool_specific_arg + self.exclude_path = exclude_path + self.region = "global" + self.audited_account = "local-tool" + self._session = None + self._identity = "prowler" + self._auth_method = "No auth" + + # Handle tool authentication if needed + if auth_token: + self.auth_token = auth_token + self._auth_method = "Token" + logger.info("Using token for tool authentication") + elif auth_username: + self.auth_username = auth_username + self._auth_method = "Username" + logger.info("Using username for tool authentication") + logger.info("Using username for tool authentication") + else: + logger.debug("No authentication method provided; proceeding without authentication.") + + # Audit Config + if config_content: + self._audit_config = config_content + else: + if not config_path: + config_path = default_config_file_path + self._audit_config = load_and_validate_config_file(self._type, config_path) + + # Fixer Config + self._fixer_config = fixer_config + + # Mutelist (not needed for tool providers since tools have their own mutelist logic) + self._mutelist = None + + Provider.set_global_provider(self) + + @property + def auth_method(self): + """Returns the authentication method used.""" + return self._auth_method + + @property + def type(self): + """Returns the type of the provider.""" + return self._type + + @property + def identity(self): + """Returns the provider identity.""" + return self._identity + + @property + def session(self): + """Returns the session (not used for tool providers).""" + return self._session + + @property + def audit_config(self): + """Returns the audit configuration.""" + return self._audit_config + + @property + def fixer_config(self): + """Returns the fixer configuration.""" + return self._fixer_config + + def setup_session(self): + """Tool providers don't need a session since they use external tools directly""" + return None + + def _process_check(self, finding: dict, check: dict, status: str) -> CheckReportYourTool: + """ + Process a single check (failed or passed) and create a CheckReportYourTool object. + + Args: + finding: The finding object from tool output + check: The individual check data + status: The status of the check ("FAIL", "PASS", or "MUTED") + + Returns: + CheckReportYourTool: The processed check report + """ + try: + metadata_dict = { + "Provider": "your_tool_provider", + "CheckID": check.get("check_id", ""), + "CheckTitle": check.get("check_name", ""), + "CheckType": ["Your Tool Provider"], + "ServiceName": finding["check_type"], + "SubServiceName": "", + "ResourceIdTemplate": "", + "Severity": ( + check.get("severity", "low").lower() + if check.get("severity") + else "low" + ), + "ResourceType": "your_tool", + "Description": check.get("check_name", ""), + "Risk": "", + "RelatedUrl": ( + check.get("guideline", "") if check.get("guideline") else "" + ), + "Remediation": { + "Code": { + "NativeIaC": "", + "Terraform": "", + "CLI": "", + "Other": "", + }, + "Recommendation": { + "Text": "", + "Url": ( + check.get("guideline", "") if check.get("guideline") else "" + ), + }, + }, + "Categories": [], + "DependsOn": [], + "RelatedTo": [], + "Notes": "", + } + + # Convert metadata dict to JSON string + metadata = json.dumps(metadata_dict) + + report = CheckReportYourTool(metadata=metadata, finding=check) + report.status = status + report.resource_tags = check.get("entity_tags", {}) + report.status_extended = check.get("check_name", "") + if status == "MUTED": + report.muted = True + return report + except Exception as error: + logger.critical( + f"{error.__class__.__name__}:{error.__traceback__.tb_lineno} -- {error}" + ) + sys.exit(1) + + def run(self) -> List[CheckReportYourTool]: + """ + Main execution method that handles tool execution. + + Returns: + List[CheckReportYourTool]: List of check reports from the tool scan + """ + return self.run_scan(self.scan_path, self.exclude_path) + + def run_scan( + self, directory: str, exclude_path: list[str] + ) -> List[CheckReportYourTool]: + """ + Execute the external tool and parse its output. + + Args: + directory: Directory to scan + frameworks: List of frameworks to scan + exclude_path: List of paths to exclude + + Returns: + List[CheckReportYourTool]: List of check reports + """ + try: + logger.info(f"Running YourTool scan on {directory} ...") + + # Build the tool command + tool_command = [ + "your_tool_command", + # Add your tool-specific arguments here, this are just examples + "-d", + directory, + "-o", + "json", + "-f", + ",".join(frameworks), + ] + if exclude_path: + tool_command.extend(["--skip-path", ",".join(exclude_path)]) + + # Run the tool with JSON output + process = subprocess.run( + tool_command, + capture_output=True, + text=True, + ) + + # Log tool's error output if any + if process.stderr: + logger.error(process.stderr) + + try: + output = json.loads(process.stdout) + if not output: + logger.warning("No findings returned from YourTool scan") + return [] + except Exception as error: + logger.critical( + f"{error.__class__.__name__}:{error.__traceback__.tb_lineno} -- {error}" + ) + sys.exit(1) + + reports = [] + + # If only one framework has findings, the output is a dict, otherwise it's a list of dicts + if isinstance(output, dict): + output = [output] + + # Process all frameworks findings + for finding in output: + results = finding.get("results", {}) + + # Process failed checks + failed_checks = results.get("failed_checks", []) + for failed_check in failed_checks: + report = self._process_check(finding, failed_check, "FAIL") + reports.append(report) + + # Process passed checks + passed_checks = results.get("passed_checks", []) + for passed_check in passed_checks: + report = self._process_check(finding, passed_check, "PASS") + reports.append(report) + + # Process skipped checks (muted) + skipped_checks = results.get("skipped_checks", []) + for skipped_check in skipped_checks: + report = self._process_check(finding, skipped_check, "MUTED") + reports.append(report) + + return reports + + except Exception as error: + if "No such file or directory: 'your_tool_command'" in str(error): + logger.critical("Please, install your_tool using 'pip install your_tool'") + sys.exit(1) + logger.critical( + f"{error.__class__.__name__}:{error.__traceback__.tb_lineno} -- {error}" + ) + sys.exit(1) + + def print_credentials(self): + """ + Display scan information with color formatting. + + This method prints the tool scan information in a formatted way + using colorama for better readability. + """ + if self.scan_repository_url: + report_title = ( + f"{Style.BRIGHT}Scanning remote repository:{Style.RESET_ALL}" + ) + report_lines = [ + f"Repository: {Fore.YELLOW}{self.scan_repository_url}{Style.RESET_ALL}", + ] + else: + report_title = ( + f"{Style.BRIGHT}Scanning local directory:{Style.RESET_ALL}" + ) + report_lines = [ + f"Directory: {Fore.YELLOW}{self.scan_path}{Style.RESET_ALL}", + ] + + if self.exclude_path: + report_lines.append( + f"Excluded paths: {Fore.YELLOW}{', '.join(self.exclude_path)}{Style.RESET_ALL}" + ) + + report_lines.append( + f"Frameworks: {Fore.YELLOW}{', '.join(self.frameworks)}{Style.RESET_ALL}" + ) + + report_lines.append( + f"Authentication method: {Fore.YELLOW}{self.auth_method}{Style.RESET_ALL}" + ) + + print_boxes(report_lines, report_title) +``` + +#### Step 3: Create Models + +**Explanation:** +Models define the data structures used by your tool provider. They include output options and tool-specific configurations. These models ensure type safety and consistent data handling across the provider. + +**File:** `prowler/providers//models.py` + +```python +from prowler.config.config import output_file_timestamp +from prowler.providers.common.models import ProviderOutputOptions + +class YourToolOutputOptions(ProviderOutputOptions): + """ + YourToolOutputOptions overrides ProviderOutputOptions for tool-specific output logic. + For example, generating a filename that includes the tool name. + + Attributes inherited from ProviderOutputOptions: + - output_filename (str): The base filename used for generated reports. + - output_directory (str): The directory to store the output files. + - ... see ProviderOutputOptions for more details. + + Methods: + - __init__: Customizes the output filename logic for the tool provider. + """ + + def __init__(self, arguments, bulk_checks_metadata): + super().__init__(arguments, bulk_checks_metadata) + + # If --output-filename is not specified, build a default name. + if not getattr(arguments, "output_filename", None): + self.output_filename = f"prowler-output-your_tool-{output_file_timestamp}" + # If --output-filename was explicitly given, respect that + else: + self.output_filename = arguments.output_filename +``` + +#### Step 4: Implement Arguments + +**Explanation:** +Argument validation ensures that the tool provider receives valid configuration parameters. This step is crucial for preventing runtime errors and providing clear error messages to users. + +**File:** `prowler/providers//lib/arguments/arguments.py` + +```python +# Add your tool-specific choices if needed +TOOL_SPECIFIC_CHOICES = [ + "option1", + "option2", + "option3", + # Add your tool's supported options +] + +def init_parser(self): + """Init the Provider CLI parser""" + _parser = self.subparsers.add_parser( + "", parents=[self.common_providers_parser], help=" Provider" + ) + + # Scan Path + _scan_subparser = _parser.add_argument_group("Scan Path") + _scan_subparser.add_argument( + "--scan-path", + "-P", + dest="scan_path", + default=".", + help="Path to the folder containing your files to scan. Default: current directory.", + ) + + _scan_subparser.add_argument( + "--tool-specific-arg", + dest="tool_specific_arg", + default="default_value", + choices=TOOL_SPECIFIC_CHOICES, + help="Tool-specific argument for your external tool. Default: default_value", + ) + + _scan_subparser.add_argument( + "--exclude-path", + dest="exclude_path", + nargs="+", + default=[], + help="Comma-separated list of paths to exclude from the scan. Default: none", + ) + + # Authentication (if needed for your tool) + _scan_subparser.add_argument( + "--auth-token", + dest="auth_token", + nargs="?", + default=None, + help="Authentication token for your tool. If not provided, will use YOUR_TOOL_AUTH_TOKEN env var.", + ) + _scan_subparser.add_argument( + "--auth-username", + dest="auth_username", + nargs="?", + default=None, + help="Username for your tool authentication. If not provided, will use YOUR_TOOL_AUTH_USERNAME env var.", + ) + +def validate_arguments(arguments): + """ + Validate tool-specific arguments. + + Args: + arguments: The parsed arguments + + Returns: + tuple: (is_valid, error_message) + """ + scan_path = getattr(arguments, "scan_path", None) + scan_repository_url = getattr(arguments, "scan_repository_url", None) + + if scan_path and scan_repository_url: + # If scan_path is set to default ("."), allow scan_repository_url + if scan_path != ".": + return ( + False, + "--scan-path (-P) and --scan-repository-url (-R) are mutually exclusive. Please specify only one.", + ) + return (True, "") +``` + +#### Step 5: Register in CLI + +**Explanation:** +Add your provider to the available providers in the CLI. + +**File:** `prowler/lib/cli/parser.py` + +This step is the same as the [SDK providers](#step-9-register-in-cli). + +#### Step 6: Register in Main + +**Explanation:** +Main registration makes your provider discoverable by Prowler's core system. It's needed to add your provider to the output options and to the compliance evaluation. + +**File:** `prowler/__main__.py` + +This step is the same as the [SDK providers](#step-10-register-in-main). + +#### Step 7: Register in the list of providers + +**Explanation:** +This is needed to be able to use the provider in the generic checks. The provider must be registered in the `init_global_provider` method to handle CLI arguments and initialization. + +**File:** `prowler/providers/common/provider.py` + +This step is the same as the [SDK providers](#step-11-register-in-the-list-of-providers). + +#### Step 8: Add to Config + +**Explanation:** +Configuration registration ensures your tool provider is recognized by Prowler's configuration system. This enables proper handling of provider-specific settings and defaults. + +**File:** `prowler/config/config.py` + +This step is the same as the [SDK providers](#step-12-add-to-config). + +In some cases, you may need to create a new configuration file for your provider, for example, the AWS one that is inside `prowler/providers/aws/config.py`. + +#### Step 9: Create Compliance Files + +**Explanation:** +Compliance files define the security checks and standards that your provider supports. These JSON files map security controls to specific checks and provide remediation guidance. It's needed to create the folder with an init file to ensure the provider will work, however, adding different compliance files is optional. + +**Folder:** `prowler/compliance//` + +This step is the same as the [SDK providers](#step-13-create-compliance-files). + +#### Step 10: Add Output Support + +**Explanation:** +Output support ensures that your provider's results are properly formatted in Prowler's various output formats (CSV, JSON, HTML, etc.). This step integrates your provider with Prowler's reporting system. + +**File:** `prowler/lib/outputs/summary_table.py` + +This step is the same as the [SDK providers](#step-14-add-output-support). + +#### Step 11: Generate the HTML Report + +**Explanation:** +The HTML file is needed to be able to generate the HTML report. This step involves adding support for your provider in the HTML output generation system to ensure proper display of assessment summaries and findings. + +**File:** `prowler/lib/outputs/html/html.py` + +This step is the same as the [SDK providers](#step-15-generate-the-html-report). + +#### Step 12: Add the Check Report Model + +**Explanation:** +Add the provider to the generic models, this is needed to be able to use the provider in the generic checks. + +**File:** `prowler/providers/check/models.py` + +This step is the same as the [SDK providers](#step-16-add-the-check-report-model). + +#### Step 13: Create Tests + +**Explanation:** +Testing ensures that your tool provider works correctly and maintains compatibility as Prowler evolves. Comprehensive tests cover tool execution, output parsing, and provider-specific functionality. + +**Folder:** `tests/providers//` + +```python +import pytest +import tempfile +import os +from prowler.providers.your_tool_provider.your_tool_provider import ToolProvider + +class TestToolProvider: + """Test cases for ToolProvider.""" + + def test_provider_initialization(self): + """Test provider initialization with valid parameters.""" + provider = ToolProvider( + scan_path=".", + frameworks=["framework1"] + ) + assert provider.type == "your_tool_provider" + assert provider.identity == "prowler" + assert provider.scan_path == "." + + def test_tool_execution(self): + """Test tool execution and output parsing.""" + provider = ToolProvider(scan_path=".") + # Mock the subprocess call and test output parsing + # This will depend on your specific tool's output format + + def test_argument_validation(self): + """Test argument validation.""" + from prowler.providers.your_tool_provider.lib.arguments.arguments import ( + validate_arguments + ) + + # Valid arguments + class MockArgs: + scan_path = "." + tool_specific_arg = "value" + + is_valid, message = validate_arguments(MockArgs()) + assert is_valid is True + + # Add more test cases as needed for your specific tool provider + + def test_print_credentials(self): + """Test print_credentials method.""" + provider = ToolProvider( + scan_path="/test/path", + frameworks=["framework1"] + ) + # This should not raise any exceptions + provider.print_credentials() +``` + +#### Step 14: Update Documentation + +**Explanation:** +Documentation updates ensure that users can find information about your tool provider in Prowler's documentation. This includes examples, configuration guides, and troubleshooting information. + +Update the provider documentation to include your new tool provider in the examples and implementation guidance. + +--- + + +## Step 2: Integrate the Provider in the API + +This step is required only if you want your provider to be available in the API and UI. The API integration involves several components: + +### 2.1. Backend API Models + +**Location:** `api/src/backend/api/models.py` + +Add your provider to the `ProviderChoices` enum and implement UID validation: + +```python +class ProviderChoices(models.TextChoices): + AWS = "aws", "AWS" + AZURE = "azure", "Azure" + GCP = "gcp", "GCP" + KUBERNETES = "kubernetes", "Kubernetes" + M365 = "m365", "Microsoft 365" + GITHUB = "github", "GitHub" + NHN = "nhn", "NHN Cloud" + IAC = "iac", "Infrastructure as Code" + YOUR_PROVIDER = "your_provider", "Your Provider" # Add your provider here + +@staticmethod +def validate_your_provider_uid(value): + """Validate your provider UID format.""" + if not re.match(r"^your-regex-pattern$", value): + raise ModelValidationError( + detail="Your provider UID must follow the specified format.", + code="your-provider-uid", + pointer="/data/attributes/uid", + ) +``` + +**Provider Model:** +The `Provider` model already exists and supports all provider types. Ensure your provider type is included in the choices. + +### 2.2. Add the provider to the Provider Choices + +Update the `return_prowler_provider` function to include your provider. This function is crucial for the API to instantiate the correct provider class. + +**File:** `api/src/backend/api/utils.py` + +```python +from prowler.providers.your_provider.your_provider import YourProvider # Add your import + +def return_prowler_provider( + provider: Provider, +) -> [ + AwsProvider + | AzureProvider + | GcpProvider + | GithubProvider + | KubernetesProvider + | M365Provider + | YourProvider # Add your provider to the return type annotation +]: + """Return the Prowler provider class based on the given provider type.""" + match provider.provider: + case Provider.ProviderChoices.AWS.value: + prowler_provider = AwsProvider + case Provider.ProviderChoices.AZURE.value: + prowler_provider = AzureProvider + case Provider.ProviderChoices.GCP.value: + prowler_provider = GcpProvider + case Provider.ProviderChoices.KUBERNETES.value: + prowler_provider = KubernetesProvider + case Provider.ProviderChoices.M365.value: + prowler_provider = M365Provider + case Provider.ProviderChoices.GITHUB.value: + prowler_provider = GithubProvider + case Provider.ProviderChoices.YOUR_PROVIDER.value: # Add your provider here + prowler_provider = YourProvider + case _: + raise ValueError(f"Provider type {provider.provider} not supported") + return prowler_provider +``` + +**Also update the `initialize_prowler_provider` function:** + +```python +def initialize_prowler_provider( + provider: Provider, + mutelist_processor: Processor | None = None, +) -> ( + AwsProvider + | AzureProvider + | GcpProvider + | GithubProvider + | KubernetesProvider + | M365Provider + | YourProvider # Add your provider to the return type annotation +): + """Initialize a Prowler provider instance based on the given provider type.""" + prowler_provider = return_prowler_provider(provider) + prowler_provider_kwargs = get_prowler_provider_kwargs(provider, mutelist_processor) + return prowler_provider(**prowler_provider_kwargs) +``` + +**Note:** The `match` statement requires Python 3.10+. If you're using an older version, you can use traditional `if-elif` statements instead. + +### 2.3. API Serializers + +Create or update serializers for your provider. You'll need to add your provider to the validation logic: + +**File:** `api/src/backend/api/v1/serializers.py` + +```python +def validate_secret_based_on_provider(provider_type, secret): + """Validate provider-specific secrets.""" + if provider_type == Provider.ProviderChoices.AWS.value: + serializer = AWSProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.AZURE.value: + serializer = AzureProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.GCP.value: + serializer = GCPProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.YOUR_PROVIDER.value: # Add your provider here + serializer = YourProviderSecret(data=secret) + # ... other providers + + if serializer.is_valid(): + return serializer.validated_data + else: + raise serializers.ValidationError(serializer.errors) + +class YourProviderSecret(serializers.Serializer): + """Serializer for your provider credentials.""" + your_auth_field = serializers.CharField(required=True) + your_optional_field = serializers.CharField(required=False) + + class Meta: + resource_name = "provider-secrets" +``` + +Also update the providers included in the serializer: + +**File:** `api/src/backend/api/v1/serializer_utils/providers.py` + +```python +@extend_schema_field( + { + "oneOf": [ + # ... existing provider schemas ... + { + "type": "object", + "title": "Your Provider Credentials", + "properties": { + "your_auth_field": { + "type": "string", + "description": "Your provider authentication field description.", + }, + "your_optional_field": { + "type": "string", + "description": "Optional field for your provider (if applicable).", + }, + "your_required_field": { + "type": "string", + "description": "Required field for your provider authentication.", + } + }, + "required": ["your_auth_field", "your_required_field"] + }, + # ... other existing schemas ... + ] + } +) +``` + +### 2.4. Database Migration + +Create a new migration to add your provider to the database. This is crucial for the API to recognize your provider type. + +**File:** `api/src/backend/api/migrations/XXXX_your_provider.py` + +```python +# Generated by Django X.X.X on YYYY-MM-DD + +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "previous_migration_name"), # Update this to the latest migration + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("your_provider", "Your Provider"), # Add your provider here + ], + default="aws", + ), + ), + migrations.RunSQL( + "ALTER TYPE provider ADD VALUE IF NOT EXISTS 'your_provider';", + reverse_sql=migrations.RunSQL.noop, + ), + ] +``` + +**Important Notes:** + +- **Migration Number**: Use the next sequential number (e.g., if latest is 0044, use 0045) +- **Dependencies**: Update the `dependencies` list to point to the most recent migration +- **Choices Array**: Add your provider to the `choices` array with proper display name +- **SQL Operation**: The `RunSQL` operation adds your provider to the PostgreSQL enum type +- **Reverse SQL**: Use `migrations.RunSQL.noop` since adding enum values cannot be easily reversed + +**Migration Naming Convention:** + +- Format: `XXXX_your_provider.py` (e.g., `0045_your_provider.py`) +- Use descriptive names that indicate what the migration does +- Follow the existing pattern in the migrations folder + +### 2.5. Update the V1 Yaml + +Update the OpenAPI specification (`v1.yaml`) to include your provider in all relevant endpoints and schemas. This is crucial for API documentation and client generation. + +**File:** `api/src/backend/api/specs/v1.yaml` + +#### 2.5.1. Provider Enum Values + +Add your provider to the provider enum in the Provider schema: + +```yaml +# Around line 12150 in v1.yaml +Provider: + type: object + properties: + attributes: + properties: + provider: + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + - github + - your_provider # Add your provider here + type: string + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `your_provider` - Your Provider # Add your provider here +``` + +#### 2.5.2. Provider Credential Schemas + +Add your provider's credential schema to the integration configuration. This defines how your provider's credentials are structured: + +```yaml +# Around line 11100 in v1.yaml, in the integration configuration +- type: object + title: Your Provider Credentials # Add your provider here + properties: + your_auth_field: + type: string + description: Your provider authentication field description. + your_optional_field: + type: string + description: Optional field for your provider (if applicable). + your_required_field: + type: string + description: Required field for your provider authentication. + required: + - your_auth_field + - your_required_field +``` + +#### 2.5.3. Example Provider Schemas + +Here are examples of how existing providers are documented: + +**AWS Provider:** +```yaml +- type: object + title: AWS Static Credentials + properties: + aws_access_key_id: + type: string + description: The AWS access key ID. + aws_secret_access_key: + type: string + description: The AWS secret access key. + required: + - aws_access_key_id + - aws_secret_access_key + +- type: object + title: AWS Assume Role + properties: + role_arn: + type: string + description: The Amazon Resource Name (ARN) of the role to assume. + external_id: + type: string + description: An identifier to enhance security for role assumption. + required: + - role_arn + - external_id +``` + +**GitHub Provider:** +```yaml +- type: object + title: GitHub Personal Access Token + properties: + personal_access_token: + type: string + description: GitHub personal access token for authentication. + required: + - personal_access_token + +- type: object + title: GitHub OAuth App Token + properties: + oauth_app_token: + type: string + description: GitHub OAuth App token for authentication. + required: + - oauth_app_token +``` + +**M365 Provider:** +```yaml +- type: object + title: M365 Static Credentials + properties: + client_id: + type: string + description: The Azure application (client) ID for authentication in Azure AD. + client_secret: + type: string + description: The client secret associated with the application (client) ID. + tenant_id: + type: string + description: The Azure tenant ID, representing the directory where the application is registered. + user: + type: email + description: User microsoft email address. + password: + type: string + description: User password. + required: + - client_id + - client_secret + - tenant_id + - user + - password +``` + +#### 2.5.4. Important Notes + +- **Position**: Add your schema in the `oneOf` array alongside existing providers +- **Structure**: Follow the exact pattern of other providers (title, properties, required fields) +- **Descriptions**: Provide clear, helpful descriptions for each field +- **Required Fields**: Specify which fields are mandatory in the `required` array +- **Field Types**: Use appropriate JSON schema types (`string`, `integer`, `boolean`, `email`, etc.) +- **Validation**: Add any field-specific validation patterns or constraints +- **Documentation**: Ensure your provider appears in the generated API documentation + +### 2.6. Testing API Integration + +Create tests for your provider: + +**Location:** `api/src/backend/api/tests/` + +```python +class YourProviderAPITestCase(APITestCase): + def setUp(self): + self.user = User.objects.create_user(username='testuser', password='testpass') + self.client.force_authenticate(user=self.user) + + def test_create_your_provider(self): + data = { + 'provider': 'your_provider', + 'uid': 'valid-uid-123', + 'alias': 'Test Account' + } + response = self.client.post('/api/v1/providers/', data) + self.assertEqual(response.status_code, 201) + self.assertEqual(response.data['provider'], 'your_provider') + + def test_your_provider_uid_validation(self): + """Test UID validation for your provider.""" + invalid_uids = [ + 'invalid@uid', + '-invalid-start', + 'a' * 40, # Too long + ] + + for invalid_uid in invalid_uids: + data = { + 'provider': 'your_provider', + 'uid': invalid_uid, + 'alias': 'Test' + } + response = self.client.post('/api/v1/providers/', data) + self.assertEqual(response.status_code, 400) + self.assertIn('your-provider-uid', str(response.data)) + + def test_add_your_provider_credentials(self): + # Create provider first + provider = Provider.objects.create( + user=self.user, + provider='your_provider', + uid='valid-uid-123' + ) + + # Add credentials + credentials_data = { + 'secret_type': 'your_provider_credentials', + 'secret': { + 'your_auth_field': 'auth_value', + 'your_optional_field': 'optional_value' + }, + 'provider': provider.id + } + response = self.client.post('/api/v1/providers/secrets/', credentials_data) + self.assertEqual(response.status_code, 201) +``` + +#### 2.6.1. Add your mocked provider to the tests + +If needed, add your mocked provider to the tests config file so you can use it on the tests. + +**File:** `api/src/backend/conftest.py` + +```python +@pytest.fixture +def providers_fixture(tenants_fixture): + tenant, *_ = tenants_fixture + providerX = Provider.objects.create( + provider="your_provider", + uid="your_uid", + alias="your_alias", + tenant_id=tenant.id, + ) + return provider1, provider2, provider3, ... providerX +``` + +### 2.7. Compliance and Output Support + +Add your provider to the compliance export functionality: + +**File:** `api/src/backend/tasks/jobs/export.py` + +```python +COMPLIANCE_FRAMEWORKS = { + "aws": [...], + "azure": [...], + "gcp": [...], + "kubernetes": [...], + "m365": [...], + "github": [...], + "your_provider": [ # Add your provider here + (lambda name: name.startswith("cis_"), YourProviderCIS), + (lambda name: name.startswith("iso27001_"), YourProviderISO27001), + ], +} +``` + +If your provider has specific fields, add them to the finding transformation: + +**File:** `prowler/lib/outputs/finding.py` + +```python +def transform_api_finding(cls, finding, provider) -> "Finding": + # ... existing code ... + + # Your provider specific field + if provider.type == "your_provider": + finding.your_field = resource.your_field + + # ... rest of the code ... +``` + +### 2.8. API Endpoints + +Your provider will be available through these endpoints: + +- `GET /api/v1/providers/` - List all providers +- `POST /api/v1/providers/` - Create a new provider +- `GET /api/v1/providers/{id}/` - Get provider details +- `PUT /api/v1/providers/{id}/` - Update provider +- `DELETE /api/v1/providers/{id}/` - Delete provider +- `POST /api/v1/providers/secrets/` - Add provider credentials + +### 2.9. Update the provider if needed + +Depending on your provider's authentication requirements, you may need to add new authentication methods that are compatible with the API. This involves updating the provider class to support additional credential types beyond the basic ones. + +#### 2.9.1. Adding New Authentication Methods + +If your provider requires specific authentication methods, you'll need to: + +1. **Update the provider constructor** to accept new authentication parameters +2. **Extend the credential handling** to support the new authentication method +3. **Update the API serializers** to include the new credential fields +4. **Modify the OpenAPI specification** to document the new authentication schema + +#### 2.9.2. Example: GitHub Provider Authentication Methods + +The GitHub provider demonstrates how to implement multiple authentication methods: + +```python +# In prowler/providers/github/github_provider.py +def __init__( + self, + # Authentication methods + personal_access_token: str = "", + oauth_app_token: str = "", + github_app_key: str = "", + #Needed for the API integration + github_app_key_content: str = "", + github_app_id: int = 0, + # Provider configuration + config_path: str = None, + # ... other parameters +): + """ + Initialize GitHub provider. + + Args: + personal_access_token (str): GitHub personal access token. + oauth_app_token (str): GitHub OAuth App token. + github_app_key (str): GitHub App key. + github_app_key_content (str): GitHub App key content. + github_app_id (int): GitHub App ID. + config_path (str): Path to the audit configuration file. + # ... other parameters + """ + super().__init__( + personal_access_token, + oauth_app_token, + github_app_id, + github_app_key, + github_app_key_content, + ) +``` + +--- + +## Step 3: Integrate the Provider in the UI + +TBD + +--- + +## Provider Implementation Guidance + +Use existing providers as templates, this will help you to understand better the structure and the implementation will be easier: + +- [AWS (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/aws/aws_provider.py) +- [Azure (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/azure/azure_provider.py) +- [GCP (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/gcp/gcp_provider.py) +- [Kubernetes (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/kubernetes/kubernetes_provider.py) +- [M365 (SDK/Wrapper)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/m365/m365_provider.py) +- [GitHub (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/github/github_provider.py) +- [NHN (API)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/nhn/nhn_provider.py) +- [IAC (Tool)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/iac/iac_provider.py) +- [MongoDB Atlas](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/mongodbatlas/mongodbatlas_provider.py) + +--- + +## Best Practices + +- **Code Quality & Documentation** + + - **Comprehensive Docstrings**: Every class, method, and function should have detailed docstrings following Prowler's format + ```python + def method_name(self, param: str) -> str: + """ + Brief description of what the method does. + + Args: + param: Description of the parameter + + Returns: + Description of the return value + + Raises: + ExceptionType: When and why this exception occurs + """ + ``` + + - **Type Hints**: Use type hints for all function parameters and return values + ```python + from typing import Optional, List, Dict, Any + ``` + + - **Logging**: Implement proper logging using Prowler's logger + ```python + from prowler.lib.logger import logger + + logger.info("Operation completed successfully") + logger.warning("Something to be aware of") + logger.error("Something went wrong") + logger.critical("Critical error that may cause failure") + ``` + +- **Error Handling & Validation** + + - **Custom Exceptions**: Create provider-specific exceptions for better error handling + - **Input Validation**: Validate all inputs and provide clear error messages + - **Graceful Degradation**: Handle errors gracefully without crashing the entire scan + - **Raise on Exception**: Use `raise_on_exception` parameter for test methods + +- **Testing & Quality Assurance** + + - **Comprehensive Test Coverage**: Aim for >80% test coverage + - **Test Naming**: Use descriptive test names: `test_method_name_scenario` + - **Test Organization**: Group related tests in test classes + - **Mock External Dependencies**: Mock external API calls and services + - **Test Edge Cases**: Include tests for error conditions and edge cases + - **End-to-End Testing**: Test the provider on real infrastructure + +- **Performance & Security** + + - **Session Management**: Reuse sessions when possible, don't create new ones unnecessarily + - **Rate Limiting**: Implement rate limiting for API calls to avoid hitting limits + - **Resource Cleanup**: Ensure proper cleanup of temporary resources + - **Authentication Security**: Never log sensitive credentials or tokens + +- **Code Organization** + + - **Single Responsibility**: Each method should have one clear purpose + - **Consistent Naming**: Follow Prowler's naming conventions + - **Modular Design**: Break complex functionality into smaller, testable methods + - **Configuration Management**: Use configuration files for provider-specific settings + +- **Documentation & Maintenance** + + - **README Updates**: Update provider-specific documentation + - **Changelog**: Document changes and new features + - **Examples**: Provide usage examples and common scenarios + - **Troubleshooting**: Include common issues and solutions + - **Documentation**: Update the provider documentation to include your new tool provider in the examples and implementation guidance. + +- **Integration Standards** + + - **CLI Consistency**: Follow Prowler's CLI argument patterns + - **Output Format**: Ensure outputs are compatible with Prowler's reporting system + - **Compliance Mapping**: Map provider checks to relevant compliance frameworks + - **Backward Compatibility**: Maintain compatibility when possible + +- **AI-Assisted Development** + + - **Use Rules**: Use rules to ensure the code generated by AI is following the way of working in Prowler. + +## Checklist for New Providers + +### CLI Integration Only + +**Phase 1: Research & Planning** + +- [ ] Soft research completed +- [ ] Spike date scheduled +- [ ] Deeper research completed +- [ ] Action plan created + +**Phase 2: Implementation** + +- [ ] Folder and files created in `prowler/providers/` +- [ ] Provider class implemented and inherits from `Provider` +- [ ] Authentication/session logic implemented +- [ ] Arguments/flags mapped and documented +- [ ] Outputs and metadata standardized +- [ ] Registered in the CLI +- [ ] Minimal usage example provided + +**Phase 3: Delivery** + +- [ ] PoC delivered +- [ ] MVP delivered +- [ ] Version 1 completed +- [ ] QA and documentation completed +- [ ] GA release ready + +### API Integration + +- [ ] All CLI integration items completed +- [ ] Provider added to `ProviderChoices` enum in API models +- [ ] API serializers created/updated for the provider +- [ ] API views support the new provider type +- [ ] Provider credentials model supports the new provider +- [ ] API endpoints tested and working +- [ ] Provider-specific validation implemented +- [ ] API tests created and passing + +### UI Integration + +- TBD + +--- + +## Next Steps + +- [How to add a new Service](./services) +- [How to add new Checks](./checks) +- [How to contribute](./introduction#contributing-to-prowler) diff --git a/docs/developer-guide/services.mdx b/docs/developer-guide/services.mdx index 8f7382f0e9..ac7088dda0 100644 --- a/docs/developer-guide/services.mdx +++ b/docs/developer-guide/services.mdx @@ -5,8 +5,7 @@ title: 'Prowler Services' Here you can find how to create a new service, or to complement an existing one, for a [Prowler Provider](/developer-guide/provider). -First ensure that the provider you want to add the service is already created. It can be checked [here](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). If the provider is not present, please refer to the [Provider](/developer-guide/provider) documentation to create it from scratch. - +First ensure that the provider you want to add the service is already created. It can be checked [here](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). If the provider is not present, please refer to the [Provider](./provider.md) documentation to create it from scratch. ## Introduction @@ -201,11 +200,11 @@ class (BaseModel): #### Service Attributes -*Optimized Data Storage with Python Dictionaries* +_Optimized Data Storage with Python Dictionaries_ Each group of resources within a service should be structured as a Python [dictionary](https://docs.python.org/3/tutorial/datastructures.html#dictionaries) to enable efficient lookups. The dictionary lookup operation has [O(1) complexity](https://en.wikipedia.org/wiki/Big_O_notation#Orders_of_common_functions), and lookups are constantly executed. -*Assigning Unique Identifiers* +_Assigning Unique Identifiers_ Each dictionary key must be a unique ID to identify the resource in a univocal way. @@ -241,6 +240,301 @@ Provider-Specific Permissions Documentation: - [M365](/user-guide/providers/microsoft365/authentication#required-permissions) - [GitHub](/user-guide/providers/github/authentication) +## Service Architecture and Cross-Service Communication + +### Core Principle: Service Isolation with Client Communication + +Each service must contain **ONLY** the information unique to that specific service. When a check requires information from multiple services, it must use the **client objects** of other services rather than directly accessing their data structures. + +This architecture ensures: + +- **Loose coupling** between services +- **Clear separation of concerns** +- **Maintainable and testable code** +- **Consistent data access patterns** + +### Cross-Service Communication Pattern + +Instead of services directly accessing each other's internal data, checks should import and use client objects: + +**❌ INCORRECT - Direct data access:** + +```python +# DON'T DO THIS +from prowler.providers.aws.services.cloudtrail.cloudtrail_service import cloudtrail_service +from prowler.providers.aws.services.s3.s3_service import s3_service + +class cloudtrail_bucket_requires_mfa_delete(Check): + def execute(self): + # WRONG: Directly accessing service data + for trail in cloudtrail_service.trails.values(): + for bucket in s3_service.buckets.values(): + # Direct access violates separation of concerns +``` + +**✅ CORRECT - Client-based communication:** + +```python +# DO THIS INSTEAD +from prowler.providers.aws.services.cloudtrail.cloudtrail_client import cloudtrail_client +from prowler.providers.aws.services.s3.s3_client import s3_client + +class cloudtrail_bucket_requires_mfa_delete(Check): + def execute(self): + # CORRECT: Using client objects for cross-service communication + for trail in cloudtrail_client.trails.values(): + trail_bucket = trail.s3_bucket + for bucket in s3_client.buckets.values(): + if trail_bucket == bucket.name: + # Use bucket properties through s3_client + if bucket.mfa_delete: + # Implementation logic +``` + +### Real-World Example: CloudTrail + S3 Integration + +This example demonstrates how CloudTrail checks validate S3 bucket configurations: + +```python +from prowler.lib.check.models import Check, Check_Report_AWS +from prowler.providers.aws.services.cloudtrail.cloudtrail_client import cloudtrail_client +from prowler.providers.aws.services.s3.s3_client import s3_client + +class cloudtrail_bucket_requires_mfa_delete(Check): + def execute(self): + findings = [] + if cloudtrail_client.trails is not None: + for trail in cloudtrail_client.trails.values(): + if trail.is_logging: + trail_bucket_is_in_account = False + trail_bucket = trail.s3_bucket + + # Cross-service communication: CloudTrail check uses S3 client + for bucket in s3_client.buckets.values(): + if trail_bucket == bucket.name: + trail_bucket_is_in_account = True + if bucket.mfa_delete: + report.status = "PASS" + report.status_extended = f"Trail {trail.name} bucket ({trail_bucket}) has MFA delete enabled." + + # Handle cross-account scenarios + if not trail_bucket_is_in_account: + report.status = "MANUAL" + report.status_extended = f"Trail {trail.name} bucket ({trail_bucket}) is a cross-account bucket or out of Prowler's audit scope, please check it manually." + + findings.append(report) + return findings +``` + +**Key Benefits:** + +- **CloudTrail service** only contains CloudTrail-specific data (trails, configurations) +- **S3 service** only contains S3-specific data (buckets, policies, ACLs) +- **Check logic** orchestrates between services using their public client interfaces +- **Cross-account detection** is handled gracefully when resources span accounts + +### Service Consolidation Guidelines + +**When to combine services in the same file:** + +Implement multiple services as **separate classes in the same file** when two services are **practically the same** or one is a **direct extension** of another. + +**Example: S3 and S3Control** + +S3Control is an extension of S3 that provides account-level controls and access points. Both are implemented in `s3_service.py`: + +```python +# File: prowler/providers/aws/services/s3/s3_service.py + +class S3(AWSService): + """Standard S3 service for bucket operations""" + def __init__(self, provider): + super().__init__(__class__.__name__, provider) + self.buckets = {} + self.regions_with_buckets = [] + + # S3-specific initialization + self._list_buckets(provider) + self._get_bucket_versioning() + # ... other S3-specific operations + +class S3Control(AWSService): + """S3Control service for account-level and access point operations""" + def __init__(self, provider): + super().__init__(__class__.__name__, provider) + self.account_public_access_block = None + self.access_points = {} + + # S3Control-specific initialization + self._get_public_access_block() + self._list_access_points() + # ... other S3Control-specific operations +``` + +**Separate client files:** + +```python +# File: prowler/providers/aws/services/s3/s3_client.py +from prowler.providers.aws.services.s3.s3_service import S3 +s3_client = S3(Provider.get_global_provider()) + +# File: prowler/providers/aws/services/s3/s3control_client.py +from prowler.providers.aws.services.s3.s3_service import S3Control +s3control_client = S3Control(Provider.get_global_provider()) +``` + +**When NOT to consolidate services:** + +Keep services separate when they: + +- **Operate on different resource types** (EC2 vs RDS) +- **Have different authentication mechanisms** (different API endpoints) +- **Serve different operational domains** (IAM vs CloudTrail) +- **Have different regional behaviors** (global vs regional services) + +### Cross-Service Dependencies Guidelines + +**1. Always use client imports:** + +```python +# Correct pattern +from prowler.providers.aws.services.service_a.service_a_client import service_a_client +from prowler.providers.aws.services.service_b.service_b_client import service_b_client +``` + +**2. Handle missing resources gracefully:** + +```python +# Handle cross-service scenarios +resource_found_in_account = False +for external_resource in other_service_client.resources.values(): + if target_resource_id == external_resource.id: + resource_found_in_account = True + # Process found resource + break + +if not resource_found_in_account: + # Handle cross-account or missing resource scenarios + report.status = "MANUAL" + report.status_extended = "Resource is cross-account or out of audit scope" +``` + +**3. Document cross-service dependencies:** + +```python +class check_with_dependencies(Check): + """ + Check Description + + Dependencies: + - service_a_client: For primary resource information + - service_b_client: For related resource validation + - service_c_client: For policy analysis + """ +``` + +## Regional Service Implementation + +When implementing services for regional providers (like AWS, Azure, GCP), special considerations are needed to handle resource discovery across multiple geographic locations. This section provides a complete guide using AWS as the reference example. + +### Regional vs Non-Regional Services + +**Regional Services:** Require iteration across multiple geographic locations where resources may exist (e.g., EC2 instances, VPC, RDS databases). + +**Non-Regional/Global Services:** Operate at a global or tenant level without regional concepts (e.g., IAM users, Route53 hosted zones). + +### AWS Regional Implementation Example + +AWS is the perfect example of a regional provider. Here's how Prowler handles AWS's regional architecture: + + +```python +# File: prowler/providers/aws/services/ec2/ec2_service.py +class EC2(AWSService): + def __init__(self, provider): + super().__init__(__class__.__name__, provider) + self.instances = {} + self.security_groups = {} + + # Regional resource discovery across all AWS regions + self.__threading_call__(self._describe_instances) + self.__threading_call__(self._describe_security_groups) + + def _describe_instances(self, regional_client): + """Discover EC2 instances in a specific region""" + try: + describe_instances_paginator = regional_client.get_paginator("describe_instances") + for page in describe_instances_paginator.paginate(): + for reservation in page["Reservations"]: + for instance in reservation["Instances"]: + # Each instance includes its region + self.instances[instance["InstanceId"]] = Instance( + id=instance["InstanceId"], + region=regional_client.region, + state=instance["State"]["Name"], + # ... other properties + ) + except Exception as error: + logger.error(f"Failed to describe instances in {regional_client.region}: {error}") +``` + +#### Regional Check Execution + +```python +# File: prowler/providers/aws/services/ec2/ec2_instance_public_ip/ec2_instance_public_ip.py +class ec2_instance_public_ip(Check): + def execute(self): + findings = [] + + # Automatically iterates across ALL AWS regions where instances exist + for instance in ec2_client.instances.values(): + report = Check_Report_AWS(metadata=self.metadata(), resource=instance) + report.region = instance.region # Critical: region attribution + report.resource_arn = f"arn:aws:ec2:{instance.region}:{instance.account_id}:instance/{instance.id}" + + if instance.public_ip: + report.status = "FAIL" + report.status_extended = f"Instance {instance.id} in {instance.region} has public IP {instance.public_ip}" + else: + report.status = "PASS" + report.status_extended = f"Instance {instance.id} in {instance.region} does not have a public IP" + + findings.append(report) + + return findings +``` + +#### Key AWS Regional Features + +**Region-Specific ARNs:** + +``` +arn:aws:ec2:us-east-1:123456789012:instance/i-1234567890abcdef0 +arn:aws:s3:eu-west-1:123456789012:bucket/my-bucket +arn:aws:rds:ap-southeast-2:123456789012:db:my-database +``` + +**Parallel Processing:** + +- Each region processed independently in separate threads +- Failed regions don't affect other regions +- User can filter specific regions: `-f us-east-1` + +**Global vs Regional Services:** + +- **Regional**: EC2, RDS, VPC (require region iteration) +- **Global**: IAM, Route53, CloudFront (single `us-east-1` call) + +This architecture allows Prowler to efficiently scan AWS accounts with resources spread across multiple regions while maintaining performance and error isolation. + +### Regional Service Best Practices + +1. **Use Threading for Regional Discovery**: Leverage the `__threading_call__` method to parallelize resource discovery across regions +2. **Store Region Information**: Always include region metadata in resource objects for proper attribution +3. **Handle Regional Failures Gracefully**: Ensure that failures in one region don't affect others +4. **Optimize for Performance**: Use paginated calls and efficient data structures for large-scale resource discovery +5. **Support Region Filtering**: Allow users to limit scans to specific regions for focused audits + ## Best Practices - When available in the provider, use threading or parallelization utilities for all methods that can be parallelized by to maximize performance and reduce scan time. @@ -252,3 +546,5 @@ Provider-Specific Permissions Documentation: - Collect and store resource tags and additional attributes to support richer checks and reporting. - Leverage shared utility helpers for session setup, identifier parsing, and other cross-cutting concerns to avoid code duplication. This kind of code is typically stored in a `lib` folder in the service folder. - Keep code modular, maintainable, and well-documented for ease of extension and troubleshooting. +- **Each service should contain only information unique to that specific service** - use client objects for cross-service communication. +- **Handle cross-account and missing resources gracefully** when checks span multiple services. diff --git a/docs/docs.json b/docs/docs.json index fb7e91a774..54402e5d8a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -98,7 +98,7 @@ ] }, "user-guide/tutorials/prowler-app-rbac", - "user-guide/providers/prowler-app-api-keys", + "user-guide/tutorials/prowler-app-api-keys", "user-guide/tutorials/prowler-app-mute-findings", { "group": "Integrations", @@ -114,7 +114,8 @@ "group": "Tutorials", "pages": [ "user-guide/tutorials/prowler-app-sso-entra", - "user-guide/tutorials/bulk-provider-provisioning" + "user-guide/tutorials/bulk-provider-provisioning", + "user-guide/tutorials/aws-organizations-bulk-provisioning" ] } ] diff --git a/docs/getting-started/basic-usage/prowler-mcp.mdx b/docs/getting-started/basic-usage/prowler-mcp.mdx index 90cc3e0e63..5d89756acf 100644 --- a/docs/getting-started/basic-usage/prowler-mcp.mdx +++ b/docs/getting-started/basic-usage/prowler-mcp.mdx @@ -10,7 +10,7 @@ Configure your MCP client to connect to Prowler MCP Server. **Authentication is optional**: Prowler Hub and Prowler Documentation features work without authentication. An API key is only required for Prowler Cloud and Prowler App (Self-Managed) features. -To use Prowler Cloud or Prowler App (Self-Managed) features. To get the API key, please refer to the [API Keys](/user-guide/providers/prowler-app-api-keys) guide. +To use Prowler Cloud or Prowler App (Self-Managed) features. To get the API key, please refer to the [API Keys](/user-guide/tutorials/prowler-app-api-keys) guide. Keep the API key secure. Never share it publicly or commit it to version control. diff --git a/docs/getting-started/installation/prowler-app.mdx b/docs/getting-started/installation/prowler-app.mdx index 862d76a73f..4c49b8d4f0 100644 --- a/docs/getting-started/installation/prowler-app.mdx +++ b/docs/getting-started/installation/prowler-app.mdx @@ -25,6 +25,9 @@ Prowler configuration is based in `.env` files. Every version of Prowler can hav curl -LO https://raw.githubusercontent.com/prowler-cloud/prowler/refs/heads/master/.env docker compose up -d ``` + + > Containers are built for `linux/amd64`. If your workstation's architecture is different, please set `DOCKER_DEFAULT_PLATFORM=linux/amd64` in your environment or use the `--platform linux/amd64` flag in the docker command. + _Requirements_: diff --git a/docs/img/provider-decision-tree.png b/docs/img/provider-decision-tree.png new file mode 100644 index 0000000000..f76d375f94 Binary files /dev/null and b/docs/img/provider-decision-tree.png differ diff --git a/docs/snippets/version-badge.mdx b/docs/snippets/version-badge.mdx new file mode 100644 index 0000000000..7541ff823a --- /dev/null +++ b/docs/snippets/version-badge.mdx @@ -0,0 +1,12 @@ +export const VersionBadge = ({ version }) => { + return ( + +

+ Added in:  + {version} +

+
+ + + ); +}; diff --git a/docs/style.css b/docs/style.css new file mode 100644 index 0000000000..3e9bedbc80 --- /dev/null +++ b/docs/style.css @@ -0,0 +1,51 @@ +/* Version Badge Styling */ +.version-badge-container { + display: inline-block; + margin: 0 0 1rem 0; + padding: 0; +} + +.version-badge { + display: inline-flex; + align-items: center; + margin: 0; + padding: 0.375rem 0.75rem; + background: linear-gradient(135deg, #1a1a1a 0%, #000000 100%); + color: #ffffff; + border-radius: 1.25rem; + font-weight: 400; + font-size: 0.875rem; + line-height: 1.25rem; + border: 1px solid rgba(0, 0, 0, 0.15); + box-shadow: none; +} + +.version-badge-label { + font-weight: 400; + opacity: 1; +} + +.version-badge-version { + background: rgba(255, 255, 255, 0.12); + padding: 0.125rem 0.5rem; + border-radius: 0.875rem; + font-family: ui-monospace, SFMono-Regular, 'SF Mono', Menlo, Monaco, 'Cascadia Code', 'Roboto Mono', Consolas, 'Courier New', monospace; + font-weight: 600; + font-size: 0.875rem; + color: #ffffff; + border: none; +} + + +.dark .version-badge { + background: #55B685; + color: #000000; + border: 2px solid rgba(85, 182, 133, 0.3); + box-shadow: none; + } + + .dark .version-badge-version { + background: rgba(0, 0, 0, 0.1); + color: #000000; + border: none; +} diff --git a/docs/user-guide/tutorials/aws-organizations-bulk-provisioning.mdx b/docs/user-guide/tutorials/aws-organizations-bulk-provisioning.mdx new file mode 100644 index 0000000000..e19d416d32 --- /dev/null +++ b/docs/user-guide/tutorials/aws-organizations-bulk-provisioning.mdx @@ -0,0 +1,491 @@ +--- +title: 'AWS Organizations Bulk Provisioning in Prowler' +--- + +Prowler offers an automated tool to discover and provision all AWS accounts within an AWS Organization. This streamlines onboarding for organizations managing multiple AWS accounts by automatically generating the configuration needed for bulk provisioning. + +The tool, `aws_org_generator.py`‎, complements the [Bulk Provider Provisioning](./bulk-provider-provisioning) tool and is available in the Prowler repository at: [util/prowler-bulk-provisioning](https://github.com/prowler-cloud/prowler/tree/master/util/prowler-bulk-provisioning) + + +Native support for bulk provisioning AWS Organizations and similar multi-account structures directly in the Prowler UI/API is on the official roadmap. + +Track progress and vote for this feature at: [Bulk Provisioning in the UI/API for AWS Organizations](https://roadmap.prowler.com/p/builk-provisioning-in-the-uiapi-for-aws-organizations-and-alike) + + +{/* TODO: Add screenshot of the tool in action */} + +## Overview + +The AWS Organizations Bulk Provisioning tool simplifies multi-account onboarding by: + +* Automatically discovering all active accounts in an AWS Organization +* Generating YAML configuration files for bulk provisioning +* Supporting account filtering and custom role configurations +* Eliminating manual entry of account IDs and role ARNs + +## Prerequisites + +### Requirements + +* Python 3.7 or higher +* AWS credentials with Organizations read access +* ProwlerRole (or custom role) deployed across all target accounts +* Prowler API key (from Prowler Cloud or self-hosted Prowler App) + * For self-hosted Prowler App, remember to [point to your API base URL](./bulk-provider-provisioning#custom-api-endpoints) + * Learn how to create API keys: [Prowler App API Keys](../tutorials/prowler-app-api-keys) + +### Deploying ProwlerRole Across AWS Organizations + +Before using the AWS Organizations generator, deploy the ProwlerRole across all accounts in the organization using CloudFormation StackSets. + + +**Follow the official documentation:** +[Deploying Prowler IAM Roles Across AWS Organizations](../providers/aws/organizations#deploying-prowler-iam-roles-across-aws-organizations) + +**Key points:** + +* Use CloudFormation StackSets from the management account +* Deploy to all organizational units (OUs) or specific OUs +* Use an external ID for enhanced security +* Ensure the role has necessary permissions for Prowler scans + + +### Installation + +Clone the repository and install required dependencies: + +```bash +git clone https://github.com/prowler-cloud/prowler.git +cd prowler/util/prowler-bulk-provisioning +pip install -r requirements-aws-org.txt +``` + +### AWS Credentials Setup + +Configure AWS credentials with Organizations read access: + +* **Management account credentials**, or +* **Delegated administrator account** with `organizations:ListAccounts` permission + +Required IAM permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "organizations:ListAccounts", + "organizations:DescribeOrganization" + ], + "Resource": "*" + } + ] +} +``` + +### Prowler API Key Setup + +Configure your Prowler API key: + +```bash +export PROWLER_API_KEY="pk_example-api-key" +``` + +To create an API key: + +1. Log in to Prowler Cloud or Prowler App +2. Click **Profile** → **Account** +3. Click **Create API Key** +4. Provide a descriptive name and optionally set an expiration date +5. Copy the generated API key (it will only be shown once) + +For detailed instructions, see: [Prowler App API Keys](../tutorials/prowler-app-api-keys) + +## Basic Usage + +### Generate Configuration for All Accounts + +To generate a YAML configuration file for all active accounts in the organization: + +```bash +python aws_org_generator.py -o aws-accounts.yaml --external-id prowler-ext-id-2024 +``` + +This command: + +1. Lists all ACTIVE accounts in the organization +2. Generates YAML entries for each account +3. Saves the configuration to `aws-accounts.yaml` + +**Output:** + +``` +Fetching accounts from AWS Organizations... +Found 47 active accounts in organization +Generated configuration for 47 accounts + +Configuration written to: aws-accounts.yaml + +Next steps: + 1. Review the generated file: cat aws-accounts.yaml | head -n 20 + 2. Run bulk provisioning: python prowler_bulk_provisioning.py aws-accounts.yaml +``` + +### Review Generated Configuration + +Review the generated YAML configuration: + +```bash +head -n 20 aws-accounts.yaml +``` + +**Example output:** + +```yaml +- provider: aws + uid: '111111111111' + alias: Production-Account + auth_method: role + credentials: + role_arn: arn:aws:iam::111111111111:role/ProwlerRole + external_id: prowler-ext-id-2024 + +- provider: aws + uid: '222222222222' + alias: Development-Account + auth_method: role + credentials: + role_arn: arn:aws:iam::222222222222:role/ProwlerRole + external_id: prowler-ext-id-2024 +``` + +### Dry Run Mode + +Test the configuration without writing a file: + +```bash +python aws_org_generator.py \ + --external-id prowler-ext-id-2024 \ + --dry-run +``` + +## Advanced Configuration + +### Using a Specific AWS Profile + +Specify an AWS profile when multiple profiles are configured: + +```bash +python aws_org_generator.py \ + -o aws-accounts.yaml \ + --profile org-management-admin \ + --external-id prowler-ext-id-2024 +``` + +### Excluding Specific Accounts + +Exclude the management account or other accounts from provisioning: + +```bash +python aws_org_generator.py \ + -o aws-accounts.yaml \ + --external-id prowler-ext-id-2024 \ + --exclude 123456789012,210987654321 +``` + +Common exclusion scenarios: + +* Management account (requires different permissions) +* Break-glass accounts (emergency access) +* Suspended or archived accounts + +### Including Only Specific Accounts + +Generate configuration for specific accounts only: + +```bash +python aws_org_generator.py \ + -o aws-accounts.yaml \ + --external-id prowler-ext-id-2024 \ + --include 111111111111,222222222222,333333333333 +``` + +### Custom Role Name + +Specify a custom role name if not using the default `ProwlerRole`: + +```bash +python aws_org_generator.py \ + -o aws-accounts.yaml \ + --role-name ProwlerExecutionRole \ + --external-id prowler-ext-id-2024 +``` + +### Custom Alias Format + +Customize account aliases using template variables: + +```bash +# Use account name and ID +python aws_org_generator.py \ + -o aws-accounts.yaml \ + --alias-format "{name}-{id}" \ + --external-id prowler-ext-id-2024 + +# Use email prefix +python aws_org_generator.py \ + -o aws-accounts.yaml \ + --alias-format "{email}" \ + --external-id prowler-ext-id-2024 +``` + +Available template variables: + +* `{name}` - Account name +* `{id}` - Account ID +* `{email}` - Account email + +### Additional Role Assumption Options + +Configure optional role assumption parameters: + +```bash +python aws_org_generator.py \ + -o aws-accounts.yaml \ + --role-name ProwlerRole \ + --external-id prowler-ext-id-2024 \ + --session-name prowler-scan-session \ + --duration-seconds 3600 +``` + +## Complete Workflow Example + + + + 1. Log in to the AWS management account + 2. Open CloudFormation → StackSets + 3. Create a new StackSet using the [Prowler role template](https://github.com/prowler-cloud/prowler/blob/master/permissions/templates/cloudformation/prowler-scan-role.yml) + 4. Deploy to all organizational units + 5. Use a unique external ID (e.g., `prowler-org-2024-abc123`) + + {/* TODO: Add screenshot of CloudFormation StackSets deployment */} + + + + Configure AWS credentials and generate the YAML file: + + ```bash + # Using management account credentials + export AWS_PROFILE=org-management + + # Generate configuration + python aws_org_generator.py \ + -o aws-org-accounts.yaml \ + --external-id prowler-org-2024-abc123 \ + --exclude 123456789012 + ``` + + **Output:** + + ``` + Fetching accounts from AWS Organizations... + Using AWS profile: org-management + Found 47 active accounts in organization + Generated configuration for 46 accounts + + Configuration written to: aws-org-accounts.yaml + + Next steps: + 1. Review the generated file: cat aws-org-accounts.yaml | head -n 20 + 2. Run bulk provisioning: python prowler_bulk_provisioning.py aws-org-accounts.yaml + ``` + + + + Verify the generated YAML configuration: + + ```bash + # View first 20 lines + head -n 20 aws-org-accounts.yaml + + # Check for unexpected accounts + grep "uid:" aws-org-accounts.yaml + + # Verify role ARNs + grep "role_arn:" aws-org-accounts.yaml | head -5 + + # Count accounts + grep "provider: aws" aws-org-accounts.yaml | wc -l + ``` + + + + Provision all accounts to Prowler Cloud or Prowler App: + + ```bash + # Set Prowler API key + export PROWLER_API_KEY="pk_example-api-key" + + # Run bulk provisioning with connection testing + python prowler_bulk_provisioning.py aws-org-accounts.yaml + ``` + + **With custom options:** + + ```bash + python prowler_bulk_provisioning.py aws-org-accounts.yaml \ + --concurrency 10 \ + --timeout 120 + ``` + + **Successful output:** + + ``` + [1] ✅ Created provider (id=db9a8985-f9ec-4dd8-b5a0-e05ab3880bed) + [1] ✅ Created secret (id=466f76c6-5878-4602-a4bc-13f9522c1fd2) + [1] ✅ Connection test: Connected + + [2] ✅ Created provider (id=7a99f789-0cf5-4329-8279-2d443a962676) + [2] ✅ Created secret (id=c5702180-f7c4-40fd-be0e-f6433479b126) + [2] ✅ Connection test: Connected + + Done. Success: 47 Failures: 0 + ``` + + {/* TODO: Add screenshot of successful bulk provisioning output */} + + + +## Command Reference + +### Full Command-Line Options + +```bash +python aws_org_generator.py \ + -o OUTPUT_FILE \ + --role-name ROLE_NAME \ + --external-id EXTERNAL_ID \ + --session-name SESSION_NAME \ + --duration-seconds SECONDS \ + --alias-format FORMAT \ + --exclude ACCOUNT_IDS \ + --include ACCOUNT_IDS \ + --profile AWS_PROFILE \ + --region AWS_REGION \ + --dry-run +``` + +## Troubleshooting + +### Error: "No AWS credentials found" + +**Solution:** Configure AWS credentials using one of these methods: + +```bash +# Method 1: AWS CLI configure +aws configure + +# Method 2: Environment variables +export AWS_ACCESS_KEY_ID=your-key-id +export AWS_SECRET_ACCESS_KEY=your-secret-key + +# Method 3: Use AWS profile +export AWS_PROFILE=org-management +``` + +### Error: "Access denied to AWS Organizations API" + +**Cause:** Current credentials don't have permission to list organization accounts. + +**Solution:** + +* Ensure management account credentials are used +* Verify IAM permissions include `organizations:ListAccounts` +* Check IAM policies for Organizations access + +### Error: "AWS Organizations is not enabled" + +**Cause:** The account is not part of an organization. + +**Solution:** This tool requires an AWS Organization. Create one in the AWS Organizations console or use standard bulk provisioning for standalone accounts. + +### No Accounts Generated After Filters + +**Cause:** All accounts were filtered out by `--exclude` or `--include` options. + +**Solution:** Review filter options and verify account IDs are correct: + +```bash +# List all accounts in organization +aws organizations list-accounts --query "Accounts[?Status=='ACTIVE'].[Id,Name]" --output table +``` + +### Connection Test Failures During Bulk Provisioning + +**Cause:** ProwlerRole may not be deployed correctly or credentials are invalid. + +**Solution:** + +* Verify StackSet deployment status in CloudFormation +* Check role trust policy includes correct external ID +* Test role assumption manually: + +```bash +aws sts assume-role \ + --role-arn arn:aws:iam::123456789012:role/ProwlerRole \ + --role-session-name test \ + --external-id prowler-ext-id-2024 +``` + +## Security Best Practices + +### Use External ID + +Always use an external ID when assuming cross-account roles: + +```bash +python aws_org_generator.py \ + -o aws-accounts.yaml \ + --external-id $(uuidgen | tr '[:upper:]' '[:lower:]') +``` + +The external ID must match the one configured in the ProwlerRole trust policy across all accounts. + +### Exclude Sensitive Accounts + +Exclude accounts that shouldn't be scanned or require special handling: + +```bash +python aws_org_generator.py \ + -o aws-accounts.yaml \ + --external-id prowler-ext-id \ + --exclude 123456789012,111111111111 # management, break-glass accounts +``` + +### Review Generated Configuration + +Always review the generated YAML before provisioning: + +```bash +# Check for unexpected accounts +grep "uid:" aws-org-accounts.yaml + +# Verify role ARNs +grep "role_arn:" aws-org-accounts.yaml | head -5 + +# Count accounts +grep "provider: aws" aws-org-accounts.yaml | wc -l +``` + +## Next Steps + + + + Learn how to bulk provision providers in Prowler. + + + Detailed instructions on how to use Prowler. + + diff --git a/docs/user-guide/tutorials/bulk-provider-provisioning.mdx b/docs/user-guide/tutorials/bulk-provider-provisioning.mdx index 11e7f14cf8..3ad5d9c0d5 100644 --- a/docs/user-guide/tutorials/bulk-provider-provisioning.mdx +++ b/docs/user-guide/tutorials/bulk-provider-provisioning.mdx @@ -17,14 +17,18 @@ The Bulk Provider Provisioning tool automates the creation of cloud providers in * Testing connections to verify successful authentication * Processing multiple providers concurrently for efficiency + +**Using AWS Organizations?** For organizations with many AWS accounts, use the automated [AWS Organizations Bulk Provisioning](./aws-organizations-bulk-provisioning) tool to automatically discover and generate configuration for all accounts in your organization. + ## Prerequisites ### Requirements * Python 3.7 or higher -* Prowler API token (from Prowler Cloud or self-hosted Prowler App) +* Prowler API key (from Prowler Cloud or self-hosted Prowler App) * For self-hosted Prowler App, remember to [point to your API base URL](#custom-api-endpoints) + * Learn how to create API keys: [Prowler App API Keys](../tutorials/prowler-app-api-keys) * Authentication credentials for target cloud providers ### Installation @@ -39,28 +43,21 @@ pip install -r requirements.txt ### Authentication Setup -Configure your Prowler API token: +Configure your Prowler API key: ```bash -export PROWLER_API_TOKEN="your-prowler-api-token" +export PROWLER_API_KEY="pk_example-api-key" ``` -To obtain an API token programmatically: +To create an API key: -```bash -export PROWLER_API_TOKEN=$(curl --location 'https://api.prowler.com/api/v1/tokens' \ - --header 'Content-Type: application/vnd.api+json' \ - --header 'Accept: application/vnd.api+json' \ - --data-raw '{ - "data": { - "type": "tokens", - "attributes": { - "email": "your@email.com", - "password": "your-password" - } - } - }' | jq -r .data.attributes.access) -``` +1. Log in to Prowler Cloud or Prowler App +2. Click **Profile** → **Account** +3. Click **Create API Key** +4. Provide a descriptive name and optionally set an expiration date +5. Copy the generated API key (it will only be shown once) + +For detailed instructions, see: [Prowler App API Keys](../tutorials/prowler-app-api-keys) ## Configuration File Structure @@ -340,11 +337,11 @@ Done. Success: 2 Failures: 0 ## Troubleshooting -### Invalid API Token +### Invalid API Key ``` Error: 401 Unauthorized -Solution: Verify your PROWLER_API_TOKEN or --token parameter +Solution: Verify your PROWLER_API_KEY environment variable or --api-key parameter ``` ### Network Timeouts diff --git a/docs/user-guide/providers/prowler-app-api-keys.mdx b/docs/user-guide/tutorials/prowler-app-api-keys.mdx similarity index 99% rename from docs/user-guide/providers/prowler-app-api-keys.mdx rename to docs/user-guide/tutorials/prowler-app-api-keys.mdx index f7a656e173..4e51e62bac 100644 --- a/docs/user-guide/providers/prowler-app-api-keys.mdx +++ b/docs/user-guide/tutorials/prowler-app-api-keys.mdx @@ -2,6 +2,10 @@ title: 'API Keys' --- +import { VersionBadge } from "/snippets/version-badge.mdx" + + + API key authentication in Prowler App provides an alternative to JWT tokens and empowers automation, CI/CD pipelines, and third-party integrations. This guide explains how to create, manage, and safeguard API keys when working with the Prowler API. ## API Key Advantages diff --git a/docs/user-guide/tutorials/prowler-app-jira-integration.mdx b/docs/user-guide/tutorials/prowler-app-jira-integration.mdx index a4eed1e494..19a31347c9 100644 --- a/docs/user-guide/tutorials/prowler-app-jira-integration.mdx +++ b/docs/user-guide/tutorials/prowler-app-jira-integration.mdx @@ -1,6 +1,9 @@ --- title: "Jira Integration" --- +import { VersionBadge } from "/snippets/version-badge.mdx" + + Prowler App enables automatic export of security findings to Jira, providing seamless integration with Atlassian's work item tracking and project management platform. This comprehensive guide demonstrates how to configure and manage Jira integrations to streamline security incident management and enhance team collaboration across security workflows. diff --git a/docs/user-guide/tutorials/prowler-app-lighthouse.mdx b/docs/user-guide/tutorials/prowler-app-lighthouse.mdx index 1ea3da3ac5..abff053dc8 100644 --- a/docs/user-guide/tutorials/prowler-app-lighthouse.mdx +++ b/docs/user-guide/tutorials/prowler-app-lighthouse.mdx @@ -2,6 +2,10 @@ title: 'Prowler Lighthouse AI' --- +import { VersionBadge } from "/snippets/version-badge.mdx" + + + Prowler Lighthouse AI is a Cloud Security Analyst chatbot that helps you understand, prioritize, and remediate security findings in your cloud environments. It's designed to provide security expertise for teams without dedicated resources, acting as your 24/7 virtual cloud security analyst. Prowler Lighthouse diff --git a/docs/user-guide/tutorials/prowler-app-mute-findings.mdx b/docs/user-guide/tutorials/prowler-app-mute-findings.mdx index 76b85f0e66..9f3bdad640 100644 --- a/docs/user-guide/tutorials/prowler-app-mute-findings.mdx +++ b/docs/user-guide/tutorials/prowler-app-mute-findings.mdx @@ -1,6 +1,9 @@ --- title: 'Mute Findings (Mutelist)' --- +import { VersionBadge } from "/snippets/version-badge.mdx" + + Prowler App allows users to mute specific findings to focus on the most critical security issues. This comprehensive guide demonstrates how to effectively use the Mutelist feature to manage and prioritize security findings. diff --git a/docs/user-guide/tutorials/prowler-app-rbac.mdx b/docs/user-guide/tutorials/prowler-app-rbac.mdx index 8accc77488..72059436f5 100644 --- a/docs/user-guide/tutorials/prowler-app-rbac.mdx +++ b/docs/user-guide/tutorials/prowler-app-rbac.mdx @@ -2,6 +2,10 @@ title: 'Managing Users and Role-Based Access Control (RBAC)' --- +import { VersionBadge } from "/snippets/version-badge.mdx" + + + **Prowler App** supports multiple users within a single tenant, enabling seamless collaboration by allowing team members to easily share insights and manage security findings. [Roles](#roles) help you control user permissions, determining what actions each user can perform and the data they can access within Prowler. By default, each account includes an immutable **admin** role, ensuring that your account always retains administrative access. diff --git a/docs/user-guide/tutorials/prowler-app-s3-integration.mdx b/docs/user-guide/tutorials/prowler-app-s3-integration.mdx index 8e8158b0c0..728e4a4354 100644 --- a/docs/user-guide/tutorials/prowler-app-s3-integration.mdx +++ b/docs/user-guide/tutorials/prowler-app-s3-integration.mdx @@ -2,6 +2,10 @@ title: 'Amazon S3 Integration' --- +import { VersionBadge } from "/snippets/version-badge.mdx" + + + **Prowler App** allows automatic export of scan results to Amazon S3 buckets, providing seamless integration with existing data workflows and storage infrastructure. This comprehensive guide demonstrates configuration and management of Amazon S3 integrations to streamline security finding management and reporting. When enabled and configured, scan results are automatically stored in the configured bucket. Results are provided in `csv`, `html` and `json-ocsf` formats, offering flexibility for custom integrations: diff --git a/docs/user-guide/tutorials/prowler-app-security-hub-integration.mdx b/docs/user-guide/tutorials/prowler-app-security-hub-integration.mdx index 4c192e7a0c..592ff2b26d 100644 --- a/docs/user-guide/tutorials/prowler-app-security-hub-integration.mdx +++ b/docs/user-guide/tutorials/prowler-app-security-hub-integration.mdx @@ -1,6 +1,9 @@ --- title: "AWS Security Hub Integration" --- +import { VersionBadge } from "/snippets/version-badge.mdx" + + Prowler App enables automatic export of security findings to AWS Security Hub, providing seamless integration with AWS's native security and compliance service. This comprehensive guide demonstrates how to configure and manage AWS Security Hub integrations to centralize security findings and enhance compliance tracking across AWS environments. diff --git a/docs/user-guide/tutorials/prowler-app-social-login.mdx b/docs/user-guide/tutorials/prowler-app-social-login.mdx index ecc75959e9..45fe8ec62f 100644 --- a/docs/user-guide/tutorials/prowler-app-social-login.mdx +++ b/docs/user-guide/tutorials/prowler-app-social-login.mdx @@ -2,6 +2,10 @@ title: 'Social Login Configuration' --- +import { VersionBadge } from "/snippets/version-badge.mdx" + + + **Prowler App** supports social login using Google and GitHub OAuth providers. This document guides you through configuring the required environment variables to enable social authentication. Social login buttons diff --git a/docs/user-guide/tutorials/prowler-app-sso.mdx b/docs/user-guide/tutorials/prowler-app-sso.mdx index b22ae7941a..f56336f415 100644 --- a/docs/user-guide/tutorials/prowler-app-sso.mdx +++ b/docs/user-guide/tutorials/prowler-app-sso.mdx @@ -2,6 +2,10 @@ title: 'SAML Single Sign-On (SSO)' --- +import { VersionBadge } from "/snippets/version-badge.mdx" + + + This guide provides comprehensive instructions to configure SAML-based Single Sign-On (SSO) in Prowler App. This configuration allows users to authenticate using the organization's Identity Provider (IdP). This document is divided into two main sections: diff --git a/mcp_server/CHANGELOG.md b/mcp_server/CHANGELOG.md index 490c1878e9..80b43bcfff 100644 --- a/mcp_server/CHANGELOG.md +++ b/mcp_server/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to the **Prowler MCP Server** are documented in this file. -## [0.1.0] (Prowler UNRELEASED) +## [0.1.0] (Prowler 5.13.0) ### Added - Initial release of Prowler MCP Server [(#8695)](https://github.com/prowler-cloud/prowler/pull/8695) diff --git a/prowler/CHANGELOG.md b/prowler/CHANGELOG.md index 087c9241f0..ed809c4f17 100644 --- a/prowler/CHANGELOG.md +++ b/prowler/CHANGELOG.md @@ -2,15 +2,17 @@ All notable changes to the **Prowler SDK** are documented in this file. -## [Unreleased] +## [v5.14.0] (Prowler UNRELEASED) + +### Added +- GitHub provider check `organization_default_repository_permission_strict` [(#8785)](https://github.com/prowler-cloud/prowler/pull/8785) ### Changed - - Adapt IaC provider to be used in the Prowler App [(#8751)](https://github.com/prowler-cloud/prowler/pull/8751) --- -## [v5.13.0] (Prowler UNRELEASED) +## [v5.13.0] (Prowler v5.13.0) ### Added - Support for AdditionalURLs in outputs [(#8651)](https://github.com/prowler-cloud/prowler/pull/8651) @@ -25,6 +27,8 @@ All notable changes to the **Prowler SDK** are documented in this file. - Oracle Cloud provider with CIS 3.0 benchmark [(#8893)](https://github.com/prowler-cloud/prowler/pull/8893) - Support for Atlassian Document Format (ADF) in Jira integration [(#8878)](https://github.com/prowler-cloud/prowler/pull/8878) - Add Common Cloud Controls for AWS, Azure and GCP [(#8000)](https://github.com/prowler-cloud/prowler/pull/8000) +- Improve Provider documentation guide [(#8430)](https://github.com/prowler-cloud/prowler/pull/8430) +- `cloudstorage_bucket_lifecycle_management_enabled` check for GCP provider [(#8936)](https://github.com/prowler-cloud/prowler/pull/8936) ### Changed @@ -58,13 +62,8 @@ All notable changes to the **Prowler SDK** are documented in this file. - Prowler ThreatScore scoring calculation CLI [(#8582)](https://github.com/prowler-cloud/prowler/pull/8582) - Add missing attributes for Mitre Attack AWS, Azure and GCP [(#8907)](https://github.com/prowler-cloud/prowler/pull/8907) - Fix KeyError in CloudSQL and Monitoring services in GCP provider [(#8909)](https://github.com/prowler-cloud/prowler/pull/8909) +- Fix Value Errors in Entra service for M365 provider [(#8919)](https://github.com/prowler-cloud/prowler/pull/8919) - Fix ResourceName in GCP provider [(#8928)](https://github.com/prowler-cloud/prowler/pull/8928) - ---- - -## [v5.12.4] (Prowler UNRELEASED) - -### Fixed - Fix KeyError in `elb_ssl_listeners_use_acm_certificate` check and handle None cluster version in `eks_cluster_uses_a_supported_version` check [(#8791)](https://github.com/prowler-cloud/prowler/pull/8791) - Fix file extension parsing for compliance reports [(#8791)](https://github.com/prowler-cloud/prowler/pull/8791) - Added user pagination to Entra and Admincenter services [(#8858)](https://github.com/prowler-cloud/prowler/pull/8858) diff --git a/prowler/compliance/github/cis_1.0_github.json b/prowler/compliance/github/cis_1.0_github.json index 731e75edbd..d488b92a00 100644 --- a/prowler/compliance/github/cis_1.0_github.json +++ b/prowler/compliance/github/cis_1.0_github.json @@ -753,7 +753,9 @@ { "Id": "1.3.8", "Description": "Base permissions define the permission level automatically granted to all organization members. Define strict base access permissions for all of the repositories in the organization, including new ones.", - "Checks": [], + "Checks": [ + "organization_default_repository_permission_strict" + ], "Attributes": [ { "Section": "1 Source Code", diff --git a/prowler/config/config.py b/prowler/config/config.py index 97200fe0b1..b8fbe91074 100644 --- a/prowler/config/config.py +++ b/prowler/config/config.py @@ -12,7 +12,7 @@ from prowler.lib.logger import logger timestamp = datetime.today() timestamp_utc = datetime.now(timezone.utc).replace(tzinfo=timezone.utc) -prowler_version = "5.13.0" +prowler_version = "5.14.0" html_logo_url = "https://github.com/prowler-cloud/prowler/" square_logo_img = "https://prowler.com/wp-content/uploads/logo-html.png" aws_logo = "https://user-images.githubusercontent.com/38561120/235953920-3e3fba08-0795-41dc-b480-9bea57db9f2e.png" diff --git a/prowler/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/__init__.py b/prowler/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/cloudstorage_bucket_lifecycle_management_enabled.metadata.json b/prowler/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/cloudstorage_bucket_lifecycle_management_enabled.metadata.json new file mode 100644 index 0000000000..450f2d96c5 --- /dev/null +++ b/prowler/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/cloudstorage_bucket_lifecycle_management_enabled.metadata.json @@ -0,0 +1,34 @@ +{ + "Provider": "gcp", + "CheckID": "cloudstorage_bucket_lifecycle_management_enabled", + "CheckTitle": "Cloud Storage buckets have lifecycle management enabled", + "CheckType": [], + "ServiceName": "cloudstorage", + "SubServiceName": "", + "ResourceIdTemplate": "", + "Severity": "medium", + "ResourceType": "storage.googleapis.com/Bucket", + "Description": "**Google Cloud Storage buckets** are evaluated for the presence of **lifecycle management** with at least one valid rule (supported action and non-empty condition) to automatically transition or delete objects and optimize storage costs.", + "Risk": "Buckets without lifecycle rules can accumulate stale data, increase storage costs, and fail to meet data retention and internal compliance requirements.", + "RelatedUrl": "", + "AdditionalURLs": [ + "https://www.trendmicro.com/cloudoneconformity/knowledge-base/gcp/CloudStorage/enable-lifecycle-management.html", + "https://cloud.google.com/storage/docs/lifecycle" + ], + "Remediation": { + "Code": { + "CLI": "gcloud storage buckets update gs:// --lifecycle-file=", + "NativeIaC": "", + "Other": "1) Open Google Cloud Console → Storage → Buckets → \n2) Tab 'Lifecycle'\n3) Add rule(s) to delete or transition objects (e.g., delete after 365 days; transition STANDARD→NEARLINE after 90 days)\n4) Save", + "Terraform": "```hcl\n# Example: enable lifecycle to transition and delete objects\nresource \"google_storage_bucket\" \"example\" {\n name = var.bucket_name\n location = var.location\n\n # Transition STANDARD → NEARLINE after 90 days\n lifecycle_rule {\n action {\n type = \"SetStorageClass\"\n storage_class = \"NEARLINE\"\n }\n condition {\n age = 90\n matches_storage_class = [\"STANDARD\"]\n }\n }\n\n # Delete objects after 365 days\n lifecycle_rule {\n action {\n type = \"Delete\"\n }\n condition {\n age = 365\n }\n }\n}\n```" + }, + "Recommendation": { + "Text": "Configure lifecycle rules to automatically delete stale objects or transition them to colder storage classes according to your organization's retention and cost-optimization policy.", + "Url": "https://hub.prowler.com/check/cloudstorage_bucket_lifecycle_management_enabled" + } + }, + "Categories": [], + "DependsOn": [], + "RelatedTo": [], + "Notes": "" +} diff --git a/prowler/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/cloudstorage_bucket_lifecycle_management_enabled.py b/prowler/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/cloudstorage_bucket_lifecycle_management_enabled.py new file mode 100644 index 0000000000..951bf57d4f --- /dev/null +++ b/prowler/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/cloudstorage_bucket_lifecycle_management_enabled.py @@ -0,0 +1,48 @@ +from prowler.lib.check.models import Check, Check_Report_GCP +from prowler.providers.gcp.services.cloudstorage.cloudstorage_client import ( + cloudstorage_client, +) + + +class cloudstorage_bucket_lifecycle_management_enabled(Check): + """Ensure Cloud Storage buckets have lifecycle management enabled with at least one valid rule. + + Reports PASS if a bucket has at least one valid lifecycle rule + (with a supported action and condition), otherwise FAIL. + + """ + + def execute(self) -> list[Check_Report_GCP]: + """Run the lifecycle management check for each Cloud Storage bucket. + + Returns: + list[Check_Report_GCP]: Results for all evaluated buckets. + """ + + findings = [] + for bucket in cloudstorage_client.buckets: + report = Check_Report_GCP(metadata=self.metadata(), resource=bucket) + report.status = "FAIL" + report.status_extended = ( + f"Bucket {bucket.name} does not have lifecycle management enabled." + ) + + rules = bucket.lifecycle_rules + + if rules: + valid_rules = [] + for rule in rules: + action_type = rule.get("action", {}).get("type") + condition = rule.get("condition") + if action_type and condition: + valid_rules.append(rule) + + if valid_rules: + report.status = "PASS" + report.status_extended = f"Bucket {bucket.name} has lifecycle management enabled with {len(valid_rules)} valid rule(s)." + else: + report.status = "FAIL" + report.status_extended = f"Bucket {bucket.name} has lifecycle rules configured but none are valid." + + findings.append(report) + return findings diff --git a/prowler/providers/gcp/services/cloudstorage/cloudstorage_service.py b/prowler/providers/gcp/services/cloudstorage/cloudstorage_service.py index 294455077c..7d155d254f 100644 --- a/prowler/providers/gcp/services/cloudstorage/cloudstorage_service.py +++ b/prowler/providers/gcp/services/cloudstorage/cloudstorage_service.py @@ -31,6 +31,14 @@ class CloudStorage(GCPService): bucket_iam ) or "allUsers" in str(bucket_iam): public = True + + lifecycle_rules = None + lifecycle = bucket.get("lifecycle") + if isinstance(lifecycle, dict): + rules = lifecycle.get("rule") + if isinstance(rules, list): + lifecycle_rules = rules + self.buckets.append( Bucket( name=bucket["name"], @@ -42,6 +50,7 @@ class CloudStorage(GCPService): public=public, retention_policy=bucket.get("retentionPolicy"), project_id=project_id, + lifecycle_rules=lifecycle_rules, ) ) @@ -62,3 +71,4 @@ class Bucket(BaseModel): public: bool project_id: str retention_policy: Optional[dict] = None + lifecycle_rules: Optional[list[dict]] = None diff --git a/prowler/providers/github/services/organization/organization_default_repository_permission_strict/__init__.py b/prowler/providers/github/services/organization/organization_default_repository_permission_strict/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/providers/github/services/organization/organization_default_repository_permission_strict/organization_default_repository_permission_strict.metadata.json b/prowler/providers/github/services/organization/organization_default_repository_permission_strict/organization_default_repository_permission_strict.metadata.json new file mode 100644 index 0000000000..7c2e738958 --- /dev/null +++ b/prowler/providers/github/services/organization/organization_default_repository_permission_strict/organization_default_repository_permission_strict.metadata.json @@ -0,0 +1,32 @@ +{ + "Provider": "github", + "CheckID": "organization_default_repository_permission_strict", + "CheckTitle": "Ensure strict base repository permissions are set for the organization", + "CheckType": [], + "ServiceName": "organization", + "SubServiceName": "", + "ResourceIdTemplate": "", + "Severity": "high", + "ResourceType": "GitHubOrganization", + "Description": "Ensure the organization's base repository permission for members is set to 'read' or 'none' to minimize risk.", + "Risk": "If base repository permissions allow 'write' or 'admin' by default, organization members may unintentionally gain excessive privileges across repositories, increasing the risk of unauthorized changes or accidental modifications.", + "RelatedUrl": "https://docs.github.com/en/organizations/managing-user-access-to-your-organizations-repositories/managing-repository-roles/setting-base-permissions-for-an-organization", + "Remediation": { + "Code": { + "CLI": "", + "NativeIaC": "", + "Other": "", + "Terraform": "" + }, + "Recommendation": { + "Text": "Set the organization's base repository permission to 'read' or 'none' for members, unless stricter requirements are needed.", + "Url": "https://docs.github.com/en/organizations/managing-user-access-to-your-organizations-repositories/managing-repository-roles/setting-base-permissions-for-an-organization" + } + }, + "AdditionalURLs": [], + "Categories": [], + "DependsOn": [], + "RelatedTo": [], + "Notes": "" +} + diff --git a/prowler/providers/github/services/organization/organization_default_repository_permission_strict/organization_default_repository_permission_strict.py b/prowler/providers/github/services/organization/organization_default_repository_permission_strict/organization_default_repository_permission_strict.py new file mode 100644 index 0000000000..3d31f00bbc --- /dev/null +++ b/prowler/providers/github/services/organization/organization_default_repository_permission_strict/organization_default_repository_permission_strict.py @@ -0,0 +1,36 @@ +from typing import List + +from prowler.lib.check.models import Check, CheckReportGithub +from prowler.providers.github.services.organization.organization_client import ( + organization_client, +) + + +class organization_default_repository_permission_strict(Check): + """Check if an organization's base repository permission is set to a strict level. + + PASS: base permission is "read" or "none" + FAIL: base permission is "write" or "admin" (or any other non-strict value) + """ + + def execute(self) -> List[CheckReportGithub]: + findings = [] + for org in organization_client.organizations.values(): + base_perm = getattr(org, "base_permission", None) + if base_perm is None: + # Unknown / no permission to read → skip producing a finding + continue + + p = str(base_perm).lower() + report = CheckReportGithub(metadata=self.metadata(), resource=org) + + if p in ("read", "none"): + report.status = "PASS" + report.status_extended = f"Organization {org.name} base repository permission is '{p}', which is strict." + else: + report.status = "FAIL" + report.status_extended = f"Organization {org.name} base repository permission is '{p}', which is not strict." + + findings.append(report) + + return findings diff --git a/prowler/providers/github/services/organization/organization_service.py b/prowler/providers/github/services/organization/organization_service.py index 9bbc5f2671..484ebcd378 100644 --- a/prowler/providers/github/services/organization/organization_service.py +++ b/prowler/providers/github/services/organization/organization_service.py @@ -5,7 +5,7 @@ from pydantic.v1 import BaseModel from prowler.lib.logger import logger from prowler.providers.github.lib.service.service import GithubService -from prowler.providers.github.models import GithubAppIdentityInfo, GithubIdentityInfo +from prowler.providers.github.models import GithubAppIdentityInfo class Organization(GithubService): @@ -38,13 +38,15 @@ class Organization(GithubService): org_names_to_check = set() try: - for client in self.clients: - if self.provider.organizations: + for client in getattr(self, "clients", []) or []: + if getattr(self.provider, "organizations", None): org_names_to_check.update(self.provider.organizations) # If repositories are specified without organizations, don't perform organization checks # Only add repository owners to organization checks if organizations are also specified - if self.provider.repositories and self.provider.organizations: + if getattr(self.provider, "repositories", None) and getattr( + self.provider, "organizations", None + ): for repo_name in self.provider.repositories: if "/" in repo_name: owner_name = repo_name.split("/")[0] @@ -111,18 +113,19 @@ class Organization(GithubService): logger.error( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - elif not self.provider.repositories: + elif not getattr(self.provider, "repositories", None): # Default behavior: get all organizations the user is a member of # Only when no repositories are specified - if isinstance(self.provider.identity, GithubIdentityInfo): + if isinstance(self.provider.identity, GithubAppIdentityInfo): + orgs = client.get_organizations() + if getattr(orgs, "totalCount", 0) > 0: + for org in orgs: + self._process_organization(org, organizations) + else: + # Default (personal access/OAuth): use user organizations orgs = client.get_user().get_orgs() for org in orgs: self._process_organization(org, organizations) - elif isinstance(self.provider.identity, GithubAppIdentityInfo): - orgs = client.get_organizations() - if orgs.totalCount > 0: - for org in orgs: - self._process_organization(org, organizations) except github.RateLimitExceededException as error: logger.error(f"GitHub API rate limit exceeded: {error}") @@ -144,10 +147,22 @@ class Organization(GithubService): logger.error( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) + + # Base permission (default repository permission for members) + base_perm: Optional[str] = None + try: + base_perm = getattr(org, "default_repository_permission", None) + except Exception as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + base_perm = None + organizations[org.id] = Org( id=org.id, name=org.login, mfa_required=require_mfa, + base_permission=base_perm, ) @@ -157,3 +172,4 @@ class Org(BaseModel): id: int name: str mfa_required: Optional[bool] = False + base_permission: Optional[str] = None diff --git a/prowler/providers/m365/services/entra/entra_admin_users_phishing_resistant_mfa_enabled/entra_admin_users_phishing_resistant_mfa_enabled.py b/prowler/providers/m365/services/entra/entra_admin_users_phishing_resistant_mfa_enabled/entra_admin_users_phishing_resistant_mfa_enabled.py index 6f0fbc281d..647eb8176a 100644 --- a/prowler/providers/m365/services/entra/entra_admin_users_phishing_resistant_mfa_enabled/entra_admin_users_phishing_resistant_mfa_enabled.py +++ b/prowler/providers/m365/services/entra/entra_admin_users_phishing_resistant_mfa_enabled/entra_admin_users_phishing_resistant_mfa_enabled.py @@ -2,7 +2,6 @@ from prowler.lib.check.models import Check, CheckReportM365 from prowler.providers.m365.services.entra.entra_client import entra_client from prowler.providers.m365.services.entra.entra_service import ( AdminRoles, - AuthenticationStrength, ConditionalAccessPolicyState, ) @@ -47,7 +46,25 @@ class entra_admin_users_phishing_resistant_mfa_enabled(Check): if ( policy.grant_controls.authentication_strength is not None and policy.grant_controls.authentication_strength - == AuthenticationStrength.PHISHING_RESISTANT_MFA + != "Multifactor authentication" + and policy.grant_controls.authentication_strength != "Passwordless MFA" + and policy.grant_controls.authentication_strength + != "Phishing-resistant MFA" + ): + report = CheckReportM365( + metadata=self.metadata(), + resource=policy, + resource_name=policy.display_name, + resource_id=policy.id, + ) + report.status = "MANUAL" + report.status_extended = f"Conditional Access Policy '{policy.display_name}' has a custom authentication strength, review it is Phishing-resistant MFA." + continue + + if ( + policy.grant_controls.authentication_strength is not None + and policy.grant_controls.authentication_strength + == "Phishing-resistant MFA" ): report = CheckReportM365( metadata=self.metadata(), diff --git a/prowler/providers/m365/services/entra/entra_service.py b/prowler/providers/m365/services/entra/entra_service.py index 7c566b484a..e7751fdef9 100644 --- a/prowler/providers/m365/services/entra/entra_service.py +++ b/prowler/providers/m365/services/entra/entra_service.py @@ -253,9 +253,7 @@ class Entra(M365Service): ) ), authentication_strength=( - AuthenticationStrength( - policy.grant_controls.authentication_strength.display_name - ) + policy.grant_controls.authentication_strength.display_name if policy.grant_controls is not None and policy.grant_controls.authentication_strength is not None @@ -455,6 +453,7 @@ class ConditionalAccessPolicyState(Enum): class UserAction(Enum): REGISTER_SECURITY_INFO = "urn:user:registersecurityinfo" + REGISTER_DEVICE = "urn:user:registerdevice" class ApplicationsConditions(BaseModel): @@ -523,11 +522,19 @@ class SessionControls(BaseModel): class ConditionalAccessGrantControl(Enum): + """ + Built-in grant controls for Conditional Access policies. + Reference: https://learn.microsoft.com/en-us/graph/api/resources/conditionalaccessgrantcontrols + """ + MFA = "mfa" BLOCK = "block" DOMAIN_JOINED_DEVICE = "domainJoinedDevice" PASSWORD_CHANGE = "passwordChange" COMPLIANT_DEVICE = "compliantDevice" + APPROVED_APPLICATION = "approvedApplication" + COMPLIANT_APPLICATION = "compliantApplication" + TERMS_OF_USE = "termsOfUse" class GrantControlOperator(Enum): @@ -535,16 +542,10 @@ class GrantControlOperator(Enum): OR = "OR" -class AuthenticationStrength(Enum): - MFA = "Multifactor authentication" - PASSWORDLESS_MFA = "Passwordless MFA" - PHISHING_RESISTANT_MFA = "Phishing-resistant MFA" - - class GrantControls(BaseModel): built_in_controls: List[ConditionalAccessGrantControl] operator: GrantControlOperator - authentication_strength: Optional[AuthenticationStrength] + authentication_strength: Optional[str] class ConditionalAccessPolicy(BaseModel): diff --git a/pyproject.toml b/pyproject.toml index 7b7934011c..0af7b5c479 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ maintainers = [{name = "Prowler Engineering", email = "engineering@prowler.com"} name = "prowler" readme = "README.md" requires-python = ">3.9.1,<3.13" -version = "5.13.0" +version = "5.14.0" [project.scripts] prowler = "prowler.__main__:prowler" diff --git a/tests/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/cloudstorage_bucket_lifecycle_management_enabled_test.py b/tests/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/cloudstorage_bucket_lifecycle_management_enabled_test.py new file mode 100644 index 0000000000..17016645a7 --- /dev/null +++ b/tests/providers/gcp/services/cloudstorage/cloudstorage_bucket_lifecycle_management_enabled/cloudstorage_bucket_lifecycle_management_enabled_test.py @@ -0,0 +1,223 @@ +from unittest import mock + +from tests.providers.gcp.gcp_fixtures import ( + GCP_PROJECT_ID, + GCP_US_CENTER1_LOCATION, + set_mocked_gcp_provider, +) + + +class TestCloudStorageBucketLifecycleManagementEnabled: + def test_bucket_without_lifecycle_rules(self): + cloudstorage_client = mock.MagicMock() + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_gcp_provider(), + ), + mock.patch( + "prowler.providers.gcp.services.cloudstorage.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_client", + new=cloudstorage_client, + ), + ): + from prowler.providers.gcp.services.cloudstorage.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_bucket_lifecycle_management_enabled import ( + cloudstorage_bucket_lifecycle_management_enabled, + ) + from prowler.providers.gcp.services.cloudstorage.cloudstorage_service import ( + Bucket, + ) + + cloudstorage_client.project_ids = [GCP_PROJECT_ID] + cloudstorage_client.region = GCP_US_CENTER1_LOCATION + + cloudstorage_client.buckets = [ + Bucket( + name="no-lifecycle", + id="no-lifecycle", + region=GCP_US_CENTER1_LOCATION, + uniform_bucket_level_access=True, + public=False, + retention_policy=None, + project_id=GCP_PROJECT_ID, + lifecycle_rules=[], + ) + ] + + check = cloudstorage_bucket_lifecycle_management_enabled() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == f"Bucket {cloudstorage_client.buckets[0].name} does not have lifecycle management enabled." + ) + assert result[0].resource_id == "no-lifecycle" + assert result[0].resource_name == "no-lifecycle" + assert result[0].location == GCP_US_CENTER1_LOCATION + assert result[0].project_id == GCP_PROJECT_ID + + def test_bucket_with_minimal_delete_rule(self): + cloudstorage_client = mock.MagicMock() + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_gcp_provider(), + ), + mock.patch( + "prowler.providers.gcp.services.cloudstorage.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_client", + new=cloudstorage_client, + ), + ): + from prowler.providers.gcp.services.cloudstorage.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_bucket_lifecycle_management_enabled import ( + cloudstorage_bucket_lifecycle_management_enabled, + ) + from prowler.providers.gcp.services.cloudstorage.cloudstorage_service import ( + Bucket, + ) + + cloudstorage_client.project_ids = [GCP_PROJECT_ID] + cloudstorage_client.region = GCP_US_CENTER1_LOCATION + + cloudstorage_client.buckets = [ + Bucket( + name="delete-rule", + id="delete-rule", + region=GCP_US_CENTER1_LOCATION, + uniform_bucket_level_access=True, + public=False, + retention_policy=None, + project_id=GCP_PROJECT_ID, + lifecycle_rules=[ + {"action": {"type": "Delete"}, "condition": {"age": 30}} + ], + ) + ] + + check = cloudstorage_bucket_lifecycle_management_enabled() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == f"Bucket {cloudstorage_client.buckets[0].name} has lifecycle management enabled with 1 valid rule(s)." + ) + assert result[0].resource_id == "delete-rule" + assert result[0].resource_name == "delete-rule" + assert result[0].location == GCP_US_CENTER1_LOCATION + assert result[0].project_id == GCP_PROJECT_ID + + def test_bucket_with_transition_and_delete_rules(self): + cloudstorage_client = mock.MagicMock() + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_gcp_provider(), + ), + mock.patch( + "prowler.providers.gcp.services.cloudstorage.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_client", + new=cloudstorage_client, + ), + ): + from prowler.providers.gcp.services.cloudstorage.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_bucket_lifecycle_management_enabled import ( + cloudstorage_bucket_lifecycle_management_enabled, + ) + from prowler.providers.gcp.services.cloudstorage.cloudstorage_service import ( + Bucket, + ) + + cloudstorage_client.project_ids = [GCP_PROJECT_ID] + cloudstorage_client.region = GCP_US_CENTER1_LOCATION + + cloudstorage_client.buckets = [ + Bucket( + name="transition-delete", + id="transition-delete", + region=GCP_US_CENTER1_LOCATION, + uniform_bucket_level_access=True, + public=False, + retention_policy=None, + project_id=GCP_PROJECT_ID, + lifecycle_rules=[ + { + "action": { + "type": "SetStorageClass", + "storageClass": "NEARLINE", + }, + "condition": {"matchesStorageClass": ["STANDARD"]}, + }, + {"action": {"type": "Delete"}, "condition": {"age": 365}}, + ], + ) + ] + + check = cloudstorage_bucket_lifecycle_management_enabled() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == f"Bucket {cloudstorage_client.buckets[0].name} has lifecycle management enabled with 2 valid rule(s)." + ) + assert result[0].resource_id == "transition-delete" + assert result[0].resource_name == "transition-delete" + assert result[0].location == GCP_US_CENTER1_LOCATION + assert result[0].project_id == GCP_PROJECT_ID + + def test_bucket_with_invalid_lifecycle_rules(self): + cloudstorage_client = mock.MagicMock() + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_gcp_provider(), + ), + mock.patch( + "prowler.providers.gcp.services.cloudstorage.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_client", + new=cloudstorage_client, + ), + ): + from prowler.providers.gcp.services.cloudstorage.cloudstorage_bucket_lifecycle_management_enabled.cloudstorage_bucket_lifecycle_management_enabled import ( + cloudstorage_bucket_lifecycle_management_enabled, + ) + from prowler.providers.gcp.services.cloudstorage.cloudstorage_service import ( + Bucket, + ) + + cloudstorage_client.project_ids = [GCP_PROJECT_ID] + cloudstorage_client.region = GCP_US_CENTER1_LOCATION + + cloudstorage_client.buckets = [ + Bucket( + name="invalid-rules", + id="invalid-rules", + region=GCP_US_CENTER1_LOCATION, + uniform_bucket_level_access=True, + public=False, + retention_policy=None, + project_id=GCP_PROJECT_ID, + lifecycle_rules=[ + {"action": {}, "condition": {"age": 30}}, + {"action": {"type": "Delete"}, "condition": {}}, + ], + ) + ] + + check = cloudstorage_bucket_lifecycle_management_enabled() + result = check.execute() + + assert len(result) == 1 + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == f"Bucket {cloudstorage_client.buckets[0].name} has lifecycle rules configured but none are valid." + ) + assert result[0].resource_id == "invalid-rules" + assert result[0].resource_name == "invalid-rules" + assert result[0].location == GCP_US_CENTER1_LOCATION + assert result[0].project_id == GCP_PROJECT_ID diff --git a/tests/providers/github/services/organization/organization_default_repository_permission_strict/organization_default_repository_permission_strict_test.py b/tests/providers/github/services/organization/organization_default_repository_permission_strict/organization_default_repository_permission_strict_test.py new file mode 100644 index 0000000000..c1001a44e1 --- /dev/null +++ b/tests/providers/github/services/organization/organization_default_repository_permission_strict/organization_default_repository_permission_strict_test.py @@ -0,0 +1,201 @@ +from unittest import mock + +from prowler.providers.github.services.organization.organization_service import Org +from tests.providers.github.github_fixtures import set_mocked_github_provider + + +class Test_organization_default_repository_permission_strict: + def test_no_organizations(self): + organization_client = mock.MagicMock + organization_client.organizations = {} + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_github_provider(), + ), + mock.patch( + "prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict.organization_client", + new=organization_client, + ), + ): + from prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict import ( + organization_default_repository_permission_strict, + ) + + check = organization_default_repository_permission_strict() + result = check.execute() + assert len(result) == 0 + + def test_permission_read(self): + organization_client = mock.MagicMock + org_name = "test-organization" + organization_client.organizations = { + 1: Org( + id=1, + name=org_name, + base_permission="read", + ), + } + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_github_provider(), + ), + mock.patch( + "prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict.organization_client", + new=organization_client, + ), + ): + from prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict import ( + organization_default_repository_permission_strict, + ) + + check = organization_default_repository_permission_strict() + result = check.execute() + assert len(result) == 1 + assert result[0].resource_id == 1 + assert result[0].resource_name == org_name + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == f"Organization {org_name} base repository permission is 'read', which is strict." + ) + + def test_permission_none(self): + organization_client = mock.MagicMock + org_name = "test-organization" + organization_client.organizations = { + 1: Org( + id=1, + name=org_name, + base_permission="none", + ), + } + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_github_provider(), + ), + mock.patch( + "prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict.organization_client", + new=organization_client, + ), + ): + from prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict import ( + organization_default_repository_permission_strict, + ) + + check = organization_default_repository_permission_strict() + result = check.execute() + assert len(result) == 1 + assert result[0].resource_id == 1 + assert result[0].resource_name == org_name + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == f"Organization {org_name} base repository permission is 'none', which is strict." + ) + + def test_permission_write(self): + organization_client = mock.MagicMock + org_name = "test-organization" + organization_client.organizations = { + 1: Org( + id=1, + name=org_name, + base_permission="write", + ), + } + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_github_provider(), + ), + mock.patch( + "prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict.organization_client", + new=organization_client, + ), + ): + from prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict import ( + organization_default_repository_permission_strict, + ) + + check = organization_default_repository_permission_strict() + result = check.execute() + assert len(result) == 1 + assert result[0].resource_id == 1 + assert result[0].resource_name == org_name + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == f"Organization {org_name} base repository permission is 'write', which is not strict." + ) + + def test_permission_admin(self): + organization_client = mock.MagicMock + org_name = "test-organization" + organization_client.organizations = { + 1: Org( + id=1, + name=org_name, + base_permission="admin", + ), + } + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_github_provider(), + ), + mock.patch( + "prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict.organization_client", + new=organization_client, + ), + ): + from prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict import ( + organization_default_repository_permission_strict, + ) + + check = organization_default_repository_permission_strict() + result = check.execute() + assert len(result) == 1 + assert result[0].resource_id == 1 + assert result[0].resource_name == org_name + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == f"Organization {org_name} base repository permission is 'admin', which is not strict." + ) + + def test_permission_unknown_none_skipped(self): + organization_client = mock.MagicMock + org_name = "test-organization" + organization_client.organizations = { + 1: Org( + id=1, + name=org_name, + base_permission=None, + ), + } + + with ( + mock.patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=set_mocked_github_provider(), + ), + mock.patch( + "prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict.organization_client", + new=organization_client, + ), + ): + from prowler.providers.github.services.organization.organization_default_repository_permission_strict.organization_default_repository_permission_strict import ( + organization_default_repository_permission_strict, + ) + + check = organization_default_repository_permission_strict() + result = check.execute() + assert len(result) == 0 diff --git a/tests/providers/github/services/organization/organization_service_test.py b/tests/providers/github/services/organization/organization_service_test.py index 45b85198de..22b65e3267 100644 --- a/tests/providers/github/services/organization/organization_service_test.py +++ b/tests/providers/github/services/organization/organization_service_test.py @@ -45,11 +45,13 @@ class Test_Organization_Scoping: self.mock_org1.id = 1 self.mock_org1.login = "test-org1" self.mock_org1.two_factor_requirement_enabled = True + self.mock_org1.default_repository_permission = None self.mock_org2 = MagicMock() self.mock_org2.id = 2 self.mock_org2.login = "test-org2" self.mock_org2.two_factor_requirement_enabled = False + self.mock_org2.default_repository_permission = None self.mock_user = MagicMock() self.mock_user.id = 100 @@ -175,6 +177,35 @@ class Test_Organization_Scoping: assert len(orgs) == 0 + def test_base_permission_extraction(self): + """Test that base_permission is populated from organization's default_repository_permission""" + provider = set_mocked_github_provider() + provider.repositories = [] + provider.organizations = ["test-org1"] + + mock_client = MagicMock() + # Organization with default_repository_permission set to "read" + org_with_perm = MagicMock() + org_with_perm.id = 1 + org_with_perm.login = "test-org1" + org_with_perm.two_factor_requirement_enabled = True + org_with_perm.default_repository_permission = "read" + mock_client.get_organization.return_value = org_with_perm + + with patch( + "prowler.providers.github.services.organization.organization_service.GithubService.__init__" + ): + organization_service = Organization(provider) + organization_service.clients = [mock_client] + organization_service.provider = provider + + orgs = organization_service._list_organizations() + + assert len(orgs) == 1 + assert 1 in orgs + assert orgs[1].name == "test-org1" + assert orgs[1].base_permission == "read" + def test_specific_organization_scoping(self): """Test that only specified organizations are returned""" provider = set_mocked_github_provider() @@ -287,11 +318,13 @@ class Test_Organization_Scoping: mock_owner_org.id = 1 mock_owner_org.login = "owner1" mock_owner_org.two_factor_requirement_enabled = True + mock_owner_org.default_repository_permission = None mock_specific_org = MagicMock() mock_specific_org.id = 2 mock_specific_org.login = "specific-org" mock_specific_org.two_factor_requirement_enabled = False + mock_specific_org.default_repository_permission = None mock_client.get_organization.side_effect = [ mock_owner_org, @@ -393,6 +426,7 @@ class Test_Organization_ErrorHandling: self.mock_org1.id = 1 self.mock_org1.login = "test-org1" self.mock_org1.two_factor_requirement_enabled = True + self.mock_org1.default_repository_permission = None def test_github_api_error_handling(self): """Test that GitHub API errors are handled properly""" diff --git a/tests/providers/m365/services/entra/entra_admin_users_phishing_resistant_mfa_enabled/entra_admin_users_phishing_resistant_mfa_enabled_test.py b/tests/providers/m365/services/entra/entra_admin_users_phishing_resistant_mfa_enabled/entra_admin_users_phishing_resistant_mfa_enabled_test.py index aa9ec89bca..b7ac4c2d12 100644 --- a/tests/providers/m365/services/entra/entra_admin_users_phishing_resistant_mfa_enabled/entra_admin_users_phishing_resistant_mfa_enabled_test.py +++ b/tests/providers/m365/services/entra/entra_admin_users_phishing_resistant_mfa_enabled/entra_admin_users_phishing_resistant_mfa_enabled_test.py @@ -3,7 +3,6 @@ from uuid import uuid4 from prowler.providers.m365.services.entra.entra_service import ( ApplicationsConditions, - AuthenticationStrength, ConditionalAccessGrantControl, ConditionalAccessPolicyState, Conditions, @@ -114,7 +113,7 @@ class Test_entra_admin_users_phishing_resistant_mfa_enabled: grant_controls=GrantControls( built_in_controls=[ConditionalAccessGrantControl.BLOCK], operator=GrantControlOperator.AND, - authentication_strength=AuthenticationStrength.PHISHING_RESISTANT_MFA, + authentication_strength="Phishing-resistant MFA", ), session_controls=SessionControls( persistent_browser=PersistentBrowser( @@ -206,7 +205,7 @@ class Test_entra_admin_users_phishing_resistant_mfa_enabled: grant_controls=GrantControls( built_in_controls=[ConditionalAccessGrantControl.BLOCK], operator=GrantControlOperator.AND, - authentication_strength=AuthenticationStrength.PHISHING_RESISTANT_MFA, + authentication_strength="Phishing-resistant MFA", ), session_controls=SessionControls( persistent_browser=PersistentBrowser( @@ -301,7 +300,7 @@ class Test_entra_admin_users_phishing_resistant_mfa_enabled: grant_controls=GrantControls( built_in_controls=[ConditionalAccessGrantControl.BLOCK], operator=GrantControlOperator.AND, - authentication_strength=AuthenticationStrength.PHISHING_RESISTANT_MFA, + authentication_strength="Phishing-resistant MFA", ), session_controls=SessionControls( persistent_browser=PersistentBrowser( diff --git a/tests/providers/m365/services/entra/microsoft365_entra_service_test.py b/tests/providers/m365/services/entra/microsoft365_entra_service_test.py index 9ff7e31b43..9848507b93 100644 --- a/tests/providers/m365/services/entra/microsoft365_entra_service_test.py +++ b/tests/providers/m365/services/entra/microsoft365_entra_service_test.py @@ -7,7 +7,6 @@ from prowler.providers.m365.services.entra.entra_service import ( AdminConsentPolicy, AdminRoles, ApplicationsConditions, - AuthenticationStrength, AuthorizationPolicy, AuthPolicyRoles, ConditionalAccessGrantControl, @@ -75,7 +74,7 @@ async def mock_entra_get_conditional_access_policies(_): ConditionalAccessGrantControl.COMPLIANT_DEVICE, ], operator=GrantControlOperator.OR, - authentication_strength=AuthenticationStrength.PHISHING_RESISTANT_MFA, + authentication_strength="Phishing-resistant MFA", ), session_controls=SessionControls( persistent_browser=PersistentBrowser( @@ -226,7 +225,7 @@ class Test_Entra_Service: ConditionalAccessGrantControl.COMPLIANT_DEVICE, ], operator=GrantControlOperator.OR, - authentication_strength=AuthenticationStrength.PHISHING_RESISTANT_MFA, + authentication_strength="Phishing-resistant MFA", ), session_controls=SessionControls( persistent_browser=PersistentBrowser( diff --git a/ui/CHANGELOG.md b/ui/CHANGELOG.md index d37f3a7548..3354d76f9a 100644 --- a/ui/CHANGELOG.md +++ b/ui/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to the **Prowler UI** are documented in this file. --- -## [1.13.0] (Prowler UNRELEASED) +## [1.13.0] (Prowler v5.13.0) ### 🚀 Added @@ -20,6 +20,7 @@ All notable changes to the **Prowler UI** are documented in this file. - React Compiler support for automatic optimization [(#8748)](https://github.com/prowler-cloud/prowler/pull/8748) - Turbopack support for faster development builds [(#8748)](https://github.com/prowler-cloud/prowler/pull/8748) - Add compliance name in compliance detail view [(#8775)](https://github.com/prowler-cloud/prowler/pull/8775) +- PDF reporting for Prowler ThreatScore [(#8867)](https://github.com/prowler-cloud/prowler/pull/8867) - Support C5 compliance framework for the AWS provider [(#8830)](https://github.com/prowler-cloud/prowler/pull/8830) - API key management in user profile [(#8308)](https://github.com/prowler-cloud/prowler/pull/8308) - Refresh access token error handling [(#8864)](https://github.com/prowler-cloud/prowler/pull/8864) diff --git a/ui/actions/scans/scans.ts b/ui/actions/scans/scans.ts index c17f056f3f..6dc3654344 100644 --- a/ui/actions/scans/scans.ts +++ b/ui/actions/scans/scans.ts @@ -268,3 +268,45 @@ export const getComplianceCsv = async ( }; } }; + +export const getThreatScorePdf = async (scanId: string) => { + const headers = await getAuthHeaders({ contentType: false }); + + const url = new URL(`${apiBaseUrl}/scans/${scanId}/threatscore`); + + try { + const response = await fetch(url.toString(), { headers }); + + if (response.status === 202) { + const json = await response.json(); + const taskId = json?.data?.id; + const state = json?.data?.attributes?.state; + return { + pending: true, + state, + taskId, + }; + } + + if (!response.ok) { + const errorData = await response.json(); + throw new Error( + errorData?.errors?.detail || + "Unable to retrieve ThreatScore PDF report. Contact support if the issue continues.", + ); + } + + const arrayBuffer = await response.arrayBuffer(); + const base64 = Buffer.from(arrayBuffer).toString("base64"); + + return { + success: true, + data: base64, + filename: `scan-${scanId}-threatscore.pdf`, + }; + } catch (error) { + return { + error: getErrorMessage(error), + }; + } +}; diff --git a/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx b/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx index 2ea1bc823b..2b7cfd8cfd 100644 --- a/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx +++ b/ui/app/(prowler)/compliance/[compliancetitle]/page.tsx @@ -29,6 +29,8 @@ import { } from "@/types/compliance"; import { ScanEntity } from "@/types/scans"; +import { ThreatScoreDownloadButton } from "./threatscore-download-button"; + interface ComplianceDetailSearchParams { complianceId: string; version?: string; @@ -143,13 +145,24 @@ export default async function ComplianceDetail({ )} - +
+
+ +
+ {attributesData?.data?.[0]?.attributes?.framework === + "ProwlerThreatScore" && + selectedScanId && ( +
+ +
+ )} +
{ + const [isDownloading, setIsDownloading] = useState(false); + + const handleDownload = async () => { + setIsDownloading(true); + try { + await downloadThreatScorePdf(scanId, toast); + } finally { + setIsDownloading(false); + } + }; + + return ( + + ); +}; diff --git a/ui/app/(prowler)/compliance/page.tsx b/ui/app/(prowler)/compliance/page.tsx index de94cf0fe2..a4bde8104f 100644 --- a/ui/app/(prowler)/compliance/page.tsx +++ b/ui/app/(prowler)/compliance/page.tsx @@ -1,16 +1,22 @@ export const dynamic = "force-dynamic"; import { Suspense } from "react"; -import { getCompliancesOverview } from "@/actions/compliances"; -import { getComplianceOverviewMetadataInfo } from "@/actions/compliances"; +import { + getComplianceAttributes, + getComplianceOverviewMetadataInfo, + getComplianceRequirements, + getCompliancesOverview, +} from "@/actions/compliances"; import { getScans } from "@/actions/scans"; import { ComplianceCard, ComplianceSkeletonGrid, NoScansAvailable, + ThreatScoreBadge, } from "@/components/compliance"; import { ComplianceHeader } from "@/components/compliance/compliance-header/compliance-header"; import { ContentLayout } from "@/components/ui"; +import { calculateThreatScore } from "@/lib/compliance/threatscore-calculator"; import { ExpandedScanData, ScanEntity, @@ -74,6 +80,7 @@ export default async function Compliance({ }) .filter(Boolean) as ExpandedScanData[]; + // Use scanId from URL, or select the first scan if not provided const selectedScanId = resolvedSearchParams.scanId || expandedScansData[0]?.id || null; const query = (filters["filter[search]"] as string) || ""; @@ -94,6 +101,7 @@ export default async function Compliance({ } : undefined; + // Fetch metadata if we have a selected scan const metadataInfoData = selectedScanId ? await getComplianceOverviewMetadataInfo({ query, @@ -105,14 +113,52 @@ export default async function Compliance({ const uniqueRegions = metadataInfoData?.data?.attributes?.regions || []; + // Fetch ThreatScore data if we have a selected scan + let threatScoreData = null; + if ( + selectedScanId && + typeof selectedScanId === "string" && + selectedScan?.providerInfo?.provider + ) { + const complianceId = `prowler_threatscore_${selectedScan.providerInfo.provider.toLowerCase()}`; + + const [attributesData, requirementsData] = await Promise.all([ + getComplianceAttributes(complianceId), + getComplianceRequirements({ + complianceId, + scanId: selectedScanId, + }), + ]); + + threatScoreData = calculateThreatScore(attributesData, requirementsData); + } + return ( {selectedScanId ? ( <> - +
+
+
+ +
+ {threatScoreData && + typeof selectedScanId === "string" && + selectedScan && ( +
+ +
+ )} +
+
}> - {compliancesData.data.map((compliance: ComplianceOverviewData) => { - const { attributes, id } = compliance; - const { framework, version, requirements_passed, total_requirements } = - attributes; + {compliancesData.data + .filter((compliance: ComplianceOverviewData) => { + // Filter out ProwlerThreatScore from the grid + return compliance.attributes.framework !== "ProwlerThreatScore"; + }) + .map((compliance: ComplianceOverviewData) => { + const { attributes, id } = compliance; + const { + framework, + version, + requirements_passed, + total_requirements, + } = attributes; - return ( - - ); - })} + return ( + + ); + })} ); }; diff --git a/ui/app/(prowler)/new-overview/components/check-findings.tsx b/ui/app/(prowler)/new-overview/components/check-findings.tsx new file mode 100644 index 0000000000..e57390afcf --- /dev/null +++ b/ui/app/(prowler)/new-overview/components/check-findings.tsx @@ -0,0 +1,134 @@ +"use client"; + +import { Bell, BellOff, ShieldCheck, TriangleAlert } from "lucide-react"; + +import { DonutChart } from "@/components/graphs/donut-chart"; +import { DonutDataPoint } from "@/components/graphs/types"; +import { + BaseCard, + CardContent, + CardHeader, + CardTitle, + ResourceStatsCard, + StatsContainer, +} from "@/components/shadcn"; +import { CardVariant } from "@/components/shadcn/card/resource-stats-card/resource-stats-card-content"; + +interface CheckFindingsProps { + failFindingsData: { + total: number; + new: number; + muted: number; + }; + passFindingsData: { + total: number; + new: number; + muted: number; + }; +} + +export const CheckFindings = ({ + failFindingsData, + passFindingsData, +}: CheckFindingsProps) => { + // Calculate total findings + const totalFindings = failFindingsData.total + passFindingsData.total; + + // Calculate percentages + const failPercentage = Math.round( + (failFindingsData.total / totalFindings) * 100, + ); + const passPercentage = Math.round( + (passFindingsData.total / totalFindings) * 100, + ); + + // Calculate change percentages (new findings as percentage change) + const failChange = + failFindingsData.total > 0 + ? Math.round((failFindingsData.new / failFindingsData.total) * 100) + : 0; + const passChange = + passFindingsData.total > 0 + ? Math.round((passFindingsData.new / passFindingsData.total) * 100) + : 0; + + // Mock data for DonutChart + const donutData: DonutDataPoint[] = [ + { + name: "Fail Findings", + value: failFindingsData.total, + color: "#f43f5e", // Rose-500 + percentage: Number(failPercentage), + change: Number(failChange), + }, + { + name: "Pass Findings", + value: passFindingsData.total, + color: "#4ade80", // Green-400 + percentage: Number(passPercentage), + change: Number(passChange), + }, + ]; + + return ( + + {/* Header */} + + Check Findings + + + {/* DonutChart Content */} + +
+ +
+ + {/* Footer with ResourceStatsCards */} + + + +
+
+
+ + + + + + ); +}; diff --git a/ui/app/(prowler)/new-overview/page.tsx b/ui/app/(prowler)/new-overview/page.tsx new file mode 100644 index 0000000000..71fc62a024 --- /dev/null +++ b/ui/app/(prowler)/new-overview/page.tsx @@ -0,0 +1,87 @@ +import { Suspense } from "react"; + +import { getFindingsByStatus } from "@/actions/overview/overview"; +import { ContentLayout } from "@/components/ui"; +import { SearchParamsProps } from "@/types"; + +import { CheckFindings } from "./components/check-findings"; + +const FILTER_PREFIX = "filter["; + +// Extract only query params that start with "filter[" for API calls +function pickFilterParams( + params: SearchParamsProps | undefined | null, +): Record { + if (!params) return {}; + return Object.fromEntries( + Object.entries(params).filter(([key]) => key.startsWith(FILTER_PREFIX)), + ); +} + +export default async function NewOverviewPage({ + searchParams, +}: { + searchParams: Promise; +}) { + const resolvedSearchParams = await searchParams; + + return ( + +
+ +

Loading...

+
+ } + > + + +
+
+ ); +} + +const SSRCheckFindings = async ({ + searchParams, +}: { + searchParams: SearchParamsProps | undefined | null; +}) => { + const filters = pickFilterParams(searchParams); + + const findingsByStatus = await getFindingsByStatus({ filters }); + + if (!findingsByStatus) { + return ( +
+

Failed to load findings data

+
+ ); + } + + const { + fail = 0, + pass = 0, + muted_new = 0, + muted_changed = 0, + fail_new = 0, + pass_new = 0, + } = findingsByStatus?.data?.attributes || {}; + + const mutedTotal = muted_new + muted_changed; + + return ( + + ); +}; diff --git a/ui/components/compliance/compliance-header/data-compliance.tsx b/ui/components/compliance/compliance-header/data-compliance.tsx index 7d29ebfe7d..992dc3d683 100644 --- a/ui/components/compliance/compliance-header/data-compliance.tsx +++ b/ui/components/compliance/compliance-header/data-compliance.tsx @@ -19,11 +19,13 @@ export const DataCompliance = ({ scans }: DataComplianceProps) => { const selectedScanId = scanIdParam || (scans.length > 0 ? scans[0].id : ""); + // Don't auto-push scanId to URL - the server already handles the default scan selection + // This avoids duplicate API calls caused by client-side navigation useEffect(() => { if (!scanIdParam && scans.length > 0) { const params = new URLSearchParams(searchParams); params.set("scanId", scans[0].id); - router.push(`?${params.toString()}`); + router.replace(`?${params.toString()}`, { scroll: false }); } }, [scans, scanIdParam, searchParams, router]); diff --git a/ui/components/compliance/index.ts b/ui/components/compliance/index.ts index 096951a261..4365f4ec65 100644 --- a/ui/components/compliance/index.ts +++ b/ui/components/compliance/index.ts @@ -19,3 +19,5 @@ export * from "./skeletons/compliance-accordion-skeleton"; export * from "./skeletons/compliance-grid-skeleton"; export * from "./skeletons/heatmap-chart-skeleton"; export * from "./skeletons/pie-chart-skeleton"; +export * from "./threatscore-badge"; +export * from "./threatscore-logo"; diff --git a/ui/components/compliance/threatscore-badge.tsx b/ui/components/compliance/threatscore-badge.tsx new file mode 100644 index 0000000000..b9ba9d5b7b --- /dev/null +++ b/ui/components/compliance/threatscore-badge.tsx @@ -0,0 +1,148 @@ +"use client"; + +import { Button } from "@heroui/button"; +import { Card, CardBody } from "@heroui/card"; +import { Progress } from "@heroui/progress"; +import { DownloadIcon, FileTextIcon } from "lucide-react"; +import { useRouter, useSearchParams } from "next/navigation"; +import { useState } from "react"; + +import { ThreatScoreLogo } from "@/components/compliance/threatscore-logo"; +import { toast } from "@/components/ui"; +import { downloadComplianceCsv, downloadThreatScorePdf } from "@/lib/helper"; +import type { ScanEntity } from "@/types/scans"; + +interface ThreatScoreBadgeProps { + score: number; + scanId: string; + provider: string; + selectedScan?: ScanEntity; +} + +export const ThreatScoreBadge = ({ + score, + scanId, + provider, + selectedScan, +}: ThreatScoreBadgeProps) => { + const router = useRouter(); + const searchParams = useSearchParams(); + const [isDownloadingPdf, setIsDownloadingPdf] = useState(false); + const [isDownloadingCsv, setIsDownloadingCsv] = useState(false); + + const complianceId = `prowler_threatscore_${provider.toLowerCase()}`; + + const getScoreColor = (): "success" | "warning" | "danger" => { + if (score >= 80) return "success"; + if (score >= 40) return "warning"; + return "danger"; + }; + + const getTextColor = () => { + if (score >= 80) return "text-success"; + if (score >= 40) return "text-warning"; + return "text-danger"; + }; + + const handleCardClick = () => { + const title = "ProwlerThreatScore"; + const version = "1.0"; + const formattedTitleForUrl = encodeURIComponent(title); + const path = `/compliance/${formattedTitleForUrl}`; + const params = new URLSearchParams(); + + params.set("complianceId", complianceId); + params.set("version", version); + params.set("scanId", scanId); + + if (selectedScan) { + params.set( + "scanData", + JSON.stringify({ + id: selectedScan.id, + providerInfo: selectedScan.providerInfo, + attributes: selectedScan.attributes, + }), + ); + } + + const regionFilter = searchParams.get("filter[region__in]"); + if (regionFilter) { + params.set("filter[region__in]", regionFilter); + } + + router.push(`${path}?${params.toString()}`); + }; + + const handleDownloadPdf = async () => { + setIsDownloadingPdf(true); + try { + await downloadThreatScorePdf(scanId, toast); + } finally { + setIsDownloadingPdf(false); + } + }; + + const handleDownloadCsv = async () => { + setIsDownloadingCsv(true); + try { + await downloadComplianceCsv(scanId, complianceId, toast); + } finally { + setIsDownloadingCsv(false); + } + }; + + return ( + + + +
+ + +
+
+
+ ); +}; diff --git a/ui/components/compliance/threatscore-logo.tsx b/ui/components/compliance/threatscore-logo.tsx new file mode 100644 index 0000000000..d4b98d4ccc --- /dev/null +++ b/ui/components/compliance/threatscore-logo.tsx @@ -0,0 +1,79 @@ +"use client"; + +import { useTheme } from "next-themes"; +import { useEffect, useState } from "react"; + +export const ThreatScoreLogo = () => { + const { resolvedTheme } = useTheme(); + const [mounted, setMounted] = useState(false); + + // Avoid hydration mismatch by only rendering after mount + useEffect(() => { + setMounted(true); + }, []); + + if (!mounted) { + return
; + } + + const prowlerColor = resolvedTheme === "dark" ? "#fff" : "#000"; + + return ( + + {/* Prowler logo from the new SVG - scaled and positioned to match THREATSCORE size */} + + + + + {/* THREATSCORE text */} + + THREATSCORE + + + {/* Gauge icon - semicircular meter - 1.5x larger */} + + {/* Gauge arcs - drawing from left to right (orange, red, green) */} + + + + + {/* Checkmark */} + + + + ); +}; diff --git a/ui/components/graphs/BarChart.tsx b/ui/components/graphs/BarChart.tsx deleted file mode 100644 index 8ebb7eca3b..0000000000 --- a/ui/components/graphs/BarChart.tsx +++ /dev/null @@ -1,162 +0,0 @@ -"use client"; - -import { - Bar, - BarChart as RechartsBar, - CartesianGrid, - Cell, - ResponsiveContainer, - Tooltip, - XAxis, - YAxis, -} from "recharts"; - -import { ChartTooltip } from "./shared/ChartTooltip"; -import { CHART_COLORS, LAYOUT_OPTIONS } from "./shared/constants"; -import { getSeverityColorByName } from "./shared/utils"; -import { BarDataPoint, LayoutOption } from "./types"; - -interface BarChartProps { - data: BarDataPoint[]; - layout?: LayoutOption; - xLabel?: string; - yLabel?: string; - height?: number; - showValues?: boolean; -} - -const CustomLabel = ({ x, y, width, height, value, data }: any) => { - const percentage = data.percentage; - return ( - - {percentage !== undefined - ? `${percentage}% • ${value.toLocaleString()}` - : value.toLocaleString()} - - ); -}; - -export function BarChart({ - data, - layout = LAYOUT_OPTIONS.horizontal, - xLabel, - yLabel, - height = 400, - showValues = true, -}: BarChartProps) { - const isHorizontal = layout === LAYOUT_OPTIONS.horizontal; - - return ( - - - - {isHorizontal ? ( - <> - - - - ) : ( - <> - - - - )} - } /> - ( - - ) - : false - } - > - {data.map((entry, index) => ( - - ))} - - - - ); -} diff --git a/ui/components/graphs/SankeyChart.tsx b/ui/components/graphs/SankeyChart.tsx deleted file mode 100644 index d9c35cd1df..0000000000 --- a/ui/components/graphs/SankeyChart.tsx +++ /dev/null @@ -1,137 +0,0 @@ -"use client"; - -import { Rectangle, ResponsiveContainer, Sankey, Tooltip } from "recharts"; - -import { CHART_COLORS, SEVERITY_COLORS } from "./shared/constants"; - -interface SankeyNode { - name: string; -} - -interface SankeyLink { - source: number; - target: number; - value: number; -} - -interface SankeyChartProps { - data: { - nodes: SankeyNode[]; - links: SankeyLink[]; - }; - height?: number; -} - -const COLORS: Record = { - Success: "var(--color-success)", - Fail: "var(--color-destructive)", - AWS: "var(--color-orange)", - Azure: "var(--color-cyan)", - Google: "var(--color-red)", - ...SEVERITY_COLORS, -}; - -const CustomTooltip = ({ active, payload }: any) => { - if (active && payload && payload.length) { - const data = payload[0].payload; - return ( -
-

{data.name}

- {data.value && ( -

Value: {data.value}

- )} -
- ); - } - return null; -}; - -const CustomNode = ({ x, y, width, height, payload, containerWidth }: any) => { - const isOut = x + width + 6 > containerWidth; - const nodeName = payload.name; - const color = COLORS[nodeName] || CHART_COLORS.defaultColor; - - return ( - - - - {nodeName} - - - {payload.value} - - - ); -}; - -const CustomLink = (props: any) => { - const { - sourceX, - targetX, - sourceY, - targetY, - sourceControlX, - targetControlX, - linkWidth, - } = props; - - const sourceName = props.payload.source?.name || ""; - const color = COLORS[sourceName] || CHART_COLORS.defaultColor; - - return ( - - - - ); -}; - -export function SankeyChart({ data, height = 400 }: SankeyChartProps) { - return ( - - } - link={} - nodePadding={50} - margin={{ top: 20, right: 160, bottom: 20, left: 160 }} - > - } /> - - - ); -} diff --git a/ui/components/graphs/DonutChart.tsx b/ui/components/graphs/donut-chart.tsx similarity index 72% rename from ui/components/graphs/DonutChart.tsx rename to ui/components/graphs/donut-chart.tsx index 57713cc2f6..b82387944f 100644 --- a/ui/components/graphs/DonutChart.tsx +++ b/ui/components/graphs/donut-chart.tsx @@ -5,7 +5,7 @@ import { Cell, Label, Pie, PieChart, Tooltip } from "recharts"; import { ChartConfig, ChartContainer } from "@/components/ui/chart/Chart"; -import { ChartLegend } from "./shared/ChartLegend"; +import { ChartLegend } from "./shared/chart-legend"; import { DonutDataPoint } from "./types"; interface DonutChartProps { @@ -21,32 +21,39 @@ interface DonutChartProps { } const CustomTooltip = ({ active, payload }: any) => { - if (active && payload && payload.length) { - const data = payload[0].payload; - return ( -
-
-
- - {data.percentage}% {data.name} - -
- {data.change !== undefined && ( -

- - {data.change > 0 ? "+" : ""} - {data.change}% - {" "} - Since last scan -

- )} + if (!active || !payload || !payload.length) return null; + + const entry = payload[0]; + const name = entry.name; + const percentage = entry.payload?.percentage; + const color = entry.color || entry.payload?.color; + const change = entry.payload?.change; + + return ( +
+
+
+ + {percentage}% + + {name}
- ); - } - return null; +

+ {change !== undefined && ( + <> + + {change > 0 ? "+" : ""} + {change}% + + Since Last Scan + + )} +

+
+ ); }; const CustomLegend = ({ payload }: any) => { @@ -60,8 +67,8 @@ const CustomLegend = ({ payload }: any) => { export function DonutChart({ data, - innerRadius = 80, - outerRadius = 120, + innerRadius = 68, + outerRadius = 86, showLegend = true, centerLabel, }: DonutChartProps) { @@ -96,7 +103,7 @@ export function DonutChart({ })); return ( -
+ <> {chartData.map((entry, index) => { const opacity = @@ -145,14 +152,20 @@ export function DonutChart({ {formattedValue} {centerLabel.label} @@ -166,6 +179,6 @@ export function DonutChart({ {showLegend && } -
+ ); } diff --git a/ui/components/graphs/HorizontalBarChart.tsx b/ui/components/graphs/horizontal-bar-chart.tsx similarity index 70% rename from ui/components/graphs/HorizontalBarChart.tsx rename to ui/components/graphs/horizontal-bar-chart.tsx index b91e575656..c792d60d38 100644 --- a/ui/components/graphs/HorizontalBarChart.tsx +++ b/ui/components/graphs/horizontal-bar-chart.tsx @@ -26,7 +26,12 @@ export function HorizontalBarChart({ data, title }: HorizontalBarChartProps) {
{title && (
-

{title}

+

+ {title} +

)} @@ -48,8 +53,9 @@ export function HorizontalBarChart({ data, title }: HorizontalBarChartProps) { >
{isHovered && ( -
+
- + {item.value.toLocaleString()} {item.name} Risk
{item.newFindings !== undefined && (
- - + + {item.newFindings} New Findings
)} {item.change !== undefined && ( -

+

{item.change > 0 ? "+" : ""} {item.change}% @@ -102,15 +126,16 @@ export function HorizontalBarChart({ data, title }: HorizontalBarChartProps) {

{item.percentage}% - • - {item.value.toLocaleString()} + • + {item.value.toLocaleString()}
); diff --git a/ui/components/graphs/index.ts b/ui/components/graphs/index.ts index f0f0e53697..79d9b01e53 100644 --- a/ui/components/graphs/index.ts +++ b/ui/components/graphs/index.ts @@ -1,9 +1,9 @@ -export { BarChart } from "./BarChart"; -export { DonutChart } from "./DonutChart"; -export { HorizontalBarChart } from "./HorizontalBarChart"; -export { LineChart } from "./LineChart"; -export { RadarChart } from "./RadarChart"; -export { RadialChart } from "./RadialChart"; -export { SankeyChart } from "./SankeyChart"; -export { ScatterPlot } from "./ScatterPlot"; -export { ChartLegend, type ChartLegendItem } from "./shared/ChartLegend"; +export { DonutChart } from "./donut-chart"; +export { HorizontalBarChart } from "./horizontal-bar-chart"; +export { LineChart } from "./line-chart"; +export { MapChart, type MapChartData, type MapChartProps } from "./map-chart"; +export { RadarChart } from "./radar-chart"; +export { RadialChart } from "./radial-chart"; +export { SankeyChart } from "./sankey-chart"; +export { ScatterPlot } from "./scatter-plot"; +export { ChartLegend, type ChartLegendItem } from "./shared/chart-legend"; diff --git a/ui/components/graphs/LineChart.tsx b/ui/components/graphs/line-chart.tsx similarity index 82% rename from ui/components/graphs/LineChart.tsx rename to ui/components/graphs/line-chart.tsx index b19c9591f5..daefb55808 100644 --- a/ui/components/graphs/LineChart.tsx +++ b/ui/components/graphs/line-chart.tsx @@ -14,8 +14,8 @@ import { YAxis, } from "recharts"; -import { AlertPill } from "./shared/AlertPill"; -import { ChartLegend } from "./shared/ChartLegend"; +import { AlertPill } from "./shared/alert-pill"; +import { ChartLegend } from "./shared/chart-legend"; import { CHART_COLORS } from "./shared/constants"; import { LineConfig, LineDataPoint } from "./types"; @@ -48,8 +48,19 @@ const CustomLineTooltip = ({ const totalValue = typedPayload.reduce((sum, item) => sum + item.value, 0); return ( -
-

{label}

+
+

+ {label} +

@@ -67,18 +78,29 @@ const CustomLineTooltip = ({ className="h-2 w-2 rounded-full" style={{ backgroundColor: item.stroke }} /> - {item.value} + + {item.value} +
{newFindings !== undefined && (
- - + + {newFindings} New Findings
)} {change !== undefined && typeof change === "number" && ( -

+

{change > 0 ? "+" : ""} {change}% diff --git a/ui/components/graphs/map-chart.tsx b/ui/components/graphs/map-chart.tsx new file mode 100644 index 0000000000..586bb47b25 --- /dev/null +++ b/ui/components/graphs/map-chart.tsx @@ -0,0 +1,479 @@ +"use client"; + +import * as d3 from "d3"; +import type { + Feature, + FeatureCollection, + GeoJsonProperties, + Geometry, +} from "geojson"; +import { AlertTriangle, Info, MapPin } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; +import { feature } from "topojson-client"; +import type { + GeometryCollection, + Objects, + Topology, +} from "topojson-specification"; + +import { HorizontalBarChart } from "./horizontal-bar-chart"; +import { BarDataPoint } from "./types"; + +// Constants +const MAP_CONFIG = { + defaultWidth: 688, + defaultHeight: 400, + pointRadius: 6, + selectedPointRadius: 8, + transitionDuration: 300, +} as const; + +const MAP_COLORS = { + landFill: "var(--chart-border-emphasis)", + landStroke: "var(--chart-border)", + pointDefault: "#DB2B49", + pointSelected: "#86DA26", + pointHover: "#DB2B49", +} as const; + +const RISK_LEVELS = { + LOW_HIGH: "low-high", + HIGH: "high", + CRITICAL: "critical", +} as const; + +type RiskLevel = (typeof RISK_LEVELS)[keyof typeof RISK_LEVELS]; + +interface LocationPoint { + id: string; + name: string; + region: string; + coordinates: [number, number]; + totalFindings: number; + riskLevel: RiskLevel; + severityData: BarDataPoint[]; + change?: number; +} + +export interface MapChartData { + locations: LocationPoint[]; + regions: string[]; +} + +export interface MapChartProps { + data: MapChartData; + height?: number; + onLocationSelect?: (location: LocationPoint | null) => void; +} + +// Utility functions +function createProjection(width: number, height: number) { + return d3 + .geoNaturalEarth1() + .fitExtent( + [ + [1, 1], + [width - 1, height - 1], + ], + { type: "Sphere" }, + ) + .precision(0.2); +} + +async function fetchWorldData(): Promise { + try { + const worldAtlasModule = await import("world-atlas/countries-110m.json"); + const worldData = worldAtlasModule.default || worldAtlasModule; + const topology = worldData as unknown as Topology; + return feature( + topology, + topology.objects.countries as GeometryCollection, + ) as FeatureCollection; + } catch (error) { + console.error("Error loading world map data:", error); + return null; + } +} + +// Helper: Create SVG element +function createSVGElement( + type: string, + attributes: Record, +): T { + const element = document.createElementNS( + "http://www.w3.org/2000/svg", + type, + ) as T; + Object.entries(attributes).forEach(([key, value]) => { + element.setAttribute(key, value); + }); + return element; +} + +// Components +function MapTooltip({ + location, + position, +}: { + location: LocationPoint; + position: { x: number; y: number }; +}) { + const CHART_COLORS = { + tooltipBorder: "var(--chart-border-emphasis)", + tooltipBackground: "var(--chart-background)", + textPrimary: "var(--chart-text-primary)", + textSecondary: "var(--chart-text-secondary)", + }; + + return ( +

+
+ + + {location.name} + +
+
+ + + {location.totalFindings.toLocaleString()} Fail Findings + +
+ {location.change !== undefined && ( +

+ + {location.change > 0 ? "+" : ""} + {location.change}% + {" "} + since last scan +

+ )} +
+ ); +} + +function EmptyState() { + const CHART_COLORS = { + tooltipBorder: "var(--chart-border-emphasis)", + tooltipBackground: "var(--chart-background)", + textSecondary: "var(--chart-text-secondary)", + }; + + return ( +
+
+ +

+ Select a location on the map to view details +

+
+
+ ); +} + +function LoadingState({ height }: { height: number }) { + const CHART_COLORS = { + textSecondary: "var(--chart-text-secondary)", + }; + + return ( +
+
+
+ Loading map... +
+
+
+ ); +} + +export function MapChart({ + data, + height = MAP_CONFIG.defaultHeight, +}: MapChartProps) { + const svgRef = useRef(null); + const containerRef = useRef(null); + const [selectedLocation, setSelectedLocation] = + useState(null); + const [hoveredLocation, setHoveredLocation] = useState( + null, + ); + const [tooltipPosition, setTooltipPosition] = useState<{ + x: number; + y: number; + } | null>(null); + const [worldData, setWorldData] = useState(null); + const [isLoadingMap, setIsLoadingMap] = useState(true); + const [dimensions, setDimensions] = useState<{ + width: number; + height: number; + }>({ + width: MAP_CONFIG.defaultWidth, + height, + }); + + // Fetch world data once on mount + useEffect(() => { + let isMounted = true; + fetchWorldData() + .then((data) => { + if (isMounted && data) setWorldData(data); + }) + .catch(console.error) + .finally(() => { + if (isMounted) setIsLoadingMap(false); + }); + return () => { + isMounted = false; + }; + }, []); + + // Update dimensions on resize + useEffect(() => { + const updateDimensions = () => { + if (containerRef.current) { + setDimensions({ width: containerRef.current.clientWidth, height }); + } + }; + updateDimensions(); + window.addEventListener("resize", updateDimensions); + return () => window.removeEventListener("resize", updateDimensions); + }, [height]); + + // Render the map + useEffect(() => { + if (!svgRef.current || !worldData || isLoadingMap) return; + + const svg = svgRef.current; + const { width, height } = dimensions; + svg.innerHTML = ""; + + const projection = createProjection(width, height); + const path = d3.geoPath().projection(projection); + + // Render countries + const mapGroup = createSVGElement("g", { + class: "map-countries", + }); + worldData.features?.forEach( + (feature: Feature) => { + const pathData = path(feature); + if (pathData) { + const pathElement = createSVGElement("path", { + d: pathData, + fill: MAP_COLORS.landFill, + stroke: MAP_COLORS.landStroke, + "stroke-width": "0.5", + }); + mapGroup.appendChild(pathElement); + } + }, + ); + svg.appendChild(mapGroup); + + // Helper to update tooltip position + const updateTooltip = (e: MouseEvent) => { + const rect = svg.getBoundingClientRect(); + setTooltipPosition({ + x: e.clientX - rect.left, + y: e.clientY - rect.top, + }); + }; + + // Helper to create circle + const createCircle = (location: LocationPoint) => { + const projected = projection(location.coordinates); + if (!projected) return null; + + const [x, y] = projected; + if (x < 0 || x > width || y < 0 || y > height) return null; + + const isSelected = selectedLocation?.id === location.id; + const isHovered = hoveredLocation?.id === location.id; + const classes = ["cursor-pointer"]; + + if (isSelected) classes.push("drop-shadow-[0_0_8px_#86da26]"); + if (isHovered && !isSelected) classes.push("opacity-70"); + + const circle = createSVGElement("circle", { + cx: x.toString(), + cy: y.toString(), + r: (isSelected + ? MAP_CONFIG.selectedPointRadius + : MAP_CONFIG.pointRadius + ).toString(), + fill: isSelected ? MAP_COLORS.pointSelected : MAP_COLORS.pointDefault, + class: classes.join(" "), + }); + + circle.addEventListener("click", () => + setSelectedLocation(isSelected ? null : location), + ); + circle.addEventListener("mouseenter", (e) => { + setHoveredLocation(location); + updateTooltip(e); + }); + circle.addEventListener("mousemove", updateTooltip); + circle.addEventListener("mouseleave", () => { + setHoveredLocation(null); + setTooltipPosition(null); + }); + + return circle; + }; + + // Render points + const pointsGroup = createSVGElement("g", { + class: "threat-points", + }); + + // Unselected points first + data.locations.forEach((location) => { + if (selectedLocation?.id !== location.id) { + const circle = createCircle(location); + if (circle) pointsGroup.appendChild(circle); + } + }); + + // Selected point last (on top) + if (selectedLocation) { + const selectedData = data.locations.find( + (loc) => loc.id === selectedLocation.id, + ); + if (selectedData) { + const circle = createCircle(selectedData); + if (circle) pointsGroup.appendChild(circle); + } + } + + svg.appendChild(pointsGroup); + }, [ + data.locations, + dimensions, + selectedLocation, + hoveredLocation, + worldData, + isLoadingMap, + ]); + + const CHART_COLORS = { + tooltipBorder: "var(--chart-border-emphasis)", + tooltipBackground: "var(--chart-background)", + textPrimary: "var(--chart-text-primary)", + textSecondary: "var(--chart-text-secondary)", + }; + + return ( +
+ {/* Map Section */} +
+

+ Threat Map +

+ +
+ {isLoadingMap ? ( + + ) : ( + <> +
+ + {hoveredLocation && tooltipPosition && ( + + )} +
+
+
+ + {data.locations.length} Locations + +
+ + )} +
+
+ + {/* Details Section */} +
+
+ {selectedLocation ? ( +
+
+
+
+

+ {selectedLocation.name} +

+
+

+ {selectedLocation.totalFindings.toLocaleString()} Total Findings +

+
+ +
+ ) : ( + + )} +
+
+ ); +} diff --git a/ui/components/graphs/map-region-filter.tsx b/ui/components/graphs/map-region-filter.tsx new file mode 100644 index 0000000000..3b483d1c62 --- /dev/null +++ b/ui/components/graphs/map-region-filter.tsx @@ -0,0 +1,50 @@ +"use client"; + +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "../ui/select/Select"; + +interface MapRegionFilterProps { + regions: string[]; + selectedRegion: string; + onRegionChange: (region: string) => void; + chartColors: { + tooltipBorder: string; + tooltipBackground: string; + textPrimary: string; + }; +} + +export function MapRegionFilter({ + regions, + selectedRegion, + onRegionChange, + chartColors, +}: MapRegionFilterProps) { + return ( + + ); +} diff --git a/ui/components/graphs/RadarChart.tsx b/ui/components/graphs/radar-chart.tsx similarity index 80% rename from ui/components/graphs/RadarChart.tsx rename to ui/components/graphs/radar-chart.tsx index 462c29dd68..ab3a43e6da 100644 --- a/ui/components/graphs/RadarChart.tsx +++ b/ui/components/graphs/radar-chart.tsx @@ -13,7 +13,7 @@ import { ChartTooltip, } from "@/components/ui/chart/Chart"; -import { AlertPill } from "./shared/AlertPill"; +import { AlertPill } from "./shared/alert-pill"; import { CHART_COLORS } from "./shared/constants"; import { RadarDataPoint } from "./types"; @@ -28,7 +28,7 @@ interface RadarChartProps { const chartConfig = { value: { label: "Findings", - color: "var(--color-magenta)", + color: "var(--chart-radar-primary)", }, } satisfies ChartConfig; @@ -36,15 +36,27 @@ const CustomTooltip = ({ active, payload }: any) => { if (active && payload && payload.length) { const data = payload[0]; return ( -
-

+

+

{data.payload.category}

{data.payload.change !== undefined && ( -

+

{data.payload.change > 0 ? "+" : ""} {data.payload.change}% @@ -84,8 +96,11 @@ const CustomDot = (props: any) => { cx={cx} cy={cy} r={isSelected ? 9 : 6} - fill={isSelected ? "var(--color-success)" : "var(--color-purple-dark)"} + fill={ + isSelected ? "var(--chart-success-color)" : "var(--chart-radar-primary)" + } fillOpacity={1} + className={isSelected ? "drop-shadow-[0_0_8px_#86da26]" : ""} style={{ cursor: onSelectPoint ? "pointer" : "default", pointerEvents: "all", @@ -117,7 +132,7 @@ export function RadarChart({ {percentage}% diff --git a/ui/components/graphs/sankey-chart.tsx b/ui/components/graphs/sankey-chart.tsx new file mode 100644 index 0000000000..58179aadf6 --- /dev/null +++ b/ui/components/graphs/sankey-chart.tsx @@ -0,0 +1,403 @@ +"use client"; + +import { useState } from "react"; +import { Rectangle, ResponsiveContainer, Sankey, Tooltip } from "recharts"; + +import { ChartTooltip } from "./shared/chart-tooltip"; +import { CHART_COLORS } from "./shared/constants"; + +interface SankeyNode { + name: string; + newFindings?: number; + change?: number; +} + +interface SankeyLink { + source: number; + target: number; + value: number; +} + +interface SankeyChartProps { + data: { + nodes: SankeyNode[]; + links: SankeyLink[]; + }; + height?: number; +} + +interface LinkTooltipState { + show: boolean; + x: number; + y: number; + sourceName: string; + targetName: string; + value: number; + color: string; +} + +interface NodeTooltipState { + show: boolean; + x: number; + y: number; + name: string; + value: number; + color: string; + newFindings?: number; + change?: number; +} + +// Note: Using hex colors directly because Recharts SVG fill doesn't resolve CSS variables +const COLORS: Record = { + Success: "#86da26", + Fail: "#db2b49", + AWS: "#ff9900", + Azure: "#00bcd4", + Google: "#EA4335", + Critical: "#971348", + High: "#ff3077", + Medium: "#ff7d19", + Low: "#fdd34f", + Info: "#2e51b2", + Informational: "#2e51b2", +}; + +const CustomTooltip = ({ active, payload }: any) => { + if (active && payload && payload.length) { + const data = payload[0].payload; + return ( +

+

+ {data.name} +

+ {data.value && ( +

+ Value: {data.value} +

+ )} +
+ ); + } + return null; +}; + +const CustomNode = (props: any) => { + const { x, y, width, height, payload, containerWidth } = props; + const isOut = x + width + 6 > containerWidth; + const nodeName = payload.name; + const color = COLORS[nodeName] || CHART_COLORS.defaultColor; + const isHidden = nodeName === ""; + const hasTooltip = !isHidden && payload.newFindings; + + const handleMouseEnter = (e: React.MouseEvent) => { + if (!hasTooltip) return; + + const rect = e.currentTarget.closest("svg") as SVGSVGElement; + if (rect) { + const bbox = rect.getBoundingClientRect(); + props.onNodeHover?.({ + x: e.clientX - bbox.left, + y: e.clientY - bbox.top, + name: nodeName, + value: payload.value, + color, + newFindings: payload.newFindings, + change: payload.change, + }); + } + }; + + const handleMouseMove = (e: React.MouseEvent) => { + if (!hasTooltip) return; + + const rect = e.currentTarget.closest("svg") as SVGSVGElement; + if (rect) { + const bbox = rect.getBoundingClientRect(); + props.onNodeMove?.({ + x: e.clientX - bbox.left, + y: e.clientY - bbox.top, + }); + } + }; + + const handleMouseLeave = () => { + if (!hasTooltip) return; + props.onNodeLeave?.(); + }; + + return ( + + + {!isHidden && ( + <> + + {nodeName} + + + {payload.value} + + + )} + + ); +}; + +const CustomLink = (props: any) => { + const { + sourceX, + targetX, + sourceY, + targetY, + sourceControlX, + targetControlX, + linkWidth, + index, + } = props; + + const sourceName = props.payload.source?.name || ""; + const targetName = props.payload.target?.name || ""; + const value = props.payload.value || 0; + const color = COLORS[sourceName] || CHART_COLORS.defaultColor; + const isHidden = targetName === ""; + + const isHovered = props.hoveredLink !== null && props.hoveredLink === index; + const hasHoveredLink = props.hoveredLink !== null; + + const pathD = ` + M${sourceX},${sourceY + linkWidth / 2} + C${sourceControlX},${sourceY + linkWidth / 2} + ${targetControlX},${targetY + linkWidth / 2} + ${targetX},${targetY + linkWidth / 2} + L${targetX},${targetY - linkWidth / 2} + C${targetControlX},${targetY - linkWidth / 2} + ${sourceControlX},${sourceY - linkWidth / 2} + ${sourceX},${sourceY - linkWidth / 2} + Z + `; + + const getOpacity = () => { + if (isHidden) return "0"; + if (!hasHoveredLink) return "0.4"; + return isHovered ? "0.8" : "0.1"; + }; + + const handleMouseEnter = (e: React.MouseEvent) => { + const rect = e.currentTarget.parentElement?.parentElement + ?.parentElement as unknown as SVGSVGElement; + if (rect) { + const bbox = rect.getBoundingClientRect(); + props.onLinkHover?.(index, { + x: e.clientX - bbox.left, + y: e.clientY - bbox.top, + sourceName, + targetName, + value, + color, + }); + } + }; + + const handleMouseMove = (e: React.MouseEvent) => { + const rect = e.currentTarget.parentElement?.parentElement + ?.parentElement as unknown as SVGSVGElement; + if (rect && isHovered) { + const bbox = rect.getBoundingClientRect(); + props.onLinkMove?.({ + x: e.clientX - bbox.left, + y: e.clientY - bbox.top, + }); + } + }; + + const handleMouseLeave = () => { + props.onLinkLeave?.(); + }; + + return ( + + + + ); +}; + +export function SankeyChart({ data, height = 400 }: SankeyChartProps) { + const [hoveredLink, setHoveredLink] = useState(null); + const [linkTooltip, setLinkTooltip] = useState({ + show: false, + x: 0, + y: 0, + sourceName: "", + targetName: "", + value: 0, + color: "", + }); + + const [nodeTooltip, setNodeTooltip] = useState({ + show: false, + x: 0, + y: 0, + name: "", + value: 0, + color: "", + }); + + const handleLinkHover = ( + index: number, + data: Omit, + ) => { + setHoveredLink(index); + setLinkTooltip({ show: true, ...data }); + }; + + const handleLinkMove = (position: { x: number; y: number }) => { + setLinkTooltip((prev) => ({ + ...prev, + x: position.x, + y: position.y, + })); + }; + + const handleLinkLeave = () => { + setHoveredLink(null); + setLinkTooltip((prev) => ({ ...prev, show: false })); + }; + + const handleNodeHover = (data: Omit) => { + setNodeTooltip({ show: true, ...data }); + }; + + const handleNodeMove = (position: { x: number; y: number }) => { + setNodeTooltip((prev) => ({ + ...prev, + x: position.x, + y: position.y, + })); + }; + + const handleNodeLeave = () => { + setNodeTooltip((prev) => ({ ...prev, show: false })); + }; + + return ( +
+ + + } + link={ + + } + nodePadding={50} + margin={{ top: 20, right: 160, bottom: 20, left: 160 }} + sort={false} + > + } /> + + + {linkTooltip.show && ( +
+ +
+ )} + {nodeTooltip.show && ( +
+ +
+ )} +
+ ); +} diff --git a/ui/components/graphs/ScatterPlot.tsx b/ui/components/graphs/scatter-plot.tsx similarity index 81% rename from ui/components/graphs/ScatterPlot.tsx rename to ui/components/graphs/scatter-plot.tsx index 920f6c0c10..a0725f0dc5 100644 --- a/ui/components/graphs/ScatterPlot.tsx +++ b/ui/components/graphs/scatter-plot.tsx @@ -11,18 +11,11 @@ import { YAxis, } from "recharts"; -import { AlertPill } from "./shared/AlertPill"; -import { ChartLegend } from "./shared/ChartLegend"; +import { AlertPill } from "./shared/alert-pill"; +import { ChartLegend } from "./shared/chart-legend"; import { CHART_COLORS } from "./shared/constants"; import { getSeverityColorByRiskScore } from "./shared/utils"; - -interface ScatterDataPoint { - x: number; - y: number; - provider: string; - name: string; - size?: number; -} +import type { ScatterDataPoint } from "./types"; interface ScatterPlotProps { data: ScatterDataPoint[]; @@ -34,9 +27,9 @@ interface ScatterPlotProps { } const PROVIDER_COLORS = { - AWS: "var(--color-orange)", - Azure: "var(--color-cyan)", - Google: "var(--color-red)", + AWS: "var(--chart-provider-aws)", + Azure: "var(--chart-provider-azure)", + Google: "var(--chart-provider-google)", }; const CustomTooltip = ({ active, payload }: any) => { @@ -45,9 +38,23 @@ const CustomTooltip = ({ active, payload }: any) => { const severityColor = getSeverityColorByRiskScore(data.x); return ( -
-

{data.name}

-

+

+

+ {data.name} +

+

{data.x} Risk Score

@@ -69,7 +76,7 @@ const CustomScatterDot = ({ const isSelected = selectedPoint?.name === payload.name; const size = isSelected ? 18 : 8; const fill = isSelected - ? "var(--color-success)" + ? "#86DA26" : PROVIDER_COLORS[payload.provider as keyof typeof PROVIDER_COLORS] || CHART_COLORS.defaultColor; @@ -79,8 +86,9 @@ const CustomScatterDot = ({ cy={cy} r={size / 2} fill={fill} - stroke={isSelected ? "var(--color-success)" : "transparent"} + stroke={isSelected ? "#86DA26" : "transparent"} strokeWidth={2} + className={isSelected ? "drop-shadow-[0_0_8px_#86da26]" : ""} style={{ cursor: "pointer" }} onClick={() => onSelectPoint?.(payload)} /> diff --git a/ui/components/graphs/shared/AlertPill.tsx b/ui/components/graphs/shared/alert-pill.tsx similarity index 59% rename from ui/components/graphs/shared/AlertPill.tsx rename to ui/components/graphs/shared/alert-pill.tsx index f611b27766..2da8b1903f 100644 --- a/ui/components/graphs/shared/AlertPill.tsx +++ b/ui/components/graphs/shared/alert-pill.tsx @@ -17,13 +17,17 @@ export function AlertPill({ }: AlertPillProps) { return (
-
- +
+ {value} diff --git a/ui/components/graphs/shared/ChartLegend.tsx b/ui/components/graphs/shared/chart-legend.tsx similarity index 59% rename from ui/components/graphs/shared/ChartLegend.tsx rename to ui/components/graphs/shared/chart-legend.tsx index 11066aa54a..2efea3ea85 100644 --- a/ui/components/graphs/shared/ChartLegend.tsx +++ b/ui/components/graphs/shared/chart-legend.tsx @@ -9,14 +9,22 @@ interface ChartLegendProps { export function ChartLegend({ items }: ChartLegendProps) { return ( -
+
{items.map((item, index) => (
- {item.label} + + {item.label} +
))}
diff --git a/ui/components/graphs/shared/ChartTooltip.tsx b/ui/components/graphs/shared/chart-tooltip.tsx similarity index 63% rename from ui/components/graphs/shared/ChartTooltip.tsx rename to ui/components/graphs/shared/chart-tooltip.tsx index 6b4bde27bc..558da987fe 100644 --- a/ui/components/graphs/shared/ChartTooltip.tsx +++ b/ui/components/graphs/shared/chart-tooltip.tsx @@ -3,6 +3,7 @@ import { Bell, VolumeX } from "lucide-react"; import { cn } from "@/lib/utils"; import { TooltipData } from "../types"; +import { CHART_COLORS } from "./constants"; interface ChartTooltipProps { active?: boolean; @@ -27,7 +28,13 @@ export function ChartTooltip({ const color = payload[0].color || data.color; return ( -
+
{showColorIndicator && color && (
)} -

{label || data.name}

+

+ {label || data.name} +

-

+

{typeof data.value === "number" ? data.value.toLocaleString() : data.value} @@ -50,8 +59,8 @@ export function ChartTooltip({ {data.newFindings !== undefined && data.newFindings > 0 && (

- - + + {data.newFindings} New Findings
@@ -59,20 +68,24 @@ export function ChartTooltip({ {data.new !== undefined && data.new > 0 && (
- - {data.new} New + + + {data.new} New +
)} {data.muted !== undefined && data.muted > 0 && (
- - {data.muted} Muted + + + {data.muted} Muted +
)} {data.change !== undefined && ( -

+

{data.change > 0 ? "+" : ""} {data.change}% @@ -97,8 +110,10 @@ export function MultiSeriesChartTooltip({ } return ( -

-

{label}

+
+

+ {label} +

{payload.map((entry: any, index: number) => (
@@ -106,12 +121,14 @@ export function MultiSeriesChartTooltip({ className="h-2 w-2 rounded-full" style={{ backgroundColor: entry.color }} /> - {entry.name}: - + + {entry.name}: + + {entry.value} {entry.payload[`${entry.dataKey}_change`] && ( - + ({entry.payload[`${entry.dataKey}_change`] > 0 ? "+" : ""} {entry.payload[`${entry.dataKey}_change`]}%) diff --git a/ui/components/graphs/shared/constants.ts b/ui/components/graphs/shared/constants.ts index 9ab80b8fa8..4aadd4aac0 100644 --- a/ui/components/graphs/shared/constants.ts +++ b/ui/components/graphs/shared/constants.ts @@ -1,21 +1,33 @@ export const SEVERITY_COLORS = { - Informational: "var(--color-info)", - Low: "var(--color-warning)", - Medium: "var(--color-warning-emphasis)", - High: "var(--color-danger)", - Critical: "var(--color-danger-emphasis)", + Informational: "var(--chart-info)", + Info: "var(--chart-info)", + Low: "var(--chart-warning)", + Medium: "var(--chart-warning-emphasis)", + High: "var(--chart-danger)", + Critical: "var(--chart-danger-emphasis)", +} as const; + +export const PROVIDER_COLORS = { + AWS: "var(--chart-provider-aws)", + Azure: "var(--chart-provider-azure)", + Google: "var(--chart-provider-google)", +} as const; + +export const STATUS_COLORS = { + Success: "var(--chart-success-color)", + Fail: "var(--chart-fail)", } as const; export const CHART_COLORS = { - tooltipBorder: "var(--color-slate-700)", - tooltipBackground: "var(--color-slate-800)", - textPrimary: "var(--color-white)", - textSecondary: "var(--color-slate-400)", - gridLine: "var(--color-slate-700)", + tooltipBorder: "var(--chart-border-emphasis)", + tooltipBackground: "var(--chart-background)", + textPrimary: "var(--chart-text-primary)", + textSecondary: "var(--chart-text-secondary)", + gridLine: "var(--chart-border-emphasis)", backgroundTrack: "rgba(51, 65, 85, 0.5)", // slate-700 with 50% opacity - alertPillBg: "var(--color-alert-pill-bg)", - alertPillText: "var(--color-alert-pill-text)", - defaultColor: "var(--color-slate-500)", // Default fallback color for charts + alertPillBg: "var(--chart-alert-bg)", + alertPillText: "var(--chart-alert-text)", + defaultColor: "#64748b", // slate-500 } as const; export const CHART_DIMENSIONS = { diff --git a/ui/components/graphs/types.ts b/ui/components/graphs/types.ts index 0961a32105..f7a9b02542 100644 --- a/ui/components/graphs/types.ts +++ b/ui/components/graphs/types.ts @@ -36,6 +36,14 @@ export interface RadarDataPoint { change?: number; } +export interface ScatterDataPoint { + x: number; + y: number; + provider: string; + name: string; + size?: number; +} + export interface LineConfig { dataKey: string; color: string; diff --git a/ui/components/primitives.ts b/ui/components/primitives.ts deleted file mode 100644 index 04b1f22594..0000000000 --- a/ui/components/primitives.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { tv } from "tailwind-variants"; - -export const title = tv({ - base: "tracking-tight inline font-semibold", - variants: { - color: { - violet: "from-[#FF1CF7] to-[#b249f8]", - yellow: "from-[#FF705B] to-[#FFB457]", - blue: "from-[#5EA2EF] to-[#0072F5]", - cyan: "from-[#00b7fa] to-[#01cfea]", - green: "from-[#6FEE8D] to-[#17c964]", - pink: "from-[#FF72E1] to-[#F54C7A]", - foreground: "dark:from-[#FFFFFF] dark:to-[#4B4B4B]", - }, - size: { - sm: "text-3xl lg:text-4xl", - md: "text-[2.3rem] lg:text-5xl leading-9", - lg: "text-4xl lg:text-6xl", - }, - fullWidth: { - true: "w-full block", - }, - }, - defaultVariants: { - size: "md", - }, - compoundVariants: [ - { - color: [ - "violet", - "yellow", - "blue", - "cyan", - "green", - "pink", - "foreground", - ], - class: "bg-clip-text text-transparent bg-linear-to-b", - }, - ], -}); - -export const subtitle = tv({ - base: "w-full md:w-1/2 my-2 text-lg lg:text-xl text-default-600 block max-w-full", - variants: { - fullWidth: { - true: "w-full!", - }, - }, - defaultVariants: { - fullWidth: true, - }, -}); diff --git a/ui/components/shadcn/README.md b/ui/components/shadcn/README.md index 1bd28c8883..af3bc0348d 100644 --- a/ui/components/shadcn/README.md +++ b/ui/components/shadcn/README.md @@ -4,13 +4,18 @@ This directory contains all shadcn/ui based components for the Prowler applicati ## Directory Structure +Example of a custom component: + ``` shadcn/ -├── card.tsx # shadcn Card component -├── resource-stats-card/ # Custom ResourceStatsCard built on shadcn -│ ├── resource-stats-card.tsx -│ ├── resource-stats-card.example.tsx -│ └── index.ts +├── card/ +│ ├── base-card/ +│ │ ├── base-card.tsx +│ ├── card/ +│ │ ├── card.tsx +│ └── resource-stats-card/ +│ ├── resource-stats-card.tsx +│ ├── resource-stats-card.example.tsx ├── index.ts # Barrel exports └── README.md ``` diff --git a/ui/components/shadcn/card/base-card/base-card.tsx b/ui/components/shadcn/card/base-card/base-card.tsx new file mode 100644 index 0000000000..946e0cc184 --- /dev/null +++ b/ui/components/shadcn/card/base-card/base-card.tsx @@ -0,0 +1,36 @@ +import { cva, type VariantProps } from "class-variance-authority"; + +import { cn } from "@/lib/utils"; + +import { Card } from "../card"; + +const baseCardVariants = cva("", { + variants: { + variant: { + default: + "border-slate-200 bg-white dark:border-zinc-900 dark:bg-stone-950", + }, + }, + defaultVariants: { + variant: "default", + }, +}); + +interface BaseCardProps + extends React.ComponentProps, + VariantProps {} + +const BaseCard = ({ className, variant, ...props }: BaseCardProps) => { + return ( + + ); +}; + +export { BaseCard }; diff --git a/ui/components/shadcn/card.tsx b/ui/components/shadcn/card/card.tsx similarity index 89% rename from ui/components/shadcn/card.tsx rename to ui/components/shadcn/card/card.tsx index a1b4a7742d..4165221253 100644 --- a/ui/components/shadcn/card.tsx +++ b/ui/components/shadcn/card/card.tsx @@ -1,5 +1,3 @@ -import * as React from "react"; - import { cn } from "@/lib/utils"; function Card({ className, ...props }: React.ComponentProps<"div">) { @@ -20,7 +18,7 @@ function CardHeader({ className, ...props }: React.ComponentProps<"div">) {
) { return (
); diff --git a/ui/components/shadcn/resource-stats-card/resource-stats-card-container.tsx b/ui/components/shadcn/card/resource-stats-card/resource-stats-card-container.tsx similarity index 100% rename from ui/components/shadcn/resource-stats-card/resource-stats-card-container.tsx rename to ui/components/shadcn/card/resource-stats-card/resource-stats-card-container.tsx diff --git a/ui/components/shadcn/resource-stats-card/resource-stats-card-content.tsx b/ui/components/shadcn/card/resource-stats-card/resource-stats-card-content.tsx similarity index 89% rename from ui/components/shadcn/resource-stats-card/resource-stats-card-content.tsx rename to ui/components/shadcn/card/resource-stats-card/resource-stats-card-content.tsx index d165eb7944..7f7c0358d9 100644 --- a/ui/components/shadcn/resource-stats-card/resource-stats-card-content.tsx +++ b/ui/components/shadcn/card/resource-stats-card/resource-stats-card-content.tsx @@ -33,11 +33,11 @@ const badgeVariants = cva( { variants: { variant: { - [CardVariant.default]: "bg-[#535359]", - [CardVariant.fail]: "bg-[#432232]", - [CardVariant.pass]: "bg-[#204237]", - [CardVariant.warning]: "bg-[#3d3520]", - [CardVariant.info]: "bg-[#1e3a5f]", + [CardVariant.default]: "bg-slate-100 dark:bg-[#535359]", + [CardVariant.fail]: "bg-red-100 dark:bg-[#432232]", + [CardVariant.pass]: "bg-green-100 dark:bg-[#204237]", + [CardVariant.warning]: "bg-amber-100 dark:bg-[#3d3520]", + [CardVariant.info]: "bg-blue-100 dark:bg-[#1e3a5f]", }, size: { sm: "px-1 text-xs", @@ -66,7 +66,7 @@ const badgeIconVariants = cva("", { }); const labelTextVariants = cva( - "leading-6 font-semibold text-zinc-300 dark:text-zinc-300", + "leading-6 font-semibold text-slate-900 dark:text-zinc-300 whitespace-nowrap", { variants: { size: { @@ -81,7 +81,7 @@ const labelTextVariants = cva( }, ); -const statIconVariants = cva("text-zinc-300 dark:text-zinc-300", { +const statIconVariants = cva("text-slate-600 dark:text-zinc-300", { variants: { size: { sm: "h-2.5 w-2.5", @@ -95,7 +95,7 @@ const statIconVariants = cva("text-zinc-300 dark:text-zinc-300", { }); const statLabelVariants = cva( - "leading-5 font-medium text-zinc-300 dark:text-zinc-300", + "leading-5 font-medium text-slate-700 dark:text-zinc-300", { variants: { size: { diff --git a/ui/components/shadcn/resource-stats-card/resource-stats-card-divider.tsx b/ui/components/shadcn/card/resource-stats-card/resource-stats-card-divider.tsx similarity index 100% rename from ui/components/shadcn/resource-stats-card/resource-stats-card-divider.tsx rename to ui/components/shadcn/card/resource-stats-card/resource-stats-card-divider.tsx diff --git a/ui/components/shadcn/resource-stats-card/resource-stats-card-header.tsx b/ui/components/shadcn/card/resource-stats-card/resource-stats-card-header.tsx similarity index 100% rename from ui/components/shadcn/resource-stats-card/resource-stats-card-header.tsx rename to ui/components/shadcn/card/resource-stats-card/resource-stats-card-header.tsx diff --git a/ui/components/shadcn/resource-stats-card/resource-stats-card.tsx b/ui/components/shadcn/card/resource-stats-card/resource-stats-card.tsx similarity index 98% rename from ui/components/shadcn/resource-stats-card/resource-stats-card.tsx rename to ui/components/shadcn/card/resource-stats-card/resource-stats-card.tsx index 7acbd103e3..4a1520d44d 100644 --- a/ui/components/shadcn/resource-stats-card/resource-stats-card.tsx +++ b/ui/components/shadcn/card/resource-stats-card/resource-stats-card.tsx @@ -111,7 +111,7 @@ export const ResourceStatsCard = ({ {header && } {emptyState ? (
-

+

{emptyState.message}

@@ -141,7 +141,7 @@ export const ResourceStatsCard = ({ {header && } {emptyState ? (
-

+

{emptyState.message}

diff --git a/ui/components/shadcn/card/stats-container.tsx b/ui/components/shadcn/card/stats-container.tsx new file mode 100644 index 0000000000..9d37a60897 --- /dev/null +++ b/ui/components/shadcn/card/stats-container.tsx @@ -0,0 +1,25 @@ +import { cn } from "@/lib/utils"; + +interface StatsContainerProps extends React.HTMLAttributes { + children: React.ReactNode; +} + +const StatsContainer = ({ + className, + children, + ...props +}: StatsContainerProps) => { + return ( +
+ {children} +
+ ); +}; + +export { StatsContainer }; diff --git a/ui/components/shadcn/index.ts b/ui/components/shadcn/index.ts index 4291e07d5a..d29dc6f1dc 100644 --- a/ui/components/shadcn/index.ts +++ b/ui/components/shadcn/index.ts @@ -1,21 +1,8 @@ -export { - Card, - CardContent, - CardDescription, - CardFooter, - CardHeader, - CardTitle, -} from "./card"; -export { - ResourceStatsCard, - ResourceStatsCardContainer, - type ResourceStatsCardContainerProps, - ResourceStatsCardContent, - type ResourceStatsCardContentProps, - ResourceStatsCardDivider, - type ResourceStatsCardDividerProps, - ResourceStatsCardHeader, - type ResourceStatsCardHeaderProps, - type ResourceStatsCardProps, - type StatItem, -} from "./resource-stats-card"; +export * from "./card/base-card/base-card"; +export * from "./card/card"; +export * from "./card/resource-stats-card/resource-stats-card"; +export * from "./card/resource-stats-card/resource-stats-card-container"; +export * from "./card/resource-stats-card/resource-stats-card-content"; +export * from "./card/resource-stats-card/resource-stats-card-divider"; +export * from "./card/resource-stats-card/resource-stats-card-header"; +export * from "./card/stats-container"; diff --git a/ui/components/shadcn/resource-stats-card/index.ts b/ui/components/shadcn/resource-stats-card/index.ts deleted file mode 100644 index c049987ae0..0000000000 --- a/ui/components/shadcn/resource-stats-card/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -export type { ResourceStatsCardProps } from "./resource-stats-card"; -export { ResourceStatsCard } from "./resource-stats-card"; -export type { ResourceStatsCardContainerProps } from "./resource-stats-card-container"; -export { ResourceStatsCardContainer } from "./resource-stats-card-container"; -export type { - ResourceStatsCardContentProps, - StatItem, -} from "./resource-stats-card-content"; -export { ResourceStatsCardContent } from "./resource-stats-card-content"; -export type { ResourceStatsCardDividerProps } from "./resource-stats-card-divider"; -export { ResourceStatsCardDivider } from "./resource-stats-card-divider"; -export type { ResourceStatsCardHeaderProps } from "./resource-stats-card-header"; -export { ResourceStatsCardHeader } from "./resource-stats-card-header"; diff --git a/ui/components/ui/chart/Chart.tsx b/ui/components/ui/chart/Chart.tsx index fec25668b6..014fbc009d 100644 --- a/ui/components/ui/chart/Chart.tsx +++ b/ui/components/ui/chart/Chart.tsx @@ -70,6 +70,7 @@ const ChartContainer = React.forwardRef< ); }); + ChartContainer.displayName = "Chart"; const ChartStyle = ({ id, config }: { id: string; config: ChartConfig }) => { @@ -184,7 +185,7 @@ const ChartTooltipContent = React.forwardRef<
diff --git a/ui/components/ui/custom/custom-button.tsx b/ui/components/ui/custom/custom-button.tsx index 19d50ba686..8f71aabf24 100644 --- a/ui/components/ui/custom/custom-button.tsx +++ b/ui/components/ui/custom/custom-button.tsx @@ -86,7 +86,7 @@ export const CustomButton = React.forwardRef< ) => (