diff --git a/.env b/.env index 1fbf3a630a..ac2af1631f 100644 --- a/.env +++ b/.env @@ -15,6 +15,13 @@ AUTH_SECRET="N/c6mnaS5+SWq81+819OrzQZlmx1Vxtp/orjttJSmw8=" # Google Tag Manager ID NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID="" +#### MCP Server #### +PROWLER_MCP_VERSION=stable +# For UI and MCP running on docker: +PROWLER_MCP_SERVER_URL=http://mcp-server:8000/mcp +# For UI running on host, MCP in docker: +# PROWLER_MCP_SERVER_URL=http://localhost:8000/mcp + #### Code Review Configuration #### # Enable Claude Code standards validation on pre-push hook # Set to 'true' to validate changes against AGENTS.md standards via Claude Code @@ -131,7 +138,7 @@ SENTRY_RELEASE=local NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT} #### Prowler release version #### -NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.12.2 +NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.16.0 # Social login credentials SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google" diff --git a/.github/actions/trivy-scan/action.yml b/.github/actions/trivy-scan/action.yml index e3c2404748..5eca1266b0 100644 --- a/.github/actions/trivy-scan/action.yml +++ b/.github/actions/trivy-scan/action.yml @@ -87,7 +87,7 @@ runs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: - name: trivy-scan-report-${{ inputs.image-name }} + name: trivy-scan-report-${{ inputs.image-name }}-${{ inputs.image-tag }} path: trivy-report.json retention-days: ${{ inputs.artifact-retention-days }} diff --git a/.github/labeler.yml b/.github/labeler.yml index 9a56691628..fa2c7981f9 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -46,6 +46,11 @@ provider/oci: - changed-files: - any-glob-to-any-file: "prowler/providers/oraclecloud/**" - any-glob-to-any-file: "tests/providers/oraclecloud/**" + +provider/alibabacloud: + - changed-files: + - any-glob-to-any-file: "prowler/providers/alibabacloud/**" + - any-glob-to-any-file: "tests/providers/alibabacloud/**" github_actions: - changed-files: @@ -69,6 +74,8 @@ mutelist: - any-glob-to-any-file: "tests/providers/gcp/lib/mutelist/**" - any-glob-to-any-file: "tests/providers/kubernetes/lib/mutelist/**" - any-glob-to-any-file: "tests/providers/mongodbatlas/lib/mutelist/**" + - any-glob-to-any-file: "tests/providers/oci/lib/mutelist/**" + - any-glob-to-any-file: "tests/providers/alibabacloud/lib/mutelist/**" integration/s3: - changed-files: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 497c4d6214..28365d4acb 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -14,14 +14,26 @@ Please add a detailed description of how to review this PR. ### Checklist -- Are there new checks included in this PR? Yes / No - - If so, do we need to update permissions for the provider? Please review this carefully. +
+ +Community Checklist + +- [ ] This feature/issue is listed in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com +- [ ] Is it assigned to me, if not, request it via the issue/feature in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](goto.prowler.com/slack) + +
+ + - [ ] Review if the code is being covered by tests. - [ ] Review if code is being documented following this specification https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings - [ ] Review if backport is needed. - [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md) - [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable. +#### SDK/CLI +- Are there new checks included in this PR? Yes / No + - If so, do we need to update permissions for the provider? Please review this carefully. + #### UI - [ ] All issue/task requirements work as expected on the UI - [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px) @@ -30,6 +42,11 @@ Please add a detailed description of how to review this PR. - [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable. #### API +- [ ] All issue/task requirements work as expected on the API +- [ ] Endpoint response output (if applicable) +- [ ] EXPLAIN ANALYZE output for new/modified queries or indexes (if applicable) +- [ ] Performance test results (if applicable) +- [ ] Any other relevant evidence of the implementation (if applicable) - [ ] Verify if API specs need to be regenerated. - [ ] Check if version updates are required (e.g., specs, Poetry, etc.). - [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable. diff --git a/.github/scripts/slack-messages/container-release-completed.json b/.github/scripts/slack-messages/container-release-completed.json index c1fc147191..f2b9682022 100644 --- a/.github/scripts/slack-messages/container-release-completed.json +++ b/.github/scripts/slack-messages/container-release-completed.json @@ -1,5 +1,6 @@ { "channel": "${{ env.SLACK_CHANNEL_ID }}", + "ts": "${{ env.MESSAGE_TS }}", "attachments": [ { "color": "${{ env.STATUS_COLOR }}", diff --git a/.github/workflows/api-bump-version.yml b/.github/workflows/api-bump-version.yml new file mode 100644 index 0000000000..97cdd546ff --- /dev/null +++ b/.github/workflows/api-bump-version.yml @@ -0,0 +1,254 @@ +name: 'API: Bump Version' + +on: + release: + types: + - 'published' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false + +env: + PROWLER_VERSION: ${{ github.event.release.tag_name }} + BASE_BRANCH: master + +jobs: + detect-release-type: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + is_minor: ${{ steps.detect.outputs.is_minor }} + is_patch: ${{ steps.detect.outputs.is_patch }} + major_version: ${{ steps.detect.outputs.major_version }} + minor_version: ${{ steps.detect.outputs.minor_version }} + patch_version: ${{ steps.detect.outputs.patch_version }} + current_api_version: ${{ steps.get_api_version.outputs.current_api_version }} + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Get current API version + id: get_api_version + run: | + CURRENT_API_VERSION=$(grep -oP '^version = "\K[^"]+' api/pyproject.toml) + echo "current_api_version=${CURRENT_API_VERSION}" >> "${GITHUB_OUTPUT}" + echo "Current API version: $CURRENT_API_VERSION" + + - name: Detect release type and parse version + id: detect + run: | + if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + MAJOR_VERSION=${BASH_REMATCH[1]} + MINOR_VERSION=${BASH_REMATCH[2]} + PATCH_VERSION=${BASH_REMATCH[3]} + + echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}" + + if (( MAJOR_VERSION != 5 )); then + echo "::error::Releasing another Prowler major version, aborting..." + exit 1 + fi + + if (( PATCH_VERSION == 0 )); then + echo "is_minor=true" >> "${GITHUB_OUTPUT}" + echo "is_patch=false" >> "${GITHUB_OUTPUT}" + echo "✓ Minor release detected: $PROWLER_VERSION" + else + echo "is_minor=false" >> "${GITHUB_OUTPUT}" + echo "is_patch=true" >> "${GITHUB_OUTPUT}" + echo "✓ Patch release detected: $PROWLER_VERSION" + fi + else + echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)" + exit 1 + fi + + bump-minor-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_minor == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Calculate next API minor version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}" + + # API version follows Prowler minor + 1 + # For Prowler 5.17.0 -> API 1.18.0 + # For next master (Prowler 5.18.0) -> API 1.19.0 + NEXT_API_VERSION=1.$((MINOR_VERSION + 2)).0 + + echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}" + echo "NEXT_API_VERSION=${NEXT_API_VERSION}" >> "${GITHUB_ENV}" + + echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0" + echo "Current API version: $CURRENT_API_VERSION" + echo "Next API minor version (for master): $NEXT_API_VERSION" + + - name: Bump API versions in files for master + run: | + set -e + + sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_VERSION}\"|" api/pyproject.toml + sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_VERSION}\"|" api/src/backend/api/v1/views.py + sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_VERSION}|" api/src/backend/api/specs/v1.yaml + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for next API minor version to master + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: master + commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}' + branch: api-version-bump-to-v${{ env.NEXT_API_VERSION }} + title: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}' + labels: no-changelog,skip-sync + body: | + ### Description + + Bump Prowler API version to v${{ env.NEXT_API_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}. + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + - name: Checkout version branch + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }} + + - name: Calculate first API patch version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}" + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} + + # API version follows Prowler minor + 1 + # For Prowler 5.17.0 release -> version branch v5.17 should have API 1.18.1 + FIRST_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).1 + + echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}" + echo "FIRST_API_PATCH_VERSION=${FIRST_API_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" + + echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0" + echo "First API patch version (for ${VERSION_BRANCH}): $FIRST_API_PATCH_VERSION" + echo "Version branch: $VERSION_BRANCH" + + - name: Bump API versions in files for version branch + run: | + set -e + + sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${FIRST_API_PATCH_VERSION}\"|" api/pyproject.toml + sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${FIRST_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py + sed -i "s| version: ${CURRENT_API_VERSION}| version: ${FIRST_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for first API patch version to version branch + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}' + branch: api-version-bump-to-v${{ env.FIRST_API_PATCH_VERSION }} + title: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}' + labels: no-changelog,skip-sync + body: | + ### Description + + Bump Prowler API version to v${{ env.FIRST_API_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}. + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + bump-patch-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_patch == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Calculate next API patch version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }} + CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}" + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} + + # Extract current API patch to increment it + if [[ $CURRENT_API_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + API_PATCH=${BASH_REMATCH[3]} + + # API version follows Prowler minor + 1 + # Keep same API minor (based on Prowler minor), increment patch + NEXT_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).$((API_PATCH + 1)) + + echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}" + echo "NEXT_API_PATCH_VERSION=${NEXT_API_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" + + echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}" + echo "Current API version: $CURRENT_API_VERSION" + echo "Next API patch version: $NEXT_API_PATCH_VERSION" + echo "Target branch: $VERSION_BRANCH" + else + echo "::error::Invalid API version format: $CURRENT_API_VERSION" + exit 1 + fi + + - name: Bump API versions in files for version branch + run: | + set -e + + sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_PATCH_VERSION}\"|" api/pyproject.toml + sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py + sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for next API patch version to version branch + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}' + branch: api-version-bump-to-v${{ env.NEXT_API_PATCH_VERSION }} + title: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}' + labels: no-changelog,skip-sync + body: | + ### Description + + Bump Prowler API version to v${{ env.NEXT_API_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}. + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. diff --git a/.github/workflows/api-code-quality.yml b/.github/workflows/api-code-quality.yml index a9dddaa3aa..c5cc02a298 100644 --- a/.github/workflows/api-code-quality.yml +++ b/.github/workflows/api-code-quality.yml @@ -33,11 +33,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for API changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | api/** @@ -46,6 +46,7 @@ jobs: api/docs/** api/README.md api/CHANGELOG.md + api/AGENTS.md - name: Setup Python with Poetry if: steps.check-changes.outputs.any_changed == 'true' diff --git a/.github/workflows/api-codeql.yml b/.github/workflows/api-codeql.yml index 16ea82538e..45cc911050 100644 --- a/.github/workflows/api-codeql.yml +++ b/.github/workflows/api-codeql.yml @@ -42,15 +42,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Initialize CodeQL - uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/api-codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/api-container-build-push.yml b/.github/workflows/api-container-build-push.yml index 4e56dbbadd..4d8356cc14 100644 --- a/.github/workflows/api-container-build-push.yml +++ b/.github/workflows/api-container-build-push.yml @@ -48,8 +48,34 @@ jobs: id: set-short-sha run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT - container-build-push: + notify-release-started: + if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') needs: setup + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + message-ts: ${{ steps.slack-notification.outputs.ts }} + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Notify container push started + id: slack-notification + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + container-build-push: + needs: [setup, notify-release-started] + if: always() && needs.setup.result == 'success' && (needs.notify-release-started.result == 'success' || needs.notify-release-started.result == 'skipped') runs-on: ${{ matrix.runner }} strategy: matrix: @@ -67,7 +93,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Login to DockerHub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 @@ -76,21 +102,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Notify container push started - if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' - uses: ./.github/actions/slack-notification - env: - SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} - COMPONENT: API - RELEASE_TAG: ${{ env.RELEASE_TAG }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - with: - slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build and push API container for ${{ matrix.arch }} id: container-push @@ -105,36 +117,21 @@ jobs: cache-from: type=gha,scope=${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ matrix.arch }} - - name: Notify container push completed - if: (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && always() - uses: ./.github/actions/slack-notification - env: - SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} - COMPONENT: API - RELEASE_TAG: ${{ env.RELEASE_TAG }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - with: - slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" - step-outcome: ${{ steps.container-push.outcome }} - # Create and push multi-architecture manifest create-manifest: needs: [setup, container-build-push] - if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' + if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest steps: - name: Login to DockerHub - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Create and push manifests for push event if: github.event_name == 'push' @@ -166,9 +163,43 @@ jobs: regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true echo "Cleanup completed" + notify-release-completed: + if: always() && needs.notify-release-started.result == 'success' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') + needs: [setup, notify-release-started, container-build-push, create-manifest] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Determine overall outcome + id: outcome + run: | + if [[ "${{ needs.container-build-push.result }}" == "success" && "${{ needs.create-manifest.result }}" == "success" ]]; then + echo "outcome=success" >> $GITHUB_OUTPUT + else + echo "outcome=failure" >> $GITHUB_OUTPUT + fi + + - name: Notify container push completed + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + MESSAGE_TS: ${{ needs.notify-release-started.outputs.message-ts }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.outcome.outputs.outcome }} + update-ts: ${{ needs.notify-release-started.outputs.message-ts }} + trigger-deployment: - if: github.event_name == 'push' needs: [setup, container-build-push] + if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -176,7 +207,7 @@ jobs: steps: - name: Trigger API deployment - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} repository: ${{ secrets.CLOUD_DISPATCH }} diff --git a/.github/workflows/api-container-checks.yml b/.github/workflows/api-container-checks.yml index d069b0d5a6..e1bca8091c 100644 --- a/.github/workflows/api-container-checks.yml +++ b/.github/workflows/api-container-checks.yml @@ -20,6 +20,7 @@ env: jobs: api-dockerfile-lint: + if: github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -27,11 +28,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check if Dockerfile changed id: dockerfile-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: api/Dockerfile @@ -43,7 +44,17 @@ jobs: ignore: DL3013 api-container-build-and-scan: - runs-on: ubuntu-latest + if: github.repository == 'prowler-cloud/prowler' + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 timeout-minutes: 30 permissions: contents: read @@ -52,38 +63,40 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for API changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: api/** files_ignore: | api/docs/** api/README.md api/CHANGELOG.md + api/AGENTS.md - name: Set up Docker Buildx if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Build container + - name: Build container for ${{ matrix.arch }} if: steps.check-changes.outputs.any_changed == 'true' uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: ${{ env.API_WORKING_DIR }} push: false load: true - tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + platforms: ${{ matrix.platform }} + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} - - name: Scan container with Trivy - if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' + - name: Scan container with Trivy for ${{ matrix.arch }} + if: steps.check-changes.outputs.any_changed == 'true' uses: ./.github/actions/trivy-scan with: image-name: ${{ env.IMAGE_NAME }} - image-tag: ${{ github.sha }} + image-tag: ${{ github.sha }}-${{ matrix.arch }} fail-on-critical: 'false' severity: 'CRITICAL' diff --git a/.github/workflows/api-security.yml b/.github/workflows/api-security.yml index 33134ec9c9..285262ce7f 100644 --- a/.github/workflows/api-security.yml +++ b/.github/workflows/api-security.yml @@ -33,11 +33,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for API changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | api/** @@ -46,6 +46,7 @@ jobs: api/docs/** api/README.md api/CHANGELOG.md + api/AGENTS.md - name: Setup Python with Poetry if: steps.check-changes.outputs.any_changed == 'true' @@ -60,9 +61,7 @@ jobs: - name: Safety if: steps.check-changes.outputs.any_changed == 'true' - # 76352, 76353, 77323 come from SDK, but they cannot upgrade it yet. It does not affect API - # TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X - run: poetry run safety check --ignore 70612,66963,74429,76352,76353,77323,77744,77745 + run: poetry run safety check - name: Vulture if: steps.check-changes.outputs.any_changed == 'true' diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index daeb79abac..ec235fd33b 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -73,11 +73,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for API changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | api/** @@ -86,6 +86,7 @@ jobs: api/docs/** api/README.md api/CHANGELOG.md + api/AGENTS.md - name: Setup Python with Poetry if: steps.check-changes.outputs.any_changed == 'true' @@ -100,7 +101,7 @@ jobs: - name: Upload coverage reports to Codecov if: steps.check-changes.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: diff --git a/.github/workflows/docs-bump-version.yml b/.github/workflows/docs-bump-version.yml new file mode 100644 index 0000000000..dde21b9c1e --- /dev/null +++ b/.github/workflows/docs-bump-version.yml @@ -0,0 +1,247 @@ +name: 'Docs: Bump Version' + +on: + release: + types: + - 'published' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false + +env: + PROWLER_VERSION: ${{ github.event.release.tag_name }} + BASE_BRANCH: master + +jobs: + detect-release-type: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + is_minor: ${{ steps.detect.outputs.is_minor }} + is_patch: ${{ steps.detect.outputs.is_patch }} + major_version: ${{ steps.detect.outputs.major_version }} + minor_version: ${{ steps.detect.outputs.minor_version }} + patch_version: ${{ steps.detect.outputs.patch_version }} + current_docs_version: ${{ steps.get_docs_version.outputs.current_docs_version }} + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Get current documentation version + id: get_docs_version + run: | + CURRENT_DOCS_VERSION=$(grep -oP 'PROWLER_UI_VERSION="\K[^"]+' docs/getting-started/installation/prowler-app.mdx) + echo "current_docs_version=${CURRENT_DOCS_VERSION}" >> "${GITHUB_OUTPUT}" + echo "Current documentation version: $CURRENT_DOCS_VERSION" + + - name: Detect release type and parse version + id: detect + run: | + if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + MAJOR_VERSION=${BASH_REMATCH[1]} + MINOR_VERSION=${BASH_REMATCH[2]} + PATCH_VERSION=${BASH_REMATCH[3]} + + echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}" + + if (( MAJOR_VERSION != 5 )); then + echo "::error::Releasing another Prowler major version, aborting..." + exit 1 + fi + + if (( PATCH_VERSION == 0 )); then + echo "is_minor=true" >> "${GITHUB_OUTPUT}" + echo "is_patch=false" >> "${GITHUB_OUTPUT}" + echo "✓ Minor release detected: $PROWLER_VERSION" + else + echo "is_minor=false" >> "${GITHUB_OUTPUT}" + echo "is_patch=true" >> "${GITHUB_OUTPUT}" + echo "✓ Patch release detected: $PROWLER_VERSION" + fi + else + echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)" + exit 1 + fi + + bump-minor-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_minor == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Calculate next minor version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}" + + NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0 + echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}" + echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}" + + echo "Current documentation version: $CURRENT_DOCS_VERSION" + echo "Current release version: $PROWLER_VERSION" + echo "Next minor version: $NEXT_MINOR_VERSION" + + - name: Bump versions in documentation for master + run: | + set -e + + # Update prowler-app.mdx with current release version + sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx + sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for documentation update to master + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: master + commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}' + branch: docs-version-update-to-v${{ env.PROWLER_VERSION }} + title: 'docs: Update version to v${{ env.PROWLER_VERSION }}' + labels: no-changelog,skip-sync + body: | + ### Description + + Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}. + + ### Files Updated + - `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION` + - All `*.mdx` files with `` components + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + - name: Checkout version branch + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }} + + - name: Calculate first patch version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}" + + FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1 + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} + + echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}" + echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" + + echo "First patch version: $FIRST_PATCH_VERSION" + echo "Version branch: $VERSION_BRANCH" + + - name: Bump versions in documentation for version branch + run: | + set -e + + # Update prowler-app.mdx with current release version + sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx + sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for documentation update to version branch + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}' + branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}-branch + title: 'docs: Update version to v${{ env.PROWLER_VERSION }}' + labels: no-changelog,skip-sync + body: | + ### Description + + Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}. + + ### Files Updated + - `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION` + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + bump-patch-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_patch == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Calculate next patch version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }} + CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}" + + NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1)) + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} + + echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}" + echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" + + echo "Current documentation version: $CURRENT_DOCS_VERSION" + echo "Current release version: $PROWLER_VERSION" + echo "Next patch version: $NEXT_PATCH_VERSION" + echo "Target branch: $VERSION_BRANCH" + + - name: Bump versions in documentation for patch version + run: | + set -e + + # Update prowler-app.mdx with current release version + sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx + sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for documentation update to version branch + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}' + branch: docs-version-update-to-v${{ env.PROWLER_VERSION }} + title: 'docs: Update version to v${{ env.PROWLER_VERSION }}' + labels: no-changelog,skip-sync + body: | + ### Description + + Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}. + + ### Files Updated + - `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION` + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. diff --git a/.github/workflows/find-secrets.yml b/.github/workflows/find-secrets.yml index 80b49e3279..c525d8faa4 100644 --- a/.github/workflows/find-secrets.yml +++ b/.github/workflows/find-secrets.yml @@ -23,11 +23,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 - name: Scan for secrets with TruffleHog - uses: trufflesecurity/trufflehog@b84c3d14d189e16da175e2c27fa8136603783ffc # v3.90.12 + uses: trufflesecurity/trufflehog@ef6e76c3c4023279497fab4721ffa071a722fd05 # v3.92.4 with: extra_args: '--results=verified,unknown' diff --git a/.github/workflows/mcp-container-build-push.yml b/.github/workflows/mcp-container-build-push.yml index b635db1031..130ef90129 100644 --- a/.github/workflows/mcp-container-build-push.yml +++ b/.github/workflows/mcp-container-build-push.yml @@ -47,8 +47,34 @@ jobs: id: set-short-sha run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT - container-build-push: + notify-release-started: + if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') needs: setup + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + message-ts: ${{ steps.slack-notification.outputs.ts }} + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Notify container push started + id: slack-notification + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: MCP + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + container-build-push: + needs: [setup, notify-release-started] + if: always() && needs.setup.result == 'success' && (needs.notify-release-started.result == 'success' || needs.notify-release-started.result == 'skipped') runs-on: ${{ matrix.runner }} strategy: matrix: @@ -65,7 +91,7 @@ jobs: packages: write steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Login to DockerHub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 @@ -74,21 +100,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Notify container push started - if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' - uses: ./.github/actions/slack-notification - env: - SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} - COMPONENT: MCP - RELEASE_TAG: ${{ env.RELEASE_TAG }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - with: - slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build and push MCP container for ${{ matrix.arch }} id: container-push @@ -111,36 +123,21 @@ jobs: cache-from: type=gha,scope=${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ matrix.arch }} - - name: Notify container push completed - if: (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && always() - uses: ./.github/actions/slack-notification - env: - SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} - COMPONENT: MCP - RELEASE_TAG: ${{ env.RELEASE_TAG }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - with: - slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" - step-outcome: ${{ steps.container-push.outcome }} - # Create and push multi-architecture manifest create-manifest: needs: [setup, container-build-push] - if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' + if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest steps: - name: Login to DockerHub - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Create and push manifests for push event if: github.event_name == 'push' @@ -172,9 +169,43 @@ jobs: regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true echo "Cleanup completed" + notify-release-completed: + if: always() && needs.notify-release-started.result == 'success' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') + needs: [setup, notify-release-started, container-build-push, create-manifest] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Determine overall outcome + id: outcome + run: | + if [[ "${{ needs.container-build-push.result }}" == "success" && "${{ needs.create-manifest.result }}" == "success" ]]; then + echo "outcome=success" >> $GITHUB_OUTPUT + else + echo "outcome=failure" >> $GITHUB_OUTPUT + fi + + - name: Notify container push completed + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + MESSAGE_TS: ${{ needs.notify-release-started.outputs.message-ts }} + COMPONENT: MCP + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.outcome.outputs.outcome }} + update-ts: ${{ needs.notify-release-started.outputs.message-ts }} + trigger-deployment: - if: github.event_name == 'push' needs: [setup, container-build-push] + if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -182,7 +213,7 @@ jobs: steps: - name: Trigger MCP deployment - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} repository: ${{ secrets.CLOUD_DISPATCH }} diff --git a/.github/workflows/mcp-container-checks.yml b/.github/workflows/mcp-container-checks.yml index 3310cd8c5a..5ea0b6c465 100644 --- a/.github/workflows/mcp-container-checks.yml +++ b/.github/workflows/mcp-container-checks.yml @@ -20,6 +20,7 @@ env: jobs: mcp-dockerfile-lint: + if: github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -27,11 +28,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check if Dockerfile changed id: dockerfile-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: mcp_server/Dockerfile @@ -42,7 +43,17 @@ jobs: dockerfile: mcp_server/Dockerfile mcp-container-build-and-scan: - runs-on: ubuntu-latest + if: github.repository == 'prowler-cloud/prowler' + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 timeout-minutes: 30 permissions: contents: read @@ -51,11 +62,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for MCP changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: mcp_server/** files_ignore: | @@ -64,24 +75,25 @@ jobs: - name: Set up Docker Buildx if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Build MCP container + - name: Build MCP container for ${{ matrix.arch }} if: steps.check-changes.outputs.any_changed == 'true' uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: ${{ env.MCP_WORKING_DIR }} push: false load: true - tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + platforms: ${{ matrix.platform }} + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} - - name: Scan MCP container with Trivy - if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' + - name: Scan MCP container with Trivy for ${{ matrix.arch }} + if: steps.check-changes.outputs.any_changed == 'true' uses: ./.github/actions/trivy-scan with: image-name: ${{ env.IMAGE_NAME }} - image-tag: ${{ github.sha }} + image-tag: ${{ github.sha }}-${{ matrix.arch }} fail-on-critical: 'false' severity: 'CRITICAL' diff --git a/.github/workflows/mcp-pypi-release.yml b/.github/workflows/mcp-pypi-release.yml new file mode 100644 index 0000000000..7a3e5f5daa --- /dev/null +++ b/.github/workflows/mcp-pypi-release.yml @@ -0,0 +1,81 @@ +name: "MCP: PyPI Release" + +on: + release: + types: + - "published" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false + +env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + PYTHON_VERSION: "3.12" + WORKING_DIRECTORY: ./mcp_server + +jobs: + validate-release: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + prowler_version: ${{ steps.parse-version.outputs.version }} + major_version: ${{ steps.parse-version.outputs.major }} + + steps: + - name: Parse and validate version + id: parse-version + run: | + PROWLER_VERSION="${{ env.RELEASE_TAG }}" + echo "version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}" + + # Extract major version + MAJOR_VERSION="${PROWLER_VERSION%%.*}" + echo "major=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}" + + # Validate major version (only Prowler 3, 4, 5 supported) + case ${MAJOR_VERSION} in + 3|4|5) + echo "✓ Releasing Prowler MCP for tag ${PROWLER_VERSION}" + ;; + *) + echo "::error::Unsupported Prowler major version: ${MAJOR_VERSION}" + exit 1 + ;; + esac + + publish-prowler-mcp: + needs: validate-release + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + environment: + name: pypi-prowler-mcp + url: https://pypi.org/project/prowler-mcp/ + + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Build prowler-mcp package + working-directory: ${{ env.WORKING_DIRECTORY }} + run: uv build + + - name: Publish prowler-mcp package to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + packages-dir: ${{ env.WORKING_DIRECTORY }}/dist/ + print-hash: true diff --git a/.github/workflows/pr-check-changelog.yml b/.github/workflows/pr-check-changelog.yml index d1a3220402..4a1f8e502a 100644 --- a/.github/workflows/pr-check-changelog.yml +++ b/.github/workflows/pr-check-changelog.yml @@ -29,13 +29,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 - name: Get changed files id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | api/** diff --git a/.github/workflows/pr-conflict-checker.yml b/.github/workflows/pr-conflict-checker.yml index c46fafc2d0..c812f4ef19 100644 --- a/.github/workflows/pr-conflict-checker.yml +++ b/.github/workflows/pr-conflict-checker.yml @@ -25,14 +25,14 @@ jobs: steps: - name: Checkout PR head - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - name: Get changed files id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: '**' diff --git a/.github/workflows/pr-merged.yml b/.github/workflows/pr-merged.yml index d8255026e6..61970827f2 100644 --- a/.github/workflows/pr-merged.yml +++ b/.github/workflows/pr-merged.yml @@ -13,7 +13,10 @@ concurrency: jobs: trigger-cloud-pull-request: - if: github.event.pull_request.merged == true && github.repository == 'prowler-cloud/prowler' + if: | + github.event.pull_request.merged == true && + github.repository == 'prowler-cloud/prowler' && + !contains(github.event.pull_request.labels.*.name, 'skip-sync') runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -26,7 +29,7 @@ jobs: echo "SHORT_SHA=${SHORT_SHA::7}" >> $GITHUB_ENV - name: Trigger Cloud repository pull request - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} repository: ${{ secrets.CLOUD_DISPATCH }} diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index ae039bd5a6..815ade1dec 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -27,13 +27,13 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - name: Set up Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.12' @@ -88,59 +88,56 @@ jobs: - name: Read changelog versions from release branch run: | - # Function to extract the latest version from changelog - extract_latest_version() { + # Function to extract the version for a specific Prowler release from changelog + # This looks for entries with "(Prowler X.Y.Z)" to find the released version + extract_version_for_release() { local changelog_file="$1" + local prowler_version="$2" if [ -f "$changelog_file" ]; then - # Extract the first version entry (most recent) from changelog - # Format: ## [version] (1.2.3) or ## [vversion] (v1.2.3) - local version=$(grep -m 1 '^## \[' "$changelog_file" | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]') + # Extract version that matches this Prowler release + # Format: ## [version] (Prowler X.Y.Z) or ## [vversion] (Prowler vX.Y.Z) + local version=$(grep '^## \[' "$changelog_file" | grep "(Prowler v\?${prowler_version})" | head -1 | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]') echo "$version" else echo "" fi } - # Read actual versions from changelogs (source of truth) - UI_VERSION=$(extract_latest_version "ui/CHANGELOG.md") - API_VERSION=$(extract_latest_version "api/CHANGELOG.md") - SDK_VERSION=$(extract_latest_version "prowler/CHANGELOG.md") - MCP_VERSION=$(extract_latest_version "mcp_server/CHANGELOG.md") + # Read versions from changelogs for this specific Prowler release + SDK_VERSION=$(extract_version_for_release "prowler/CHANGELOG.md" "$PROWLER_VERSION") + API_VERSION=$(extract_version_for_release "api/CHANGELOG.md" "$PROWLER_VERSION") + UI_VERSION=$(extract_version_for_release "ui/CHANGELOG.md" "$PROWLER_VERSION") + MCP_VERSION=$(extract_version_for_release "mcp_server/CHANGELOG.md" "$PROWLER_VERSION") - echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}" - echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}" echo "SDK_VERSION=${SDK_VERSION}" >> "${GITHUB_ENV}" + echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}" + echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}" echo "MCP_VERSION=${MCP_VERSION}" >> "${GITHUB_ENV}" - if [ -n "$UI_VERSION" ]; then - echo "Read UI version from changelog: $UI_VERSION" + if [ -n "$SDK_VERSION" ]; then + echo "✓ SDK version for Prowler $PROWLER_VERSION: $SDK_VERSION" else - echo "Warning: No UI version found in ui/CHANGELOG.md" + echo "ℹ No SDK version found for Prowler $PROWLER_VERSION in prowler/CHANGELOG.md" fi if [ -n "$API_VERSION" ]; then - echo "Read API version from changelog: $API_VERSION" + echo "✓ API version for Prowler $PROWLER_VERSION: $API_VERSION" else - echo "Warning: No API version found in api/CHANGELOG.md" + echo "ℹ No API version found for Prowler $PROWLER_VERSION in api/CHANGELOG.md" fi - if [ -n "$SDK_VERSION" ]; then - echo "Read SDK version from changelog: $SDK_VERSION" + if [ -n "$UI_VERSION" ]; then + echo "✓ UI version for Prowler $PROWLER_VERSION: $UI_VERSION" else - echo "Warning: No SDK version found in prowler/CHANGELOG.md" + echo "ℹ No UI version found for Prowler $PROWLER_VERSION in ui/CHANGELOG.md" fi if [ -n "$MCP_VERSION" ]; then - echo "Read MCP version from changelog: $MCP_VERSION" + echo "✓ MCP version for Prowler $PROWLER_VERSION: $MCP_VERSION" else - echo "Warning: No MCP version found in mcp_server/CHANGELOG.md" + echo "ℹ No MCP version found for Prowler $PROWLER_VERSION in mcp_server/CHANGELOG.md" fi - echo "UI version: $UI_VERSION" - echo "API version: $API_VERSION" - echo "SDK version: $SDK_VERSION" - echo "MCP version: $MCP_VERSION" - - name: Extract and combine changelog entries run: | set -e @@ -166,70 +163,54 @@ jobs: # Remove --- separators sed -i '/^---$/d' "$output_file" - - # Remove only trailing empty lines (not all empty lines) - sed -i -e :a -e '/^\s*$/d;N;ba' "$output_file" } - # Calculate expected versions for this release - if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then - EXPECTED_UI_VERSION="1.${BASH_REMATCH[2]}.${BASH_REMATCH[3]}" - EXPECTED_API_VERSION="1.$((${BASH_REMATCH[2]} + 1)).${BASH_REMATCH[3]}" - - echo "Expected UI version for this release: $EXPECTED_UI_VERSION" - echo "Expected API version for this release: $EXPECTED_API_VERSION" - fi - # Determine if components have changes for this specific release - # UI has changes if its current version matches what we expect for this release - if [ -n "$UI_VERSION" ] && [ "$UI_VERSION" = "$EXPECTED_UI_VERSION" ]; then - echo "HAS_UI_CHANGES=true" >> $GITHUB_ENV - echo "✓ UI changes detected - version matches expected: $UI_VERSION" - extract_changelog "ui/CHANGELOG.md" "$UI_VERSION" "ui_changelog.md" - else - echo "HAS_UI_CHANGES=false" >> $GITHUB_ENV - echo "ℹ No UI changes for this release (current: $UI_VERSION, expected: $EXPECTED_UI_VERSION)" - touch "ui_changelog.md" - fi - - # API has changes if its current version matches what we expect for this release - if [ -n "$API_VERSION" ] && [ "$API_VERSION" = "$EXPECTED_API_VERSION" ]; then - echo "HAS_API_CHANGES=true" >> $GITHUB_ENV - echo "✓ API changes detected - version matches expected: $API_VERSION" - extract_changelog "api/CHANGELOG.md" "$API_VERSION" "api_changelog.md" - else - echo "HAS_API_CHANGES=false" >> $GITHUB_ENV - echo "ℹ No API changes for this release (current: $API_VERSION, expected: $EXPECTED_API_VERSION)" - touch "api_changelog.md" - fi - - # SDK has changes if its current version matches the input version - if [ -n "$SDK_VERSION" ] && [ "$SDK_VERSION" = "$PROWLER_VERSION" ]; then + if [ -n "$SDK_VERSION" ]; then echo "HAS_SDK_CHANGES=true" >> $GITHUB_ENV - echo "✓ SDK changes detected - version matches input: $SDK_VERSION" - extract_changelog "prowler/CHANGELOG.md" "$PROWLER_VERSION" "prowler_changelog.md" + HAS_SDK_CHANGES="true" + echo "✓ SDK changes detected - version: $SDK_VERSION" + extract_changelog "prowler/CHANGELOG.md" "$SDK_VERSION" "prowler_changelog.md" else echo "HAS_SDK_CHANGES=false" >> $GITHUB_ENV - echo "ℹ No SDK changes for this release (current: $SDK_VERSION, input: $PROWLER_VERSION)" + HAS_SDK_CHANGES="false" + echo "ℹ No SDK changes for this release" touch "prowler_changelog.md" fi - # MCP has changes if the changelog references this Prowler version - # Check if the changelog contains "(Prowler X.Y.Z)" or "(Prowler UNRELEASED)" - if [ -f "mcp_server/CHANGELOG.md" ]; then - MCP_PROWLER_REF=$(grep -m 1 "^## \[.*\] (Prowler" mcp_server/CHANGELOG.md | sed -E 's/.*\(Prowler ([^)]+)\).*/\1/' | tr -d '[:space:]') - if [ "$MCP_PROWLER_REF" = "$PROWLER_VERSION" ] || [ "$MCP_PROWLER_REF" = "UNRELEASED" ]; then - echo "HAS_MCP_CHANGES=true" >> $GITHUB_ENV - echo "✓ MCP changes detected - Prowler reference: $MCP_PROWLER_REF (version: $MCP_VERSION)" - extract_changelog "mcp_server/CHANGELOG.md" "$MCP_VERSION" "mcp_changelog.md" - else - echo "HAS_MCP_CHANGES=false" >> $GITHUB_ENV - echo "ℹ No MCP changes for this release (Prowler reference: $MCP_PROWLER_REF, input: $PROWLER_VERSION)" - touch "mcp_changelog.md" - fi + if [ -n "$API_VERSION" ]; then + echo "HAS_API_CHANGES=true" >> $GITHUB_ENV + HAS_API_CHANGES="true" + echo "✓ API changes detected - version: $API_VERSION" + extract_changelog "api/CHANGELOG.md" "$API_VERSION" "api_changelog.md" + else + echo "HAS_API_CHANGES=false" >> $GITHUB_ENV + HAS_API_CHANGES="false" + echo "ℹ No API changes for this release" + touch "api_changelog.md" + fi + + if [ -n "$UI_VERSION" ]; then + echo "HAS_UI_CHANGES=true" >> $GITHUB_ENV + HAS_UI_CHANGES="true" + echo "✓ UI changes detected - version: $UI_VERSION" + extract_changelog "ui/CHANGELOG.md" "$UI_VERSION" "ui_changelog.md" + else + echo "HAS_UI_CHANGES=false" >> $GITHUB_ENV + HAS_UI_CHANGES="false" + echo "ℹ No UI changes for this release" + touch "ui_changelog.md" + fi + + if [ -n "$MCP_VERSION" ]; then + echo "HAS_MCP_CHANGES=true" >> $GITHUB_ENV + HAS_MCP_CHANGES="true" + echo "✓ MCP changes detected - version: $MCP_VERSION" + extract_changelog "mcp_server/CHANGELOG.md" "$MCP_VERSION" "mcp_changelog.md" else echo "HAS_MCP_CHANGES=false" >> $GITHUB_ENV - echo "ℹ No MCP changelog found" + HAS_MCP_CHANGES="false" + echo "ℹ No MCP changes for this release" touch "mcp_changelog.md" fi @@ -325,6 +306,17 @@ jobs: fi echo "✓ api/src/backend/api/v1/views.py version: $CURRENT_API_VERSION" + - name: Verify API version in api/src/backend/api/specs/v1.yaml + if: ${{ env.HAS_API_CHANGES == 'true' }} + run: | + CURRENT_API_VERSION=$(grep '^ version: ' api/src/backend/api/specs/v1.yaml | sed -E 's/ version: ([0-9]+\.[0-9]+\.[0-9]+)/\1/' | tr -d '[:space:]') + API_VERSION_TRIMMED=$(echo "$API_VERSION" | tr -d '[:space:]') + if [ "$CURRENT_API_VERSION" != "$API_VERSION_TRIMMED" ]; then + echo "ERROR: API version mismatch in api/src/backend/api/specs/v1.yaml (expected: '$API_VERSION_TRIMMED', found: '$CURRENT_API_VERSION')" + exit 1 + fi + echo "✓ api/src/backend/api/specs/v1.yaml version: $CURRENT_API_VERSION" + - name: Update API prowler dependency for minor release if: ${{ env.PATCH_VERSION == '0' }} run: | @@ -352,7 +344,7 @@ jobs: - name: Create PR for API dependency update if: ${{ env.PATCH_VERSION == '0' }} - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} commit-message: 'chore(api): update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}' @@ -382,7 +374,7 @@ jobs: no-changelog - name: Create draft release - uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2.4.1 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 with: tag_name: ${{ env.PROWLER_VERSION }} name: Prowler ${{ env.PROWLER_VERSION }} diff --git a/.github/workflows/sdk-bump-version.yml b/.github/workflows/sdk-bump-version.yml index 0291502ab6..9f25065689 100644 --- a/.github/workflows/sdk-bump-version.yml +++ b/.github/workflows/sdk-bump-version.yml @@ -67,7 +67,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Calculate next minor version run: | @@ -86,13 +86,12 @@ jobs: sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_MINOR_VERSION}\"|" pyproject.toml sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_MINOR_VERSION}\"|" prowler/config/config.py - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env echo "Files modified:" git --no-pager diff - name: Create PR for next minor version to master - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 with: author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} @@ -100,7 +99,7 @@ jobs: commit-message: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}' branch: version-bump-to-v${{ env.NEXT_MINOR_VERSION }} title: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}' - labels: no-changelog + labels: no-changelog,skip-sync body: | ### Description @@ -111,7 +110,7 @@ jobs: By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. - name: Checkout version branch - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }} @@ -135,13 +134,12 @@ jobs: sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${FIRST_PATCH_VERSION}\"|" pyproject.toml sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${FIRST_PATCH_VERSION}\"|" prowler/config/config.py - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env echo "Files modified:" git --no-pager diff - name: Create PR for first patch version to version branch - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 with: author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} @@ -149,7 +147,7 @@ jobs: commit-message: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}' branch: version-bump-to-v${{ env.FIRST_PATCH_VERSION }} title: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}' - labels: no-changelog + labels: no-changelog,skip-sync body: | ### Description @@ -169,7 +167,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Calculate next patch version run: | @@ -193,13 +191,12 @@ jobs: sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_PATCH_VERSION}\"|" pyproject.toml sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_PATCH_VERSION}\"|" prowler/config/config.py - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env echo "Files modified:" git --no-pager diff - name: Create PR for next patch version to version branch - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 with: author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} @@ -207,7 +204,7 @@ jobs: commit-message: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}' branch: version-bump-to-v${{ env.NEXT_PATCH_VERSION }} title: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}' - labels: no-changelog + labels: no-changelog,skip-sync body: | ### Description diff --git a/.github/workflows/sdk-code-quality.yml b/.github/workflows/sdk-code-quality.yml index b32324d8cb..004ca0b7eb 100644 --- a/.github/workflows/sdk-code-quality.yml +++ b/.github/workflows/sdk-code-quality.yml @@ -31,11 +31,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for SDK changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: ./** files_ignore: | @@ -47,6 +47,7 @@ jobs: ui/** dashboard/** mcp_server/** + skills/** README.md mkdocs.yml .backportrc.json @@ -55,6 +56,7 @@ jobs: examples/** .gitignore contrib/** + **/AGENTS.md - name: Install Poetry if: steps.check-changes.outputs.any_changed == 'true' @@ -62,7 +64,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} if: steps.check-changes.outputs.any_changed == 'true' - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ matrix.python-version }} cache: 'poetry' @@ -79,11 +81,11 @@ jobs: - name: Lint with flake8 if: steps.check-changes.outputs.any_changed == 'true' - run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api + run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api,skills - name: Check format with black if: steps.check-changes.outputs.any_changed == 'true' - run: poetry run black --exclude api ui --check . + run: poetry run black --exclude "api|ui|skills" --check . - name: Lint with pylint if: steps.check-changes.outputs.any_changed == 'true' diff --git a/.github/workflows/sdk-codeql.yml b/.github/workflows/sdk-codeql.yml index 590ba52d34..a45f633567 100644 --- a/.github/workflows/sdk-codeql.yml +++ b/.github/workflows/sdk-codeql.yml @@ -49,15 +49,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Initialize CodeQL - uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/sdk-codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/sdk-container-build-push.yml b/.github/workflows/sdk-container-build-push.yml index c6ad6ba0bd..d510ec8568 100644 --- a/.github/workflows/sdk-container-build-push.yml +++ b/.github/workflows/sdk-container-build-push.yml @@ -50,8 +50,90 @@ env: AWS_REGION: us-east-1 jobs: - container-build-push: + setup: if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + prowler_version: ${{ steps.get-prowler-version.outputs.prowler_version }} + prowler_version_major: ${{ steps.get-prowler-version.outputs.prowler_version_major }} + latest_tag: ${{ steps.get-prowler-version.outputs.latest_tag }} + stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }} + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Poetry + run: | + pipx install poetry==2.1.1 + pipx inject poetry poetry-bumpversion + + - name: Get Prowler version and set tags + id: get-prowler-version + run: | + PROWLER_VERSION="$(poetry version -s 2>/dev/null)" + echo "prowler_version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}" + + # Extract major version + PROWLER_VERSION_MAJOR="${PROWLER_VERSION%%.*}" + echo "prowler_version_major=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_OUTPUT}" + + # Set version-specific tags + case ${PROWLER_VERSION_MAJOR} in + 3) + echo "latest_tag=v3-latest" >> "${GITHUB_OUTPUT}" + echo "stable_tag=v3-stable" >> "${GITHUB_OUTPUT}" + echo "✓ Prowler v3 detected - tags: v3-latest, v3-stable" + ;; + 4) + echo "latest_tag=v4-latest" >> "${GITHUB_OUTPUT}" + echo "stable_tag=v4-stable" >> "${GITHUB_OUTPUT}" + echo "✓ Prowler v4 detected - tags: v4-latest, v4-stable" + ;; + 5) + echo "latest_tag=latest" >> "${GITHUB_OUTPUT}" + echo "stable_tag=stable" >> "${GITHUB_OUTPUT}" + echo "✓ Prowler v5 detected - tags: latest, stable" + ;; + *) + echo "::error::Unsupported Prowler major version: ${PROWLER_VERSION_MAJOR}" + exit 1 + ;; + esac + + notify-release-started: + if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') + needs: setup + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + message-ts: ${{ steps.slack-notification.outputs.ts }} + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Notify container push started + id: slack-notification + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: SDK + RELEASE_TAG: ${{ needs.setup.outputs.prowler_version }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + container-build-push: + needs: [setup, notify-release-started] + if: always() && needs.setup.result == 'success' && (needs.notify-release-started.result == 'success' || needs.notify-release-started.result == 'skipped') runs-on: ${{ matrix.runner }} strategy: matrix: @@ -66,68 +148,10 @@ jobs: permissions: contents: read packages: write - outputs: - prowler_version: ${{ steps.get-prowler-version.outputs.prowler_version }} - prowler_version_major: ${{ steps.get-prowler-version.outputs.prowler_version_major }} - latest_tag: ${{ steps.get-prowler-version.outputs.latest_tag }} - stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }} - env: - POETRY_VIRTUALENVS_CREATE: 'false' steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install Poetry - run: | - pipx install poetry==2.1.1 - pipx inject poetry poetry-bumpversion - - - name: Get Prowler version and set tags - id: get-prowler-version - run: | - PROWLER_VERSION="$(poetry version -s 2>/dev/null)" - echo "prowler_version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}" - echo "PROWLER_VERSION=${PROWLER_VERSION}" >> "${GITHUB_ENV}" - - # Extract major version - PROWLER_VERSION_MAJOR="${PROWLER_VERSION%%.*}" - echo "prowler_version_major=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_OUTPUT}" - echo "PROWLER_VERSION_MAJOR=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_ENV}" - - # Set version-specific tags - case ${PROWLER_VERSION_MAJOR} in - 3) - echo "LATEST_TAG=v3-latest" >> "${GITHUB_ENV}" - echo "STABLE_TAG=v3-stable" >> "${GITHUB_ENV}" - echo "latest_tag=v3-latest" >> "${GITHUB_OUTPUT}" - echo "stable_tag=v3-stable" >> "${GITHUB_OUTPUT}" - echo "✓ Prowler v3 detected - tags: v3-latest, v3-stable" - ;; - 4) - echo "LATEST_TAG=v4-latest" >> "${GITHUB_ENV}" - echo "STABLE_TAG=v4-stable" >> "${GITHUB_ENV}" - echo "latest_tag=v4-latest" >> "${GITHUB_OUTPUT}" - echo "stable_tag=v4-stable" >> "${GITHUB_OUTPUT}" - echo "✓ Prowler v4 detected - tags: v4-latest, v4-stable" - ;; - 5) - echo "LATEST_TAG=latest" >> "${GITHUB_ENV}" - echo "STABLE_TAG=stable" >> "${GITHUB_ENV}" - echo "latest_tag=latest" >> "${GITHUB_OUTPUT}" - echo "stable_tag=stable" >> "${GITHUB_OUTPUT}" - echo "✓ Prowler v5 detected - tags: latest, stable" - ;; - *) - echo "::error::Unsupported Prowler major version: ${PROWLER_VERSION_MAJOR}" - exit 1 - ;; - esac + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Login to DockerHub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 @@ -145,21 +169,7 @@ jobs: AWS_REGION: ${{ env.AWS_REGION }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Notify container push started - if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' - uses: ./.github/actions/slack-notification - env: - SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} - COMPONENT: SDK - RELEASE_TAG: ${{ env.PROWLER_VERSION }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - with: - slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build and push SDK container for ${{ matrix.arch }} id: container-push @@ -171,40 +181,25 @@ jobs: push: true platforms: ${{ matrix.platform }} tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }}-${{ matrix.arch }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-${{ matrix.arch }} cache-from: type=gha,scope=${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ matrix.arch }} - - name: Notify container push completed - if: (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && always() - uses: ./.github/actions/slack-notification - env: - SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} - COMPONENT: SDK - RELEASE_TAG: ${{ env.PROWLER_VERSION }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - with: - slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" - step-outcome: ${{ steps.container-push.outcome }} - # Create and push multi-architecture manifest create-manifest: - needs: [container-build-push] - if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' + needs: [setup, container-build-push] + if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest steps: - name: Login to DockerHub - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Public ECR - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: public.ecr.aws username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }} @@ -213,30 +208,30 @@ jobs: AWS_REGION: ${{ env.AWS_REGION }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Create and push manifests for push event if: github.event_name == 'push' run: | docker buildx imagetools create \ - -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \ - -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \ - -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \ - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64 \ - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64 + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }} \ + -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }} \ + -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-arm64 - name: Create and push manifests for release event if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' run: | docker buildx imagetools create \ - -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.prowler_version }} \ - -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.stable_tag }} \ - -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.prowler_version }} \ - -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.stable_tag }} \ - -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.prowler_version }} \ - -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.stable_tag }} \ - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64 \ - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64 + -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.setup.outputs.prowler_version }} \ + -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.setup.outputs.stable_tag }} \ + -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.setup.outputs.prowler_version }} \ + -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.setup.outputs.stable_tag }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.prowler_version }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.stable_tag }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-arm64 - name: Install regctl if: always() @@ -246,13 +241,47 @@ jobs: if: always() run: | echo "Cleaning up intermediate tags..." - regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64" || true - regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64" || true + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-amd64" || true + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-arm64" || true echo "Cleanup completed" + notify-release-completed: + if: always() && needs.notify-release-started.result == 'success' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') + needs: [setup, notify-release-started, container-build-push, create-manifest] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Determine overall outcome + id: outcome + run: | + if [[ "${{ needs.container-build-push.result }}" == "success" && "${{ needs.create-manifest.result }}" == "success" ]]; then + echo "outcome=success" >> $GITHUB_OUTPUT + else + echo "outcome=failure" >> $GITHUB_OUTPUT + fi + + - name: Notify container push completed + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + MESSAGE_TS: ${{ needs.notify-release-started.outputs.message-ts }} + COMPONENT: SDK + RELEASE_TAG: ${{ needs.setup.outputs.prowler_version }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.outcome.outputs.outcome }} + update-ts: ${{ needs.notify-release-started.outputs.message-ts }} + dispatch-v3-deployment: - if: needs.container-build-push.outputs.prowler_version_major == '3' - needs: container-build-push + needs: [setup, container-build-push] + if: always() && needs.setup.outputs.prowler_version_major == '3' && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -265,7 +294,7 @@ jobs: - name: Dispatch v3 deployment (latest) if: github.event_name == 'push' - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }} @@ -274,9 +303,9 @@ jobs: - name: Dispatch v3 deployment (release) if: github.event_name == 'release' - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }} event-type: dispatch - client-payload: '{"version":"release","tag":"${{ needs.container-build-push.outputs.prowler_version }}"}' + client-payload: '{"version":"release","tag":"${{ needs.setup.outputs.prowler_version }}"}' diff --git a/.github/workflows/sdk-container-checks.yml b/.github/workflows/sdk-container-checks.yml index 4e8a7cf2ad..7a0323c216 100644 --- a/.github/workflows/sdk-container-checks.yml +++ b/.github/workflows/sdk-container-checks.yml @@ -27,11 +27,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check if Dockerfile changed id: dockerfile-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: Dockerfile @@ -44,7 +44,16 @@ jobs: sdk-container-build-and-scan: if: github.repository == 'prowler-cloud/prowler' - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 timeout-minutes: 30 permissions: contents: read @@ -53,11 +62,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for SDK changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: ./** files_ignore: | @@ -69,6 +78,7 @@ jobs: ui/** dashboard/** mcp_server/** + skills/** README.md mkdocs.yml .backportrc.json @@ -77,27 +87,29 @@ jobs: examples/** .gitignore contrib/** + **/AGENTS.md - name: Set up Docker Buildx if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Build SDK container + - name: Build SDK container for ${{ matrix.arch }} if: steps.check-changes.outputs.any_changed == 'true' uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: . push: false load: true - tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + platforms: ${{ matrix.platform }} + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} - - name: Scan SDK container with Trivy + - name: Scan SDK container with Trivy for ${{ matrix.arch }} if: steps.check-changes.outputs.any_changed == 'true' uses: ./.github/actions/trivy-scan with: image-name: ${{ env.IMAGE_NAME }} - image-tag: ${{ github.sha }} + image-tag: ${{ github.sha }}-${{ matrix.arch }} fail-on-critical: 'false' severity: 'CRITICAL' diff --git a/.github/workflows/sdk-pypi-release.yml b/.github/workflows/sdk-pypi-release.yml index 0f74ba054c..68a986d37a 100644 --- a/.github/workflows/sdk-pypi-release.yml +++ b/.github/workflows/sdk-pypi-release.yml @@ -59,13 +59,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Poetry run: pipx install poetry==2.1.1 - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ env.PYTHON_VERSION }} cache: 'poetry' @@ -91,13 +91,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Poetry run: pipx install poetry==2.1.1 - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ env.PYTHON_VERSION }} cache: 'poetry' diff --git a/.github/workflows/sdk-refresh-aws-services-regions.yml b/.github/workflows/sdk-refresh-aws-services-regions.yml index 884b677294..25839f2631 100644 --- a/.github/workflows/sdk-refresh-aws-services-regions.yml +++ b/.github/workflows/sdk-refresh-aws-services-regions.yml @@ -25,12 +25,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: 'master' - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' @@ -39,7 +39,7 @@ jobs: run: pip install boto3 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0 + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 with: aws-region: ${{ env.AWS_REGION }} role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }} @@ -50,7 +50,7 @@ jobs: - name: Create pull request id: create-pr - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} author: 'prowler-bot <179230569+prowler-bot@users.noreply.github.com>' diff --git a/.github/workflows/sdk-security.yml b/.github/workflows/sdk-security.yml index a15bd2539f..01f94d842a 100644 --- a/.github/workflows/sdk-security.yml +++ b/.github/workflows/sdk-security.yml @@ -24,13 +24,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for SDK changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: - files: ./** + files: + ./** + .github/workflows/sdk-security.yml files_ignore: | .github/** prowler/CHANGELOG.md @@ -40,6 +42,7 @@ jobs: ui/** dashboard/** mcp_server/** + skills/** README.md mkdocs.yml .backportrc.json @@ -48,6 +51,7 @@ jobs: examples/** .gitignore contrib/** + **/AGENTS.md - name: Install Poetry if: steps.check-changes.outputs.any_changed == 'true' @@ -55,7 +59,7 @@ jobs: - name: Set up Python 3.12 if: steps.check-changes.outputs.any_changed == 'true' - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.12' cache: 'poetry' @@ -70,7 +74,7 @@ jobs: - name: Security scan with Safety if: steps.check-changes.outputs.any_changed == 'true' - run: poetry run safety check --ignore 70612 -r pyproject.toml + run: poetry run safety check -r pyproject.toml - name: Dead code detection with Vulture if: steps.check-changes.outputs.any_changed == 'true' diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index b01c5e7778..b49ffd39f5 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -31,11 +31,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for SDK changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: ./** files_ignore: | @@ -47,6 +47,7 @@ jobs: ui/** dashboard/** mcp_server/** + skills/** README.md mkdocs.yml .backportrc.json @@ -55,6 +56,7 @@ jobs: examples/** .gitignore contrib/** + **/AGENTS.md - name: Install Poetry if: steps.check-changes.outputs.any_changed == 'true' @@ -62,7 +64,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} if: steps.check-changes.outputs.any_changed == 'true' - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ matrix.python-version }} cache: 'poetry' @@ -75,20 +77,121 @@ jobs: - name: Check if AWS files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-aws - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/aws/** ./tests/**/aws/** ./poetry.lock + - name: Resolve AWS services under test + if: steps.changed-aws.outputs.any_changed == 'true' + id: aws-services + shell: bash + run: | + python3 <<'PY' + import os + from pathlib import Path + + dependents = { + "acm": ["elb"], + "autoscaling": ["dynamodb"], + "awslambda": ["ec2", "inspector2"], + "backup": ["dynamodb", "ec2", "rds"], + "cloudfront": ["shield"], + "cloudtrail": ["awslambda", "cloudwatch"], + "cloudwatch": ["bedrock"], + "ec2": ["dlm", "dms", "elbv2", "emr", "inspector2", "rds", "redshift", "route53", "shield", "ssm"], + "ecr": ["inspector2"], + "elb": ["shield"], + "elbv2": ["shield"], + "globalaccelerator": ["shield"], + "iam": ["bedrock", "cloudtrail", "cloudwatch", "codebuild"], + "kafka": ["firehose"], + "kinesis": ["firehose"], + "kms": ["kafka"], + "organizations": ["iam", "servicecatalog"], + "route53": ["shield"], + "s3": ["bedrock", "cloudfront", "cloudtrail", "macie"], + "ssm": ["ec2"], + "vpc": ["awslambda", "ec2", "efs", "elasticache", "neptune", "networkfirewall", "rds", "redshift", "workspaces"], + "waf": ["elbv2"], + "wafv2": ["cognito", "elbv2"], + } + + changed_raw = """${{ steps.changed-aws.outputs.all_changed_files }}""" + # all_changed_files is space-separated, not newline-separated + # Strip leading "./" if present for consistent path handling + changed_files = [Path(f.lstrip("./")) for f in changed_raw.split() if f] + + services = set() + run_all = False + + for path in changed_files: + path_str = path.as_posix() + parts = path.parts + if path_str.startswith("prowler/providers/aws/services/"): + if len(parts) > 4 and "." not in parts[4]: + services.add(parts[4]) + else: + run_all = True + elif path_str.startswith("tests/providers/aws/services/"): + if len(parts) > 4 and "." not in parts[4]: + services.add(parts[4]) + else: + run_all = True + elif path_str.startswith("prowler/providers/aws/") or path_str.startswith("tests/providers/aws/"): + run_all = True + + # Expand with direct dependent services (one level only) + # We only test services that directly depend on the changed services, + # not transitive dependencies (services that depend on dependents) + original_services = set(services) + for svc in original_services: + for dep in dependents.get(svc, []): + services.add(dep) + + if run_all or not services: + run_all = True + services = set() + + service_paths = " ".join(sorted(f"tests/providers/aws/services/{svc}" for svc in services)) + + output_lines = [ + f"run_all={'true' if run_all else 'false'}", + f"services={' '.join(sorted(services))}", + f"service_paths={service_paths}", + ] + + with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out: + for line in output_lines: + gh_out.write(line + "\n") + + print(f"AWS changed files (filtered): {changed_raw or 'none'}") + print(f"Run all AWS tests: {run_all}") + if services: + print(f"AWS service test paths: {service_paths}") + else: + print("AWS service test paths: none detected") + PY + - name: Run AWS tests if: steps.changed-aws.outputs.any_changed == 'true' - run: poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws + run: | + echo "AWS run_all=${{ steps.aws-services.outputs.run_all }}" + echo "AWS service_paths='${{ steps.aws-services.outputs.service_paths }}'" + + if [ "${{ steps.aws-services.outputs.run_all }}" = "true" ]; then + poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws + elif [ -z "${{ steps.aws-services.outputs.service_paths }}" ]; then + echo "No AWS service paths detected; skipping AWS tests." + else + poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml ${{ steps.aws-services.outputs.service_paths }} + fi - name: Upload AWS coverage to Codecov if: steps.changed-aws.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -99,7 +202,7 @@ jobs: - name: Check if Azure files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-azure - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/azure/** @@ -112,7 +215,7 @@ jobs: - name: Upload Azure coverage to Codecov if: steps.changed-azure.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -123,7 +226,7 @@ jobs: - name: Check if GCP files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-gcp - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/gcp/** @@ -136,7 +239,7 @@ jobs: - name: Upload GCP coverage to Codecov if: steps.changed-gcp.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -147,7 +250,7 @@ jobs: - name: Check if Kubernetes files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-kubernetes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/kubernetes/** @@ -160,7 +263,7 @@ jobs: - name: Upload Kubernetes coverage to Codecov if: steps.changed-kubernetes.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -171,7 +274,7 @@ jobs: - name: Check if GitHub files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-github - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/github/** @@ -184,7 +287,7 @@ jobs: - name: Upload GitHub coverage to Codecov if: steps.changed-github.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -195,7 +298,7 @@ jobs: - name: Check if NHN files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-nhn - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/nhn/** @@ -208,7 +311,7 @@ jobs: - name: Upload NHN coverage to Codecov if: steps.changed-nhn.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -219,7 +322,7 @@ jobs: - name: Check if M365 files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-m365 - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/m365/** @@ -232,7 +335,7 @@ jobs: - name: Upload M365 coverage to Codecov if: steps.changed-m365.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -243,7 +346,7 @@ jobs: - name: Check if IaC files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-iac - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/iac/** @@ -256,7 +359,7 @@ jobs: - name: Upload IaC coverage to Codecov if: steps.changed-iac.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -267,7 +370,7 @@ jobs: - name: Check if MongoDB Atlas files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-mongodbatlas - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/mongodbatlas/** @@ -280,7 +383,7 @@ jobs: - name: Upload MongoDB Atlas coverage to Codecov if: steps.changed-mongodbatlas.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -291,7 +394,7 @@ jobs: - name: Check if OCI files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-oraclecloud - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/**/oraclecloud/** @@ -304,7 +407,7 @@ jobs: - name: Upload OCI coverage to Codecov if: steps.changed-oraclecloud.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -315,7 +418,7 @@ jobs: - name: Check if Lib files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-lib - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/lib/** @@ -328,7 +431,7 @@ jobs: - name: Upload Lib coverage to Codecov if: steps.changed-lib.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -339,7 +442,7 @@ jobs: - name: Check if Config files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-config - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ./prowler/config/** @@ -352,7 +455,7 @@ jobs: - name: Upload Config coverage to Codecov if: steps.changed-config.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: diff --git a/.github/workflows/ui-bump-version.yml b/.github/workflows/ui-bump-version.yml new file mode 100644 index 0000000000..7c576eed55 --- /dev/null +++ b/.github/workflows/ui-bump-version.yml @@ -0,0 +1,221 @@ +name: 'UI: Bump Version' + +on: + release: + types: + - 'published' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false + +env: + PROWLER_VERSION: ${{ github.event.release.tag_name }} + BASE_BRANCH: master + +jobs: + detect-release-type: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + is_minor: ${{ steps.detect.outputs.is_minor }} + is_patch: ${{ steps.detect.outputs.is_patch }} + major_version: ${{ steps.detect.outputs.major_version }} + minor_version: ${{ steps.detect.outputs.minor_version }} + patch_version: ${{ steps.detect.outputs.patch_version }} + steps: + - name: Detect release type and parse version + id: detect + run: | + if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + MAJOR_VERSION=${BASH_REMATCH[1]} + MINOR_VERSION=${BASH_REMATCH[2]} + PATCH_VERSION=${BASH_REMATCH[3]} + + echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}" + + if (( MAJOR_VERSION != 5 )); then + echo "::error::Releasing another Prowler major version, aborting..." + exit 1 + fi + + if (( PATCH_VERSION == 0 )); then + echo "is_minor=true" >> "${GITHUB_OUTPUT}" + echo "is_patch=false" >> "${GITHUB_OUTPUT}" + echo "✓ Minor release detected: $PROWLER_VERSION" + else + echo "is_minor=false" >> "${GITHUB_OUTPUT}" + echo "is_patch=true" >> "${GITHUB_OUTPUT}" + echo "✓ Patch release detected: $PROWLER_VERSION" + fi + else + echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)" + exit 1 + fi + + bump-minor-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_minor == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Calculate next minor version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + + NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0 + echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}" + + echo "Current version: $PROWLER_VERSION" + echo "Next minor version: $NEXT_MINOR_VERSION" + + - name: Bump UI version in .env for master + run: | + set -e + + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for next minor version to master + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: master + commit-message: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}' + branch: ui-version-bump-to-v${{ env.NEXT_MINOR_VERSION }} + title: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}' + labels: no-changelog,skip-sync + body: | + ### Description + + Bump Prowler UI version to v${{ env.NEXT_MINOR_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}. + + ### Files Updated + - `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION` + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + - name: Checkout version branch + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }} + + - name: Calculate first patch version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + + FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1 + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} + + echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" + + echo "First patch version: $FIRST_PATCH_VERSION" + echo "Version branch: $VERSION_BRANCH" + + - name: Bump UI version in .env for version branch + run: | + set -e + + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for first patch version to version branch + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}' + branch: ui-version-bump-to-v${{ env.FIRST_PATCH_VERSION }} + title: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}' + labels: no-changelog,skip-sync + body: | + ### Description + + Bump Prowler UI version to v${{ env.FIRST_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}. + + ### Files Updated + - `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION` + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + bump-patch-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_patch == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Calculate next patch version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }} + + NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1)) + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} + + echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" + + echo "Current version: $PROWLER_VERSION" + echo "Next patch version: $NEXT_PATCH_VERSION" + echo "Target branch: $VERSION_BRANCH" + + - name: Bump UI version in .env for version branch + run: | + set -e + + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for next patch version to version branch + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}' + branch: ui-version-bump-to-v${{ env.NEXT_PATCH_VERSION }} + title: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}' + labels: no-changelog,skip-sync + body: | + ### Description + + Bump Prowler UI version to v${{ env.NEXT_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}. + + ### Files Updated + - `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION` + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. diff --git a/.github/workflows/ui-codeql.yml b/.github/workflows/ui-codeql.yml index 2b55cf673c..581e798189 100644 --- a/.github/workflows/ui-codeql.yml +++ b/.github/workflows/ui-codeql.yml @@ -45,15 +45,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Initialize CodeQL - uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/ui-codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/ui-container-build-push.yml b/.github/workflows/ui-container-build-push.yml index 8211ba24e3..6f8c232185 100644 --- a/.github/workflows/ui-container-build-push.yml +++ b/.github/workflows/ui-container-build-push.yml @@ -50,8 +50,34 @@ jobs: id: set-short-sha run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT - container-build-push: + notify-release-started: + if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') needs: setup + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + message-ts: ${{ steps.slack-notification.outputs.ts }} + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Notify container push started + id: slack-notification + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: UI + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + container-build-push: + needs: [setup, notify-release-started] + if: always() && needs.setup.result == 'success' && (needs.notify-release-started.result == 'success' || needs.notify-release-started.result == 'skipped') runs-on: ${{ matrix.runner }} strategy: matrix: @@ -69,7 +95,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Login to DockerHub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 @@ -78,21 +104,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Notify container push started - if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' - uses: ./.github/actions/slack-notification - env: - SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} - COMPONENT: UI - RELEASE_TAG: ${{ env.RELEASE_TAG }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - with: - slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build and push UI container for ${{ matrix.arch }} id: container-push @@ -110,36 +122,21 @@ jobs: cache-from: type=gha,scope=${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ matrix.arch }} - - name: Notify container push completed - if: (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && always() - uses: ./.github/actions/slack-notification - env: - SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} - COMPONENT: UI - RELEASE_TAG: ${{ env.RELEASE_TAG }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - with: - slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" - step-outcome: ${{ steps.container-push.outcome }} - # Create and push multi-architecture manifest create-manifest: needs: [setup, container-build-push] - if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' + if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest steps: - name: Login to DockerHub - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Create and push manifests for push event if: github.event_name == 'push' @@ -171,9 +168,43 @@ jobs: regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true echo "Cleanup completed" + notify-release-completed: + if: always() && needs.notify-release-started.result == 'success' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') + needs: [setup, notify-release-started, container-build-push, create-manifest] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Determine overall outcome + id: outcome + run: | + if [[ "${{ needs.container-build-push.result }}" == "success" && "${{ needs.create-manifest.result }}" == "success" ]]; then + echo "outcome=success" >> $GITHUB_OUTPUT + else + echo "outcome=failure" >> $GITHUB_OUTPUT + fi + + - name: Notify container push completed + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + MESSAGE_TS: ${{ needs.notify-release-started.outputs.message-ts }} + COMPONENT: UI + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.outcome.outputs.outcome }} + update-ts: ${{ needs.notify-release-started.outputs.message-ts }} + trigger-deployment: - if: github.event_name == 'push' needs: [setup, container-build-push] + if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -181,7 +212,7 @@ jobs: steps: - name: Trigger UI deployment - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} repository: ${{ secrets.CLOUD_DISPATCH }} diff --git a/.github/workflows/ui-container-checks.yml b/.github/workflows/ui-container-checks.yml index e436c3974a..4c027f1cb2 100644 --- a/.github/workflows/ui-container-checks.yml +++ b/.github/workflows/ui-container-checks.yml @@ -20,6 +20,7 @@ env: jobs: ui-dockerfile-lint: + if: github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -27,11 +28,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check if Dockerfile changed id: dockerfile-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: ui/Dockerfile @@ -43,7 +44,17 @@ jobs: ignore: DL3018 ui-container-build-and-scan: - runs-on: ubuntu-latest + if: github.repository == 'prowler-cloud/prowler' + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 timeout-minutes: 30 permissions: contents: read @@ -52,22 +63,23 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for UI changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: ui/** files_ignore: | ui/CHANGELOG.md ui/README.md + ui/AGENTS.md - name: Set up Docker Buildx if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Build UI container + - name: Build UI container for ${{ matrix.arch }} if: steps.check-changes.outputs.any_changed == 'true' uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: @@ -75,17 +87,18 @@ jobs: target: prod push: false load: true - tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + platforms: ${{ matrix.platform }} + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} build-args: | NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX - - name: Scan UI container with Trivy - if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' + - name: Scan UI container with Trivy for ${{ matrix.arch }} + if: steps.check-changes.outputs.any_changed == 'true' uses: ./.github/actions/trivy-scan with: image-name: ${{ env.IMAGE_NAME }} - image-tag: ${{ github.sha }} + image-tag: ${{ github.sha }}-${{ matrix.arch }} fail-on-critical: 'false' severity: 'CRITICAL' diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index ed3b692819..054bd69f6e 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -10,6 +10,7 @@ on: - 'ui/**' jobs: + e2e-tests: if: github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest @@ -33,12 +34,50 @@ jobs: E2E_M365_SECRET_ID: ${{ secrets.E2E_M365_SECRET_ID }} E2E_M365_TENANT_ID: ${{ secrets.E2E_M365_TENANT_ID }} E2E_M365_CERTIFICATE_CONTENT: ${{ secrets.E2E_M365_CERTIFICATE_CONTENT }} - E2E_NEW_PASSWORD: ${{ secrets.E2E_NEW_PASSWORD }} + E2E_KUBERNETES_CONTEXT: 'kind-kind' + E2E_KUBERNETES_KUBECONFIG_PATH: /home/runner/.kube/config + E2E_GCP_BASE64_SERVICE_ACCOUNT_KEY: ${{ secrets.E2E_GCP_BASE64_SERVICE_ACCOUNT_KEY }} + E2E_GCP_PROJECT_ID: ${{ secrets.E2E_GCP_PROJECT_ID }} + E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }} + E2E_GITHUB_BASE64_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_BASE64_APP_PRIVATE_KEY }} + E2E_GITHUB_USERNAME: ${{ secrets.E2E_GITHUB_USERNAME }} + E2E_GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.E2E_GITHUB_PERSONAL_ACCESS_TOKEN }} + E2E_GITHUB_ORGANIZATION: ${{ secrets.E2E_GITHUB_ORGANIZATION }} + E2E_GITHUB_ORGANIZATION_ACCESS_TOKEN: ${{ secrets.E2E_GITHUB_ORGANIZATION_ACCESS_TOKEN }} + E2E_ORGANIZATION_ID: ${{ secrets.E2E_ORGANIZATION_ID }} + E2E_OCI_TENANCY_ID: ${{ secrets.E2E_OCI_TENANCY_ID }} + E2E_OCI_USER_ID: ${{ secrets.E2E_OCI_USER_ID }} + E2E_OCI_FINGERPRINT: ${{ secrets.E2E_OCI_FINGERPRINT }} + E2E_OCI_KEY_CONTENT: ${{ secrets.E2E_OCI_KEY_CONTENT }} + E2E_OCI_REGION: ${{ secrets.E2E_OCI_REGION }} + E2E_NEW_USER_PASSWORD: ${{ secrets.E2E_NEW_USER_PASSWORD }} + steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1 + with: + cluster_name: kind + - name: Modify kubeconfig + run: | + # Modify the kubeconfig to use the kind cluster server to https://kind-control-plane:6443 + # from worker service into docker-compose.yml + kubectl config set-cluster kind-kind --server=https://kind-control-plane:6443 + kubectl config view + - name: Add network kind to docker compose + run: | + # Add the network kind to the docker compose to interconnect to kind cluster + yq -i '.networks.kind.external = true' docker-compose.yml + # Add network kind to worker service and default network too + yq -i '.services.worker.networks = ["kind","default"]' docker-compose.yml - name: Fix API data directory permissions run: docker run --rm -v $(pwd)/_data/api:/data alpine chown -R 1000:1000 /data + - name: Add AWS credentials for testing AWS SDK Default Adding Provider + run: | + echo "Adding AWS credentials for testing AWS SDK Default Adding Provider..." + echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env + echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env - name: Start API services run: | # Override docker-compose image tag to use latest instead of stable @@ -75,34 +114,47 @@ jobs: echo "All database fixtures loaded successfully!" ' - name: Setup Node.js environment - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 with: node-version: '20.x' - cache: 'npm' - cache-dependency-path: './ui/package-lock.json' + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10 + run_install: false + - name: Get pnpm store directory + shell: bash + run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + - name: Setup pnpm cache + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- - name: Install UI dependencies working-directory: ./ui - run: npm ci + run: pnpm install --frozen-lockfile - name: Build UI application working-directory: ./ui - run: npm run build + run: pnpm run build - name: Cache Playwright browsers - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 id: playwright-cache with: path: ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ hashFiles('ui/package-lock.json') }} + key: ${{ runner.os }}-playwright-${{ hashFiles('ui/pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-playwright- - name: Install Playwright browsers working-directory: ./ui if: steps.playwright-cache.outputs.cache-hit != 'true' - run: npm run test:e2e:install + run: pnpm run test:e2e:install - name: Run E2E tests working-directory: ./ui - run: npm run test:e2e + run: pnpm run test:e2e - name: Upload test reports - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: failure() with: name: playwright-report diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index a5a5b5fa17..e657e56960 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -30,11 +30,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check for UI changes id: check-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: | ui/** @@ -42,23 +42,43 @@ jobs: files_ignore: | ui/CHANGELOG.md ui/README.md + ui/AGENTS.md - name: Setup Node.js ${{ env.NODE_VERSION }} if: steps.check-changes.outputs.any_changed == 'true' - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 with: node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - cache-dependency-path: './ui/package-lock.json' + + - name: Setup pnpm + if: steps.check-changes.outputs.any_changed == 'true' + uses: pnpm/action-setup@v4 + with: + version: 10 + run_install: false + + - name: Get pnpm store directory + if: steps.check-changes.outputs.any_changed == 'true' + shell: bash + run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Setup pnpm cache + if: steps.check-changes.outputs.any_changed == 'true' + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- - name: Install dependencies if: steps.check-changes.outputs.any_changed == 'true' - run: npm ci + run: pnpm install --frozen-lockfile - name: Run healthcheck if: steps.check-changes.outputs.any_changed == 'true' - run: npm run healthcheck + run: pnpm run healthcheck - name: Build application if: steps.check-changes.outputs.any_changed == 'true' - run: npm run build + run: pnpm run build diff --git a/.gitignore b/.gitignore index 7792a031b4..4dfb137b89 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,9 @@ continue.json .continuerc .continuerc.json +# AI Coding Assistants - OpenCode +opencode.json + # AI Coding Assistants - GitHub Copilot .copilot/ .github/copilot/ @@ -147,12 +150,16 @@ node_modules # Persistent data _data/ -# Claude +# AI Instructions (generated by skills/setup.sh from AGENTS.md) CLAUDE.md - -# MCP Server -mcp_server/prowler_mcp_server/prowler_app/server.py -mcp_server/prowler_mcp_server/prowler_app/utils/schema.yaml +GEMINI.md +.github/copilot-instructions.md # Compliance report *.pdf + +# AI Skills symlinks (generated by skills/setup.sh) +.claude/skills +.codex/skills +.github/skills +.gemini/skills diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea9954f082..24f8f0f211 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,6 +34,7 @@ repos: rev: v2.3.1 hooks: - id: autoflake + exclude: ^skills/ args: [ "--in-place", @@ -45,18 +46,20 @@ repos: rev: 5.13.2 hooks: - id: isort + exclude: ^skills/ args: ["--profile", "black"] - repo: https://github.com/psf/black rev: 24.4.2 hooks: - id: black + exclude: ^skills/ - repo: https://github.com/pycqa/flake8 rev: 7.0.0 hooks: - id: flake8 - exclude: contrib + exclude: (contrib|^skills/) args: ["--ignore=E266,W503,E203,E501,W605"] - repo: https://github.com/python-poetry/poetry @@ -109,7 +112,7 @@ repos: - id: bandit name: bandit description: "Bandit is a tool for finding common security issues in Python code" - entry: bash -c 'bandit -q -lll -x '*_test.py,./contrib/,./.venv/' -r .' + entry: bash -c 'bandit -q -lll -x '*_test.py,./contrib/,./.venv/,./skills/' -r .' language: system files: '.*\.py' @@ -123,7 +126,7 @@ repos: - id: vulture name: vulture description: "Vulture finds unused code in Python programs." - entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/" --min-confidence 100 .' + entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/,skills/" --min-confidence 100 .' language: system files: '.*\.py' diff --git a/AGENTS.md b/AGENTS.md index c6a6027c18..62ebb587f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,109 +2,135 @@ ## How to Use This Guide -- Start here for cross-project norms, Prowler is a monorepo with several components. Every component should have an `AGENTS.md` file that contains the guidelines for the agents in that component. The file is located beside the code you are touching (e.g. `api/AGENTS.md`, `ui/AGENTS.md`, `prowler/AGENTS.md`). -- Follow the stricter rule when guidance conflicts; component docs override this file for their scope. -- Keep instructions synchronized. When you add new workflows or scripts, update both, the relevant component `AGENTS.md` and this file if they apply broadly. +- Start here for cross-project norms. Prowler is a monorepo with several components. +- Each component has an `AGENTS.md` file with specific guidelines (e.g., `api/AGENTS.md`, `ui/AGENTS.md`). +- Component docs override this file when guidance conflicts. + +## Available Skills + +Use these skills for detailed patterns on-demand: + +### Generic Skills (Any Project) +| Skill | Description | URL | +|-------|-------------|-----| +| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) | +| `react-19` | No useMemo/useCallback, React Compiler | [SKILL.md](skills/react-19/SKILL.md) | +| `nextjs-15` | App Router, Server Actions, streaming | [SKILL.md](skills/nextjs-15/SKILL.md) | +| `tailwind-4` | cn() utility, no var() in className | [SKILL.md](skills/tailwind-4/SKILL.md) | +| `playwright` | Page Object Model, MCP workflow, selectors | [SKILL.md](skills/playwright/SKILL.md) | +| `pytest` | Fixtures, mocking, markers, parametrize | [SKILL.md](skills/pytest/SKILL.md) | +| `django-drf` | ViewSets, Serializers, Filters | [SKILL.md](skills/django-drf/SKILL.md) | +| `zod-4` | New API (z.email(), z.uuid()) | [SKILL.md](skills/zod-4/SKILL.md) | +| `zustand-5` | Persist, selectors, slices | [SKILL.md](skills/zustand-5/SKILL.md) | +| `ai-sdk-5` | UIMessage, streaming, LangChain | [SKILL.md](skills/ai-sdk-5/SKILL.md) | + +### Prowler-Specific Skills +| Skill | Description | URL | +|-------|-------------|-----| +| `prowler` | Project overview, component navigation | [SKILL.md](skills/prowler/SKILL.md) | +| `prowler-api` | Django + RLS + JSON:API patterns | [SKILL.md](skills/prowler-api/SKILL.md) | +| `prowler-ui` | Next.js + shadcn conventions | [SKILL.md](skills/prowler-ui/SKILL.md) | +| `prowler-sdk-check` | Create new security checks | [SKILL.md](skills/prowler-sdk-check/SKILL.md) | +| `prowler-mcp` | MCP server tools and models | [SKILL.md](skills/prowler-mcp/SKILL.md) | +| `prowler-test-sdk` | SDK testing (pytest + moto) | [SKILL.md](skills/prowler-test-sdk/SKILL.md) | +| `prowler-test-api` | API testing (pytest-django + RLS) | [SKILL.md](skills/prowler-test-api/SKILL.md) | +| `prowler-test-ui` | E2E testing (Playwright) | [SKILL.md](skills/prowler-test-ui/SKILL.md) | +| `prowler-compliance` | Compliance framework structure | [SKILL.md](skills/prowler-compliance/SKILL.md) | +| `prowler-compliance-review` | Review compliance framework PRs | [SKILL.md](skills/prowler-compliance-review/SKILL.md) | +| `prowler-provider` | Add new cloud providers | [SKILL.md](skills/prowler-provider/SKILL.md) | +| `prowler-ci` | CI checks and PR gates (GitHub Actions) | [SKILL.md](skills/prowler-ci/SKILL.md) | +| `prowler-pr` | Pull request conventions | [SKILL.md](skills/prowler-pr/SKILL.md) | +| `prowler-docs` | Documentation style guide | [SKILL.md](skills/prowler-docs/SKILL.md) | +| `skill-creator` | Create new AI agent skills | [SKILL.md](skills/skill-creator/SKILL.md) | + +### Auto-invoke Skills + +When performing these actions, ALWAYS invoke the corresponding skill FIRST: + +| Action | Skill | +|--------|-------| +| Adding new providers | `prowler-provider` | +| Adding services to existing providers | `prowler-provider` | +| After creating/modifying a skill | `skill-sync` | +| App Router / Server Actions | `nextjs-15` | +| Building AI chat features | `ai-sdk-5` | +| Create a PR with gh pr create | `prowler-pr` | +| Creating Zod schemas | `zod-4` | +| Creating new checks | `prowler-sdk-check` | +| Creating new skills | `skill-creator` | +| Creating/modifying Prowler UI components | `prowler-ui` | +| Creating/modifying models, views, serializers | `prowler-api` | +| Creating/updating compliance frameworks | `prowler-compliance` | +| Debug why a GitHub Actions job is failing | `prowler-ci` | +| Fill .github/pull_request_template.md (Context/Description/Steps to review/Checklist) | `prowler-pr` | +| General Prowler development questions | `prowler` | +| Generic DRF patterns | `django-drf` | +| Inspect PR CI checks and gates (.github/workflows/*) | `prowler-ci` | +| Inspect PR CI workflows (.github/workflows/*): conventional-commit, pr-check-changelog, pr-conflict-checker, labeler | `prowler-pr` | +| Mapping checks to compliance controls | `prowler-compliance` | +| Mocking AWS with moto in tests | `prowler-test-sdk` | +| Regenerate AGENTS.md Auto-invoke tables (sync.sh) | `skill-sync` | +| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` | +| Reviewing compliance framework PRs | `prowler-compliance-review` | +| Testing RLS tenant isolation | `prowler-test-api` | +| Troubleshoot why a skill is missing from AGENTS.md auto-invoke | `skill-sync` | +| Understand CODEOWNERS/labeler-based automation | `prowler-ci` | +| Understand PR title conventional-commit validation | `prowler-ci` | +| Understand changelog gate and no-changelog label behavior | `prowler-ci` | +| Understand review ownership with CODEOWNERS | `prowler-pr` | +| Updating existing checks and metadata | `prowler-sdk-check` | +| Using Zustand stores | `zustand-5` | +| Working on MCP server tools | `prowler-mcp` | +| Working on Prowler UI structure (actions/adapters/types/hooks) | `prowler-ui` | +| Working with Prowler UI test helpers/pages | `prowler-test-ui` | +| Working with Tailwind classes | `tailwind-4` | +| Writing Playwright E2E tests | `playwright` | +| Writing Prowler API tests | `prowler-test-api` | +| Writing Prowler SDK tests | `prowler-test-sdk` | +| Writing Prowler UI E2E tests | `prowler-test-ui` | +| Writing Python tests with pytest | `pytest` | +| Writing React components | `react-19` | +| Writing TypeScript types/interfaces | `typescript` | +| Writing documentation | `prowler-docs` | + +--- ## Project Overview -Prowler is an open-source cloud security assessment tool that supports multiple cloud providers (AWS, Azure, GCP, Kubernetes, GitHub, M365, etc.). The project consists in a monorepo with the following main components: +Prowler is an open-source cloud security assessment tool supporting AWS, Azure, GCP, Kubernetes, GitHub, M365, and more. -- **Prowler SDK**: Python SDK, includes the Prowler CLI, providers, services, checks, compliances, config, etc. (`prowler/`) -- **Prowler API**: Django-based REST API backend (`api/`) -- **Prowler UI**: Next.js frontend application (`ui/`) -- **Prowler MCP Server**: Model Context Protocol server that gives access to the entire Prowler ecosystem for LLMs (`mcp_server/`) -- **Prowler Dashboard**: Prowler CLI feature that allows to visualize the results of the scans in a simple dashboard (`dashboard/`) +| Component | Location | Tech Stack | +|-----------|----------|------------| +| SDK | `prowler/` | Python 3.9+, Poetry | +| API | `api/` | Django 5.1, DRF, Celery | +| UI | `ui/` | Next.js 15, React 19, Tailwind 4 | +| MCP Server | `mcp_server/` | FastMCP, Python 3.12+ | +| Dashboard | `dashboard/` | Dash, Plotly | -### Project Structure (Key Folders & Files) - -- `prowler/`: Main source code for Prowler SDK (CLI, providers, services, checks, compliances, config, etc.) -- `api/`: Django-based REST API backend components -- `ui/`: Next.js frontend application -- `mcp_server/`: Model Context Protocol server that gives access to the entire Prowler ecosystem for LLMs -- `dashboard/`: Prowler CLI feature that allows to visualize the results of the scans in a simple dashboard -- `docs/`: Documentation -- `examples/`: Example output formats for providers and scripts -- `permissions/`: Permission-related files and policies -- `contrib/`: Community-contributed scripts or modules -- `tests/`: Prowler SDK test suite -- `docker-compose.yml`: Docker compose file to run the Prowler App (API + UI) production environment -- `docker-compose-dev.yml`: Docker compose file to run the Prowler App (API + UI) development environment -- `pyproject.toml`: Poetry Prowler SDK project file -- `.pre-commit-config.yaml`: Pre-commit hooks configuration -- `Makefile`: Makefile to run the project -- `LICENSE`: License file -- `README.md`: README file -- `CONTRIBUTING.md`: Contributing guide +--- ## Python Development -Most of the code is written in Python, so the main files in the root are focused on Python code. - -### Poetry Dev Environment - -For developing in Python we recommend using `poetry` to manage the dependencies. The minimal version is `2.1.1`. So it is recommended to run all commands using `poetry run ...`. - -To install the core dependencies to develop it is needed to run `poetry install --with dev`. - -### Pre-commit hooks - -The project has pre-commit hooks to lint and format the code. They are installed by running `poetry run pre-commit install`. - -When commiting a change, the hooks will be run automatically. Some of them are: - -- Code formatting (black, isort) -- Linting (flake8, pylint) -- Security checks (bandit, safety, trufflehog) -- YAML/JSON validation -- Poetry lock file validation - - -### Linting and Formatting - -We use the following tools to lint and format the code: - -- `flake8`: for linting the code -- `black`: for formatting the code -- `pylint`: for linting the code - -You can run all using the `make` command: ```bash +# Setup +poetry install --with dev +poetry run pre-commit install + +# Code quality poetry run make lint poetry run make format +poetry run pre-commit run --all-files ``` -Or they will be run automatically when you commit your changes using pre-commit hooks. +--- ## Commit & Pull Request Guidelines -For the commit messages and pull requests name follow the conventional-commit style. +Follow conventional-commit style: `[scope]: ` -Befire creating a pull request, complete the checklist in `.github/pull_request_template.md`. Summaries should explain deployment impact, highlight review steps, and note changelog or permission updates. Run all relevant tests and linters before requesting review and link screenshots for UI or dashboard changes. +**Types:** `feat`, `fix`, `docs`, `chore`, `perf`, `refactor`, `style`, `test` -### Conventional Commit Style - -The Conventional Commits specification is a lightweight convention on top of commit messages. It provides an easy set of rules for creating an explicit commit history; which makes it easier to write automated tools on top of. - -The commit message should be structured as follows: - -``` -[optional scope]: - -[optional body] - -[optional footer(s)] -``` - -Any line of the commit message cannot be longer 100 characters! This allows the message to be easier to read on GitHub as well as in various git tools - -#### Commit Types - -- **feat**: code change introuce new functionality to the application -- **fix**: code change that solve a bug in the codebase -- **docs**: documentation only changes -- **chore**: changes related to the build process or auxiliary tools and libraries, that do not affect the application's functionality -- **perf**: code change that improves performance -- **refactor**: code change that neither fixes a bug nor adds a feature -- **style**: changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc) -- **test**: adding missing tests or correcting existing tests +Before creating a PR: +1. Complete checklist in `.github/pull_request_template.md` +2. Run all relevant tests and linters +3. Link screenshots for UI changes diff --git a/Dockerfile b/Dockerfile index f71c53e008..6819f4736b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ ENV TRIVY_VERSION=${TRIVY_VERSION} # hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ wget libicu72 libunwind8 libssl3 libcurl4 ca-certificates apt-transport-https gnupg \ + build-essential pkg-config libzstd-dev zlib1g-dev \ && rm -rf /var/lib/apt/lists/* # Install PowerShell diff --git a/Makefile b/Makefile index 861c9cf7fe..2000d6d5bd 100644 --- a/Makefile +++ b/Makefile @@ -47,12 +47,12 @@ help: ## Show this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) ##@ Build no cache -build-no-cache-dev: - docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat +build-no-cache-dev: + docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat mcp-server ##@ Development Environment -run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, and workers - docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat +run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, MCP, and workers + docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat mcp-server ##@ Development Environment build-and-run-api-dev: build-no-cache-dev run-api-dev diff --git a/README.md b/README.md index dc3e74a57b..928ebc0655 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Prowler is the Open Cloud Security platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.

-Learn more at prowler.com +Secure ANY cloud at AI Speed at prowler.com

@@ -23,6 +23,7 @@ Docker Pulls AWS ECR Gallery +

Version @@ -35,28 +36,32 @@


- +

# Description -**Prowler** is an open-source security tool designed to assess and enforce security best practices across AWS, Azure, Google Cloud, and Kubernetes. It supports tasks such as security audits, incident response, continuous monitoring, system hardening, forensic readiness, and remediation processes. +**Prowler** is the world’s most widely used _open-source cloud security platform_ that automates security and compliance across **any cloud environment**. With hundreds of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size. Prowler includes hundreds of built-in controls to ensure compliance with standards and frameworks, including: -- **Industry Standards:** CIS, NIST 800, NIST CSF, and CISA -- **Regulatory Compliance and Governance:** RBI, FedRAMP, and PCI-DSS +- **Prowler ThreatScore:** Weighted risk prioritization scoring that helps you focus on the most critical security findings first +- **Industry Standards:** CIS, NIST 800, NIST CSF, CISA, and MITRE ATT&CK +- **Regulatory Compliance and Governance:** RBI, FedRAMP, PCI-DSS, and NIS2 - **Frameworks for Sensitive Data and Privacy:** GDPR, HIPAA, and FFIEC -- **Frameworks for Organizational Governance and Quality Control:** SOC2 and GXP -- **AWS-Specific Frameworks:** AWS Foundational Technical Review (FTR) and AWS Well-Architected Framework (Security Pillar) -- **National Security Standards:** ENS (Spanish National Security Scheme) +- **Frameworks for Organizational Governance and Quality Control:** SOC2, GXP, and ISO 27001 +- **Cloud-Specific Frameworks:** AWS Foundational Technical Review (FTR), AWS Well-Architected Framework, and BSI C5 +- **National Security Standards:** ENS (Spanish National Security Scheme) and KISA ISMS-P (Korean) - **Custom Security Frameworks:** Tailored to your needs -## Prowler App +## Prowler App / Prowler Cloud -Prowler App is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments. +Prowler App / [Prowler Cloud](https://cloud.prowler.com/) is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments. ![Prowler App](docs/images/products/overview.png) +![Risk Pipeline](docs/images/products/risk-pipeline.png) +![Threat Map](docs/images/products/threat-map.png) + >For more details, refer to the [Prowler App Documentation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-app-installation) @@ -99,15 +104,16 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically | Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface | |---|---|---|---|---|---|---| -| AWS | 576 | 82 | 39 | 10 | Official | UI, API, CLI | -| GCP | 79 | 13 | 13 | 3 | Official | UI, API, CLI | -| Azure | 162 | 19 | 13 | 4 | Official | UI, API, CLI | -| Kubernetes | 83 | 7 | 5 | 7 | Official | UI, API, CLI | -| GitHub | 17 | 2 | 1 | 0 | Official | Stable | UI, API, CLI | +| AWS | 584 | 85 | 40 | 17 | Official | UI, API, CLI | +| GCP | 89 | 17 | 14 | 5 | Official | UI, API, CLI | +| Azure | 169 | 22 | 15 | 8 | Official | UI, API, CLI | +| Kubernetes | 84 | 7 | 6 | 9 | Official | UI, API, CLI | +| GitHub | 20 | 2 | 1 | 2 | Official | UI, API, CLI | | M365 | 70 | 7 | 3 | 2 | Official | UI, API, CLI | -| OCI | 51 | 13 | 1 | 10 | Official | UI, API, CLI | +| OCI | 52 | 15 | 1 | 12 | Official | UI, API, CLI | +| Alibaba Cloud | 63 | 10 | 1 | 9 | Official | CLI | | IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI | -| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | UI, API, CLI | +| MongoDB Atlas | 10 | 4 | 0 | 3 | Official | UI, API, CLI | | LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI | | NHN | 6 | 2 | 1 | 0 | Unofficial | CLI | @@ -159,9 +165,9 @@ If your workstation's architecture is incompatible, you can resolve this by: ### Common Issues with Docker Pull Installation > [!Note] - If you want to use AWS role assumption (e.g., with the "Connect assuming IAM Role" option), you may need to mount your local `.aws` directory into the container as a volume (e.g., `- "${HOME}/.aws:/home/prowler/.aws:ro"`). There are several ways to configure credentials for Docker containers. See the [Troubleshooting](./docs/troubleshooting.md) section for more details and examples. + If you want to use AWS role assumption (e.g., with the "Connect assuming IAM Role" option), you may need to mount your local `.aws` directory into the container as a volume (e.g., `- "${HOME}/.aws:/home/prowler/.aws:ro"`). There are several ways to configure credentials for Docker containers. See the [Troubleshooting](./docs/troubleshooting.mdx) section for more details and examples. -You can find more information in the [Troubleshooting](./docs/troubleshooting.md) section. +You can find more information in the [Troubleshooting](./docs/troubleshooting.mdx) section. ### From GitHub @@ -170,7 +176,7 @@ You can find more information in the [Troubleshooting](./docs/troubleshooting.md * `git` installed. * `poetry` v2 installed: [poetry installation](https://python-poetry.org/docs/#installation). -* `npm` installed: [npm installation](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). +* `pnpm` installed: [pnpm installation](https://pnpm.io/installation). * `Docker Compose` installed: https://docs.docker.com/compose/install/. **Commands to run the API** @@ -226,9 +232,9 @@ python -m celery -A config.celery beat -l info --scheduler django_celery_beat.sc ``` console git clone https://github.com/prowler-cloud/prowler cd prowler/ui -npm install -npm run build -npm start +pnpm install +pnpm run build +pnpm start ``` > Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started. @@ -288,11 +294,12 @@ python prowler-cli.py -v # ✏️ High level architecture ## Prowler App -**Prowler App** is composed of three key components: +**Prowler App** is composed of four key components: - **Prowler UI**: A web-based interface, built with Next.js, providing a user-friendly experience for executing Prowler scans and visualizing results. - **Prowler API**: A backend service, developed with Django REST Framework, responsible for running Prowler scans and storing the generated results. - **Prowler SDK**: A Python SDK designed to extend the functionality of the Prowler CLI for advanced capabilities. +- **Prowler MCP Server**: A Model Context Protocol server that provides AI tools for Lighthouse, the AI-powered security assistant. This is a critical dependency for Lighthouse functionality. ![Prowler App Architecture](docs/products/img/prowler-app-architecture.png) @@ -320,6 +327,45 @@ And many more environments. ![Architecture](docs/img/architecture.png) +# 🤖 AI Skills for Development + +Prowler includes a comprehensive set of **AI Skills** that help AI coding assistants understand Prowler's codebase patterns and conventions. + +## What are AI Skills? + +Skills are structured instructions that give AI assistants the context they need to write code that follows Prowler's standards. They include: + +- **Coding patterns** for each component (SDK, API, UI, MCP Server) +- **Testing conventions** (pytest, Playwright) +- **Architecture guidelines** (Clean Architecture, RLS patterns) +- **Framework-specific rules** (React 19, Next.js 15, Django DRF, Tailwind 4) + +## Available Skills + +| Category | Skills | +|----------|--------| +| **Generic** | `typescript`, `react-19`, `nextjs-15`, `tailwind-4`, `playwright`, `pytest`, `django-drf`, `zod-4`, `zustand-5`, `ai-sdk-5` | +| **Prowler** | `prowler`, `prowler-api`, `prowler-ui`, `prowler-mcp`, `prowler-sdk-check`, `prowler-test-ui`, `prowler-test-api`, `prowler-test-sdk`, `prowler-compliance`, `prowler-provider`, `prowler-pr`, `prowler-docs` | + +## Setup + +```bash +./skills/setup.sh +``` + +This configures skills for AI coding assistants that follow the [agentskills.io](https://agentskills.io) standard: + +| Tool | Configuration | +|------|---------------| +| **Claude Code** | `.claude/skills/` (symlink) | +| **OpenCode** | `.claude/skills/` (symlink) | +| **Codex (OpenAI)** | `.codex/skills/` (symlink) | +| **GitHub Copilot** | `.github/skills/` (symlink) | +| **Gemini CLI** | `.gemini/skills/` (symlink) | + +> **Note:** Restart your AI coding assistant after running setup to load the skills. +> Gemini CLI requires `experimental.skills` enabled in settings. + # 📖 Documentation For installation instructions, usage details, tutorials, and the Developer Guide, visit https://docs.prowler.com/ diff --git a/api/AGENTS.md b/api/AGENTS.md new file mode 100644 index 0000000000..f43e0dcbb3 --- /dev/null +++ b/api/AGENTS.md @@ -0,0 +1,151 @@ +# Prowler API - AI Agent Ruleset + +> **Skills Reference**: For detailed patterns, use these skills: +> - [`prowler-api`](../skills/prowler-api/SKILL.md) - Models, Serializers, Views, RLS patterns +> - [`prowler-test-api`](../skills/prowler-test-api/SKILL.md) - Testing patterns (pytest-django) +> - [`django-drf`](../skills/django-drf/SKILL.md) - Generic DRF patterns +> - [`pytest`](../skills/pytest/SKILL.md) - Generic pytest patterns + +### Auto-invoke Skills + +When performing these actions, ALWAYS invoke the corresponding skill FIRST: + +| Action | Skill | +|--------|-------| +| Creating/modifying models, views, serializers | `prowler-api` | +| Generic DRF patterns | `django-drf` | +| Testing RLS tenant isolation | `prowler-test-api` | +| Writing Prowler API tests | `prowler-test-api` | +| Writing Python tests with pytest | `pytest` | + +--- + +## CRITICAL RULES - NON-NEGOTIABLE + +### Models +- ALWAYS: UUIDv4 PKs, `inserted_at`/`updated_at` timestamps, `JSONAPIMeta` class +- ALWAYS: Inherit from `RowLevelSecurityProtectedModel` for tenant-scoped data +- NEVER: Auto-increment integer PKs, models without tenant isolation + +### Serializers +- ALWAYS: Separate serializers for Create/Update operations +- ALWAYS: Inherit from `RLSSerializer` for tenant-scoped models +- NEVER: Write logic in serializers (use services/utils) + +### Views +- ALWAYS: Inherit from `BaseRLSViewSet` for tenant-scoped resources +- ALWAYS: Define `filterset_class`, use `@extend_schema` for OpenAPI +- NEVER: Raw SQL queries, business logic in views + +### Row-Level Security (RLS) +- ALWAYS: Use `rls_transaction(tenant_id)` context manager +- NEVER: Query across tenants, trust client-provided tenant_id + +### Celery Tasks +- ALWAYS: `@shared_task` with `name`, `queue`, `RLSTask` base class +- NEVER: Long-running ops in views, request context in tasks + +--- + +## DECISION TREES + +### Serializer Selection +``` +Read → Serializer +Create → CreateSerializer +Update → UpdateSerializer +Nested read → IncludeSerializer +``` + +### Task vs View +``` +< 100ms → View +> 100ms or external API → Celery task +Needs retry → Celery task +``` + +--- + +## TECH STACK + +Django 5.1.x | DRF 3.15.x | djangorestframework-jsonapi 7.x | Celery 5.4.x | PostgreSQL 16 | pytest 8.x + +--- + +## PROJECT STRUCTURE + +``` +api/src/backend/ +├── api/ # Main Django app +│ ├── v1/ # API version 1 (views, serializers, urls) +│ ├── models.py # Django models +│ ├── filters.py # FilterSet classes +│ ├── base_views.py # Base ViewSet classes +│ ├── rls.py # Row-Level Security +│ └── tests/ # Unit tests +├── config/ # Django configuration +└── tasks/ # Celery tasks +``` + +--- + +## COMMANDS + +```bash +# Development +poetry run python src/backend/manage.py runserver +poetry run celery -A config.celery worker -l INFO + +# Database +poetry run python src/backend/manage.py makemigrations +poetry run python src/backend/manage.py migrate + +# Testing & Linting +poetry run pytest -x --tb=short +poetry run make lint +``` + +--- + +## QA CHECKLIST + +- [ ] `poetry run pytest` passes +- [ ] `poetry run make lint` passes +- [ ] Migrations created if models changed +- [ ] New endpoints have `@extend_schema` decorators +- [ ] RLS properly applied for tenant data +- [ ] Tests cover success and error cases + +--- + +## NAMING CONVENTIONS + +| Entity | Pattern | Example | +|--------|---------|---------| +| Serializer (read) | `Serializer` | `ProviderSerializer` | +| Serializer (create) | `CreateSerializer` | `ProviderCreateSerializer` | +| Serializer (update) | `UpdateSerializer` | `ProviderUpdateSerializer` | +| Filter | `Filter` | `ProviderFilter` | +| ViewSet | `ViewSet` | `ProviderViewSet` | +| Task | `__task` | `sync_provider_resources_task` | + +--- + +## API CONVENTIONS (JSON:API) + +```json +{ + "data": { + "type": "providers", + "id": "uuid", + "attributes": { "name": "value" }, + "relationships": { "tenant": { "data": { "type": "tenants", "id": "uuid" } } } + } +} +``` + +- Content-Type: `application/vnd.api+json` +- Pagination: `?page[number]=1&page[size]=20` +- Filtering: `?filter[field]=value`, `?filter[field__in]=val1,val2` +- Sorting: `?sort=field`, `?sort=-field` +- Including: `?include=provider,findings` diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index e8ae6f8a1c..b5925a2905 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,13 +2,78 @@ All notable changes to the **Prowler API** are documented in this file. -## [1.16.0] (Unreleased) +## [1.18.0] (Prowler UNRELEASED) + +### Added +- `/api/v1/overviews/compliance-watchlist` to retrieve the compliance watchlist [(#9596)](https://github.com/prowler-cloud/prowler/pull/9596) +- Support AlibabaCloud provider [(#9485)](https://github.com/prowler-cloud/prowler/pull/9485) +- `provider_id` and `provider_id__in` filter aliases for findings endpoints to enable consistent frontend parameter naming [(#9701)](https://github.com/prowler-cloud/prowler/pull/9701) + +--- + +## [1.17.2] (Prowler v5.16.2) + +### Security +- Updated dependencies to patch security vulnerabilities: Django 5.1.15 (CVE-2025-64460, CVE-2025-13372), Werkzeug 3.1.4 (CVE-2025-66221), sqlparse 0.5.5 (PVE-2025-82038), fonttools 4.60.2 (CVE-2025-66034) [(#9730)](https://github.com/prowler-cloud/prowler/pull/9730) + +--- + +## [1.17.1] (Prowler v5.16.1) ### Added - Attack Paths backend support [(#9344)](https://github.com/prowler-cloud/prowler/pull/9344) ### Changed -- Restore the compliance overview endpoint's mandatory filters [(#9330)](https://github.com/prowler-cloud/prowler/pull/9330) +- Security Hub integration error when no regions [(#9635)](https://github.com/prowler-cloud/prowler/pull/9635) + +### Fixed +- Orphan scheduled scans caused by transaction isolation during provider creation [(#9633)](https://github.com/prowler-cloud/prowler/pull/9633) + +--- + +## [1.17.0] (Prowler v5.16.0) + +### Added +- New endpoint to retrieve and overview of the categories based on finding severities [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529) +- Endpoints `GET /findings` and `GET /findings/latests` can now use the category filter [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529) +- Account id, alias and provider name to PDF reporting table [(#9574)](https://github.com/prowler-cloud/prowler/pull/9574) + +### Changed +- Endpoint `GET /overviews/attack-surfaces` no longer returns the related check IDs [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529) +- OpenAI provider to only load chat-compatible models with tool calling support [(#9523)](https://github.com/prowler-cloud/prowler/pull/9523) +- Increased execution delay for the first scheduled scan tasks to 5 seconds[(#9558)](https://github.com/prowler-cloud/prowler/pull/9558) + +### Fixed +- Made `scan_id` a required filter in the compliance overview endpoint [(#9560)](https://github.com/prowler-cloud/prowler/pull/9560) +- Reduced unnecessary UPDATE resources operations by only saving when tag mappings change, lowering write load during scans [(#9569)](https://github.com/prowler-cloud/prowler/pull/9569) + +--- + +## [1.16.1] (Prowler v5.15.1) + +### Fixed +- Race condition in scheduled scan creation by adding countdown to task [(#9516)](https://github.com/prowler-cloud/prowler/pull/9516) + +## [1.16.0] (Prowler v5.15.0) + +### Added +- New endpoint to retrieve an overview of the attack surfaces [(#9309)](https://github.com/prowler-cloud/prowler/pull/9309) +- New endpoint `GET /api/v1/overviews/findings_severity/timeseries` to retrieve daily aggregated findings by severity level [(#9363)](https://github.com/prowler-cloud/prowler/pull/9363) +- Lighthouse AI support for Amazon Bedrock API key [(#9343)](https://github.com/prowler-cloud/prowler/pull/9343) +- Exception handler for provider deletions during scans [(#9414)](https://github.com/prowler-cloud/prowler/pull/9414) +- Support to use admin credentials through the read replica database [(#9440)](https://github.com/prowler-cloud/prowler/pull/9440) + +### Changed +- Error messages from Lighthouse celery tasks [(#9165)](https://github.com/prowler-cloud/prowler/pull/9165) +- Restore the compliance overview endpoint's mandatory filters [(#9338)](https://github.com/prowler-cloud/prowler/pull/9338) + +--- + +## [1.15.2] (Prowler v5.14.2) + +### Fixed +- Unique constraint violation during compliance overviews task [(#9436)](https://github.com/prowler-cloud/prowler/pull/9436) +- Division by zero error in ENS PDF report when all requirements are manual [(#9443)](https://github.com/prowler-cloud/prowler/pull/9443) --- @@ -17,6 +82,7 @@ All notable changes to the **Prowler API** are documented in this file. ### Fixed - Fix typo in PDF reporting [(#9345)](https://github.com/prowler-cloud/prowler/pull/9345) - Fix IaC provider initialization failure when mutelist processor is configured [(#9331)](https://github.com/prowler-cloud/prowler/pull/9331) +- Match logic for ThreatScore when counting findings [(#9348)](https://github.com/prowler-cloud/prowler/pull/9348) --- diff --git a/api/poetry.lock b/api/poetry.lock index 4bf8a3ddae..a01bc02f63 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -103,98 +103,132 @@ files = [ [[package]] name = "aiohttp" -version = "3.12.15" +version = "3.13.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "aiohttp-3.12.15-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b6fc902bff74d9b1879ad55f5404153e2b33a82e72a95c89cec5eb6cc9e92fbc"}, - {file = "aiohttp-3.12.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:098e92835b8119b54c693f2f88a1dec690e20798ca5f5fe5f0520245253ee0af"}, - {file = "aiohttp-3.12.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:40b3fee496a47c3b4a39a731954c06f0bd9bd3e8258c059a4beb76ac23f8e421"}, - {file = "aiohttp-3.12.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ce13fcfb0bb2f259fb42106cdc63fa5515fb85b7e87177267d89a771a660b79"}, - {file = "aiohttp-3.12.15-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3beb14f053222b391bf9cf92ae82e0171067cc9c8f52453a0f1ec7c37df12a77"}, - {file = "aiohttp-3.12.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c39e87afe48aa3e814cac5f535bc6199180a53e38d3f51c5e2530f5aa4ec58c"}, - {file = "aiohttp-3.12.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f1b4ce5bc528a6ee38dbf5f39bbf11dd127048726323b72b8e85769319ffc4"}, - {file = "aiohttp-3.12.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1004e67962efabbaf3f03b11b4c43b834081c9e3f9b32b16a7d97d4708a9abe6"}, - {file = "aiohttp-3.12.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8faa08fcc2e411f7ab91d1541d9d597d3a90e9004180edb2072238c085eac8c2"}, - {file = "aiohttp-3.12.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fe086edf38b2222328cdf89af0dde2439ee173b8ad7cb659b4e4c6f385b2be3d"}, - {file = "aiohttp-3.12.15-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:79b26fe467219add81d5e47b4a4ba0f2394e8b7c7c3198ed36609f9ba161aecb"}, - {file = "aiohttp-3.12.15-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b761bac1192ef24e16706d761aefcb581438b34b13a2f069a6d343ec8fb693a5"}, - {file = "aiohttp-3.12.15-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e153e8adacfe2af562861b72f8bc47f8a5c08e010ac94eebbe33dc21d677cd5b"}, - {file = "aiohttp-3.12.15-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:fc49c4de44977aa8601a00edbf157e9a421f227aa7eb477d9e3df48343311065"}, - {file = "aiohttp-3.12.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2776c7ec89c54a47029940177e75c8c07c29c66f73464784971d6a81904ce9d1"}, - {file = "aiohttp-3.12.15-cp310-cp310-win32.whl", hash = "sha256:2c7d81a277fa78b2203ab626ced1487420e8c11a8e373707ab72d189fcdad20a"}, - {file = "aiohttp-3.12.15-cp310-cp310-win_amd64.whl", hash = "sha256:83603f881e11f0f710f8e2327817c82e79431ec976448839f3cd05d7afe8f830"}, - {file = "aiohttp-3.12.15-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d3ce17ce0220383a0f9ea07175eeaa6aa13ae5a41f30bc61d84df17f0e9b1117"}, - {file = "aiohttp-3.12.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:010cc9bbd06db80fe234d9003f67e97a10fe003bfbedb40da7d71c1008eda0fe"}, - {file = "aiohttp-3.12.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3f9d7c55b41ed687b9d7165b17672340187f87a773c98236c987f08c858145a9"}, - {file = "aiohttp-3.12.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc4fbc61bb3548d3b482f9ac7ddd0f18c67e4225aaa4e8552b9f1ac7e6bda9e5"}, - {file = "aiohttp-3.12.15-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7fbc8a7c410bb3ad5d595bb7118147dfbb6449d862cc1125cf8867cb337e8728"}, - {file = "aiohttp-3.12.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74dad41b3458dbb0511e760fb355bb0b6689e0630de8a22b1b62a98777136e16"}, - {file = "aiohttp-3.12.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b6f0af863cf17e6222b1735a756d664159e58855da99cfe965134a3ff63b0b0"}, - {file = "aiohttp-3.12.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5b7fe4972d48a4da367043b8e023fb70a04d1490aa7d68800e465d1b97e493b"}, - {file = "aiohttp-3.12.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6443cca89553b7a5485331bc9bedb2342b08d073fa10b8c7d1c60579c4a7b9bd"}, - {file = "aiohttp-3.12.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c5f40ec615e5264f44b4282ee27628cea221fcad52f27405b80abb346d9f3f8"}, - {file = "aiohttp-3.12.15-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2abbb216a1d3a2fe86dbd2edce20cdc5e9ad0be6378455b05ec7f77361b3ab50"}, - {file = "aiohttp-3.12.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:db71ce547012a5420a39c1b744d485cfb823564d01d5d20805977f5ea1345676"}, - {file = "aiohttp-3.12.15-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ced339d7c9b5030abad5854aa5413a77565e5b6e6248ff927d3e174baf3badf7"}, - {file = "aiohttp-3.12.15-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:7c7dd29c7b5bda137464dc9bfc738d7ceea46ff70309859ffde8c022e9b08ba7"}, - {file = "aiohttp-3.12.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:421da6fd326460517873274875c6c5a18ff225b40da2616083c5a34a7570b685"}, - {file = "aiohttp-3.12.15-cp311-cp311-win32.whl", hash = "sha256:4420cf9d179ec8dfe4be10e7d0fe47d6d606485512ea2265b0d8c5113372771b"}, - {file = "aiohttp-3.12.15-cp311-cp311-win_amd64.whl", hash = "sha256:edd533a07da85baa4b423ee8839e3e91681c7bfa19b04260a469ee94b778bf6d"}, - {file = "aiohttp-3.12.15-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:802d3868f5776e28f7bf69d349c26fc0efadb81676d0afa88ed00d98a26340b7"}, - {file = "aiohttp-3.12.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2800614cd560287be05e33a679638e586a2d7401f4ddf99e304d98878c29444"}, - {file = "aiohttp-3.12.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8466151554b593909d30a0a125d638b4e5f3836e5aecde85b66b80ded1cb5b0d"}, - {file = "aiohttp-3.12.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e5a495cb1be69dae4b08f35a6c4579c539e9b5706f606632102c0f855bcba7c"}, - {file = "aiohttp-3.12.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6404dfc8cdde35c69aaa489bb3542fb86ef215fc70277c892be8af540e5e21c0"}, - {file = "aiohttp-3.12.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ead1c00f8521a5c9070fcb88f02967b1d8a0544e6d85c253f6968b785e1a2ab"}, - {file = "aiohttp-3.12.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6990ef617f14450bc6b34941dba4f12d5613cbf4e33805932f853fbd1cf18bfb"}, - {file = "aiohttp-3.12.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd736ed420f4db2b8148b52b46b88ed038d0354255f9a73196b7bbce3ea97545"}, - {file = "aiohttp-3.12.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c5092ce14361a73086b90c6efb3948ffa5be2f5b6fbcf52e8d8c8b8848bb97c"}, - {file = "aiohttp-3.12.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aaa2234bb60c4dbf82893e934d8ee8dea30446f0647e024074237a56a08c01bd"}, - {file = "aiohttp-3.12.15-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6d86a2fbdd14192e2f234a92d3b494dd4457e683ba07e5905a0b3ee25389ac9f"}, - {file = "aiohttp-3.12.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a041e7e2612041a6ddf1c6a33b883be6a421247c7afd47e885969ee4cc58bd8d"}, - {file = "aiohttp-3.12.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5015082477abeafad7203757ae44299a610e89ee82a1503e3d4184e6bafdd519"}, - {file = "aiohttp-3.12.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:56822ff5ddfd1b745534e658faba944012346184fbfe732e0d6134b744516eea"}, - {file = "aiohttp-3.12.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b2acbbfff69019d9014508c4ba0401822e8bae5a5fdc3b6814285b71231b60f3"}, - {file = "aiohttp-3.12.15-cp312-cp312-win32.whl", hash = "sha256:d849b0901b50f2185874b9a232f38e26b9b3d4810095a7572eacea939132d4e1"}, - {file = "aiohttp-3.12.15-cp312-cp312-win_amd64.whl", hash = "sha256:b390ef5f62bb508a9d67cb3bba9b8356e23b3996da7062f1a57ce1a79d2b3d34"}, - {file = "aiohttp-3.12.15-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9f922ffd05034d439dde1c77a20461cf4a1b0831e6caa26151fe7aa8aaebc315"}, - {file = "aiohttp-3.12.15-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2ee8a8ac39ce45f3e55663891d4b1d15598c157b4d494a4613e704c8b43112cd"}, - {file = "aiohttp-3.12.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3eae49032c29d356b94eee45a3f39fdf4b0814b397638c2f718e96cfadf4c4e4"}, - {file = "aiohttp-3.12.15-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97752ff12cc12f46a9b20327104448042fce5c33a624f88c18f66f9368091c7"}, - {file = "aiohttp-3.12.15-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:894261472691d6fe76ebb7fcf2e5870a2ac284c7406ddc95823c8598a1390f0d"}, - {file = "aiohttp-3.12.15-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5fa5d9eb82ce98959fc1031c28198b431b4d9396894f385cb63f1e2f3f20ca6b"}, - {file = "aiohttp-3.12.15-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0fa751efb11a541f57db59c1dd821bec09031e01452b2b6217319b3a1f34f3d"}, - {file = "aiohttp-3.12.15-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5346b93e62ab51ee2a9d68e8f73c7cf96ffb73568a23e683f931e52450e4148d"}, - {file = "aiohttp-3.12.15-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:049ec0360f939cd164ecbfd2873eaa432613d5e77d6b04535e3d1fbae5a9e645"}, - {file = "aiohttp-3.12.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b52dcf013b57464b6d1e51b627adfd69a8053e84b7103a7cd49c030f9ca44461"}, - {file = "aiohttp-3.12.15-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9b2af240143dd2765e0fb661fd0361a1b469cab235039ea57663cda087250ea9"}, - {file = "aiohttp-3.12.15-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ac77f709a2cde2cc71257ab2d8c74dd157c67a0558a0d2799d5d571b4c63d44d"}, - {file = "aiohttp-3.12.15-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:47f6b962246f0a774fbd3b6b7be25d59b06fdb2f164cf2513097998fc6a29693"}, - {file = "aiohttp-3.12.15-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:760fb7db442f284996e39cf9915a94492e1896baac44f06ae551974907922b64"}, - {file = "aiohttp-3.12.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad702e57dc385cae679c39d318def49aef754455f237499d5b99bea4ef582e51"}, - {file = "aiohttp-3.12.15-cp313-cp313-win32.whl", hash = "sha256:f813c3e9032331024de2eb2e32a88d86afb69291fbc37a3a3ae81cc9917fb3d0"}, - {file = "aiohttp-3.12.15-cp313-cp313-win_amd64.whl", hash = "sha256:1a649001580bdb37c6fdb1bebbd7e3bc688e8ec2b5c6f52edbb664662b17dc84"}, - {file = "aiohttp-3.12.15-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:691d203c2bdf4f4637792efbbcdcd157ae11e55eaeb5e9c360c1206fb03d4d98"}, - {file = "aiohttp-3.12.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8e995e1abc4ed2a454c731385bf4082be06f875822adc4c6d9eaadf96e20d406"}, - {file = "aiohttp-3.12.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bd44d5936ab3193c617bfd6c9a7d8d1085a8dc8c3f44d5f1dcf554d17d04cf7d"}, - {file = "aiohttp-3.12.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46749be6e89cd78d6068cdf7da51dbcfa4321147ab8e4116ee6678d9a056a0cf"}, - {file = "aiohttp-3.12.15-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0c643f4d75adea39e92c0f01b3fb83d57abdec8c9279b3078b68a3a52b3933b6"}, - {file = "aiohttp-3.12.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0a23918fedc05806966a2438489dcffccbdf83e921a1170773b6178d04ade142"}, - {file = "aiohttp-3.12.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74bdd8c864b36c3673741023343565d95bfbd778ffe1eb4d412c135a28a8dc89"}, - {file = "aiohttp-3.12.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a146708808c9b7a988a4af3821379e379e0f0e5e466ca31a73dbdd0325b0263"}, - {file = "aiohttp-3.12.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7011a70b56facde58d6d26da4fec3280cc8e2a78c714c96b7a01a87930a9530"}, - {file = "aiohttp-3.12.15-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3bdd6e17e16e1dbd3db74d7f989e8af29c4d2e025f9828e6ef45fbdee158ec75"}, - {file = "aiohttp-3.12.15-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:57d16590a351dfc914670bd72530fd78344b885a00b250e992faea565b7fdc05"}, - {file = "aiohttp-3.12.15-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:bc9a0f6569ff990e0bbd75506c8d8fe7214c8f6579cca32f0546e54372a3bb54"}, - {file = "aiohttp-3.12.15-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:536ad7234747a37e50e7b6794ea868833d5220b49c92806ae2d7e8a9d6b5de02"}, - {file = "aiohttp-3.12.15-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f0adb4177fa748072546fb650d9bd7398caaf0e15b370ed3317280b13f4083b0"}, - {file = "aiohttp-3.12.15-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:14954a2988feae3987f1eb49c706bff39947605f4b6fa4027c1d75743723eb09"}, - {file = "aiohttp-3.12.15-cp39-cp39-win32.whl", hash = "sha256:b784d6ed757f27574dca1c336f968f4e81130b27595e458e69457e6878251f5d"}, - {file = "aiohttp-3.12.15-cp39-cp39-win_amd64.whl", hash = "sha256:86ceded4e78a992f835209e236617bffae649371c4a50d5e5a3987f237db84b8"}, - {file = "aiohttp-3.12.15.tar.gz", hash = "sha256:4fc61385e9c98d72fcdf47e6dd81833f47b2f77c114c29cd64a361be57a763a2"}, + {file = "aiohttp-3.13.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7"}, + {file = "aiohttp-3.13.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821"}, + {file = "aiohttp-3.13.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845"}, + {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af"}, + {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940"}, + {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160"}, + {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7"}, + {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455"}, + {file = "aiohttp-3.13.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279"}, + {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e"}, + {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d"}, + {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808"}, + {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40"}, + {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29"}, + {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11"}, + {file = "aiohttp-3.13.3-cp310-cp310-win32.whl", hash = "sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd"}, + {file = "aiohttp-3.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c"}, + {file = "aiohttp-3.13.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b"}, + {file = "aiohttp-3.13.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64"}, + {file = "aiohttp-3.13.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea"}, + {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a"}, + {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540"}, + {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b"}, + {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3"}, + {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1"}, + {file = "aiohttp-3.13.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3"}, + {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440"}, + {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7"}, + {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c"}, + {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51"}, + {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4"}, + {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29"}, + {file = "aiohttp-3.13.3-cp311-cp311-win32.whl", hash = "sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239"}, + {file = "aiohttp-3.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f"}, + {file = "aiohttp-3.13.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c"}, + {file = "aiohttp-3.13.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168"}, + {file = "aiohttp-3.13.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d"}, + {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29"}, + {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3"}, + {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d"}, + {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463"}, + {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc"}, + {file = "aiohttp-3.13.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf"}, + {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033"}, + {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f"}, + {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679"}, + {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423"}, + {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce"}, + {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a"}, + {file = "aiohttp-3.13.3-cp312-cp312-win32.whl", hash = "sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046"}, + {file = "aiohttp-3.13.3-cp312-cp312-win_amd64.whl", hash = "sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57"}, + {file = "aiohttp-3.13.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c"}, + {file = "aiohttp-3.13.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9"}, + {file = "aiohttp-3.13.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3"}, + {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf"}, + {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6"}, + {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d"}, + {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261"}, + {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0"}, + {file = "aiohttp-3.13.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730"}, + {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91"}, + {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3"}, + {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4"}, + {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998"}, + {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0"}, + {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591"}, + {file = "aiohttp-3.13.3-cp313-cp313-win32.whl", hash = "sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf"}, + {file = "aiohttp-3.13.3-cp313-cp313-win_amd64.whl", hash = "sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e"}, + {file = "aiohttp-3.13.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808"}, + {file = "aiohttp-3.13.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415"}, + {file = "aiohttp-3.13.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f"}, + {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6"}, + {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687"}, + {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26"}, + {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a"}, + {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1"}, + {file = "aiohttp-3.13.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25"}, + {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603"}, + {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a"}, + {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926"}, + {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba"}, + {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c"}, + {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43"}, + {file = "aiohttp-3.13.3-cp314-cp314-win32.whl", hash = "sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1"}, + {file = "aiohttp-3.13.3-cp314-cp314-win_amd64.whl", hash = "sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984"}, + {file = "aiohttp-3.13.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c"}, + {file = "aiohttp-3.13.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592"}, + {file = "aiohttp-3.13.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f"}, + {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29"}, + {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc"}, + {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2"}, + {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587"}, + {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8"}, + {file = "aiohttp-3.13.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632"}, + {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64"}, + {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0"}, + {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56"}, + {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72"}, + {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df"}, + {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa"}, + {file = "aiohttp-3.13.3-cp314-cp314t-win32.whl", hash = "sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767"}, + {file = "aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344"}, + {file = "aiohttp-3.13.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e"}, + {file = "aiohttp-3.13.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7"}, + {file = "aiohttp-3.13.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02"}, + {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43"}, + {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6"}, + {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce"}, + {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80"}, + {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a"}, + {file = "aiohttp-3.13.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6"}, + {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558"}, + {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7"}, + {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877"}, + {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3"}, + {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704"}, + {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f"}, + {file = "aiohttp-3.13.3-cp39-cp39-win32.whl", hash = "sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1"}, + {file = "aiohttp-3.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538"}, + {file = "aiohttp-3.13.3.tar.gz", hash = "sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88"}, ] [package.dependencies] @@ -207,7 +241,7 @@ propcache = ">=0.2.0" yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "brotlicffi ; platform_python_implementation != \"CPython\""] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aioitertools" @@ -389,14 +423,14 @@ tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" a [[package]] name = "authlib" -version = "1.6.5" +version = "1.6.6" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "authlib-1.6.5-py2.py3-none-any.whl", hash = "sha256:3e0e0507807f842b02175507bdee8957a1d5707fd4afb17c32fb43fee90b6e3a"}, - {file = "authlib-1.6.5.tar.gz", hash = "sha256:6aaf9c79b7cc96c900f0b284061691c5d4e61221640a948fe690b556a6d6d10b"}, + {file = "authlib-1.6.6-py2.py3-none-any.whl", hash = "sha256:7d9e9bc535c13974313a87f53e8430eb6ea3d1cf6ae4f6efcd793f2e949143fd"}, + {file = "authlib-1.6.6.tar.gz", hash = "sha256:45770e8e056d0f283451d9996fbb59b70d45722b45d854d58f32878d0a40c38e"}, ] [package.dependencies] @@ -1333,84 +1367,101 @@ files = [ [[package]] name = "cffi" -version = "1.17.1" +version = "2.0.0" description = "Foreign Function Interface for Python calling C code." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] +markers = "platform_python_implementation != \"PyPy\"" files = [ - {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, - {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, - {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, - {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, - {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, - {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, - {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, - {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, - {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, - {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, - {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, - {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, - {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, - {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, - {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, - {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, - {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, - {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, - {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, - {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, - {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, - {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, - {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, - {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, - {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, - {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, - {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, - {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, - {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, - {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, - {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, - {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, - {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, - {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, - {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, - {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, - {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb"}, + {file = "cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a"}, + {file = "cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743"}, + {file = "cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5"}, + {file = "cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5"}, + {file = "cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187"}, + {file = "cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18"}, + {file = "cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5"}, + {file = "cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b"}, + {file = "cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27"}, + {file = "cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75"}, + {file = "cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1"}, + {file = "cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f"}, + {file = "cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25"}, + {file = "cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4"}, + {file = "cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e"}, + {file = "cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6"}, + {file = "cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322"}, + {file = "cffi-2.0.0-cp39-cp39-win32.whl", hash = "sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a"}, + {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, + {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] -markers = {dev = "platform_python_implementation != \"PyPy\""} [package.dependencies] -pycparser = "*" +pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} [[package]] name = "charset-normalizer" @@ -2059,14 +2110,14 @@ with-social = ["django-allauth[socialaccount] (>=64.0.0)"] [[package]] name = "django" -version = "5.1.14" +version = "5.1.15" description = "A high-level Python web framework that encourages rapid development and clean, pragmatic design." optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "django-5.1.14-py3-none-any.whl", hash = "sha256:2a4b9c20404fd1bf50aaaa5542a19d860594cba1354f688f642feb271b91df27"}, - {file = "django-5.1.14.tar.gz", hash = "sha256:b98409fb31fdd6e8c3a6ba2eef3415cc5c0020057b43b21ba7af6eff5f014831"}, + {file = "django-5.1.15-py3-none-any.whl", hash = "sha256:117871e58d6eda37f09870b7d73a3d66567b03aecd515b386b1751177c413432"}, + {file = "django-5.1.15.tar.gz", hash = "sha256:46a356b5ff867bece73fc6365e081f21c569973403ee7e9b9a0316f27d0eb947"}, ] [package.dependencies] @@ -2633,83 +2684,75 @@ dotenv = ["python-dotenv"] [[package]] name = "fonttools" -version = "4.60.1" +version = "4.61.1" description = "Tools to manipulate font files" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "fonttools-4.60.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9a52f254ce051e196b8fe2af4634c2d2f02c981756c6464dc192f1b6050b4e28"}, - {file = "fonttools-4.60.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c7420a2696a44650120cdd269a5d2e56a477e2bfa9d95e86229059beb1c19e15"}, - {file = "fonttools-4.60.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee0c0b3b35b34f782afc673d503167157094a16f442ace7c6c5e0ca80b08f50c"}, - {file = "fonttools-4.60.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:282dafa55f9659e8999110bd8ed422ebe1c8aecd0dc396550b038e6c9a08b8ea"}, - {file = "fonttools-4.60.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4ba4bd646e86de16160f0fb72e31c3b9b7d0721c3e5b26b9fa2fc931dfdb2652"}, - {file = "fonttools-4.60.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0b0835ed15dd5b40d726bb61c846a688f5b4ce2208ec68779bc81860adb5851a"}, - {file = "fonttools-4.60.1-cp310-cp310-win32.whl", hash = "sha256:1525796c3ffe27bb6268ed2a1bb0dcf214d561dfaf04728abf01489eb5339dce"}, - {file = "fonttools-4.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:268ecda8ca6cb5c4f044b1fb9b3b376e8cd1b361cef275082429dc4174907038"}, - {file = "fonttools-4.60.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7b4c32e232a71f63a5d00259ca3d88345ce2a43295bb049d21061f338124246f"}, - {file = "fonttools-4.60.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3630e86c484263eaac71d117085d509cbcf7b18f677906824e4bace598fb70d2"}, - {file = "fonttools-4.60.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5c1015318e4fec75dd4943ad5f6a206d9727adf97410d58b7e32ab644a807914"}, - {file = "fonttools-4.60.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e6c58beb17380f7c2ea181ea11e7db8c0ceb474c9dd45f48e71e2cb577d146a1"}, - {file = "fonttools-4.60.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ec3681a0cb34c255d76dd9d865a55f260164adb9fa02628415cdc2d43ee2c05d"}, - {file = "fonttools-4.60.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f4b5c37a5f40e4d733d3bbaaef082149bee5a5ea3156a785ff64d949bd1353fa"}, - {file = "fonttools-4.60.1-cp311-cp311-win32.whl", hash = "sha256:398447f3d8c0c786cbf1209711e79080a40761eb44b27cdafffb48f52bcec258"}, - {file = "fonttools-4.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:d066ea419f719ed87bc2c99a4a4bfd77c2e5949cb724588b9dd58f3fd90b92bf"}, - {file = "fonttools-4.60.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7b0c6d57ab00dae9529f3faf187f2254ea0aa1e04215cf2f1a8ec277c96661bc"}, - {file = "fonttools-4.60.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:839565cbf14645952d933853e8ade66a463684ed6ed6c9345d0faf1f0e868877"}, - {file = "fonttools-4.60.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8177ec9676ea6e1793c8a084a90b65a9f778771998eb919d05db6d4b1c0b114c"}, - {file = "fonttools-4.60.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:996a4d1834524adbb423385d5a629b868ef9d774670856c63c9a0408a3063401"}, - {file = "fonttools-4.60.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a46b2f450bc79e06ef3b6394f0c68660529ed51692606ad7f953fc2e448bc903"}, - {file = "fonttools-4.60.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6ec722ee589e89a89f5b7574f5c45604030aa6ae24cb2c751e2707193b466fed"}, - {file = "fonttools-4.60.1-cp312-cp312-win32.whl", hash = "sha256:b2cf105cee600d2de04ca3cfa1f74f1127f8455b71dbad02b9da6ec266e116d6"}, - {file = "fonttools-4.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:992775c9fbe2cf794786fa0ffca7f09f564ba3499b8fe9f2f80bd7197db60383"}, - {file = "fonttools-4.60.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6f68576bb4bbf6060c7ab047b1574a1ebe5c50a17de62830079967b211059ebb"}, - {file = "fonttools-4.60.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:eedacb5c5d22b7097482fa834bda0dafa3d914a4e829ec83cdea2a01f8c813c4"}, - {file = "fonttools-4.60.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b33a7884fabd72bdf5f910d0cf46be50dce86a0362a65cfc746a4168c67eb96c"}, - {file = "fonttools-4.60.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2409d5fb7b55fd70f715e6d34e7a6e4f7511b8ad29a49d6df225ee76da76dd77"}, - {file = "fonttools-4.60.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c8651e0d4b3bdeda6602b85fdc2abbefc1b41e573ecb37b6779c4ca50753a199"}, - {file = "fonttools-4.60.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:145daa14bf24824b677b9357c5e44fd8895c2a8f53596e1b9ea3496081dc692c"}, - {file = "fonttools-4.60.1-cp313-cp313-win32.whl", hash = "sha256:2299df884c11162617a66b7c316957d74a18e3758c0274762d2cc87df7bc0272"}, - {file = "fonttools-4.60.1-cp313-cp313-win_amd64.whl", hash = "sha256:a3db56f153bd4c5c2b619ab02c5db5192e222150ce5a1bc10f16164714bc39ac"}, - {file = "fonttools-4.60.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a884aef09d45ba1206712c7dbda5829562d3fea7726935d3289d343232ecb0d3"}, - {file = "fonttools-4.60.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8a44788d9d91df72d1a5eac49b31aeb887a5f4aab761b4cffc4196c74907ea85"}, - {file = "fonttools-4.60.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e852d9dda9f93ad3651ae1e3bb770eac544ec93c3807888798eccddf84596537"}, - {file = "fonttools-4.60.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:154cb6ee417e417bf5f7c42fe25858c9140c26f647c7347c06f0cc2d47eff003"}, - {file = "fonttools-4.60.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5664fd1a9ea7f244487ac8f10340c4e37664675e8667d6fee420766e0fb3cf08"}, - {file = "fonttools-4.60.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:583b7f8e3c49486e4d489ad1deacfb8d5be54a8ef34d6df824f6a171f8511d99"}, - {file = "fonttools-4.60.1-cp314-cp314-win32.whl", hash = "sha256:66929e2ea2810c6533a5184f938502cfdaea4bc3efb7130d8cc02e1c1b4108d6"}, - {file = "fonttools-4.60.1-cp314-cp314-win_amd64.whl", hash = "sha256:f3d5be054c461d6a2268831f04091dc82753176f6ea06dc6047a5e168265a987"}, - {file = "fonttools-4.60.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:b6379e7546ba4ae4b18f8ae2b9bc5960936007a1c0e30b342f662577e8bc3299"}, - {file = "fonttools-4.60.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9d0ced62b59e0430b3690dbc5373df1c2aa7585e9a8ce38eff87f0fd993c5b01"}, - {file = "fonttools-4.60.1-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:875cb7764708b3132637f6c5fb385b16eeba0f7ac9fa45a69d35e09b47045801"}, - {file = "fonttools-4.60.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a184b2ea57b13680ab6d5fbde99ccef152c95c06746cb7718c583abd8f945ccc"}, - {file = "fonttools-4.60.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:026290e4ec76583881763fac284aca67365e0be9f13a7fb137257096114cb3bc"}, - {file = "fonttools-4.60.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f0e8817c7d1a0c2eedebf57ef9a9896f3ea23324769a9a2061a80fe8852705ed"}, - {file = "fonttools-4.60.1-cp314-cp314t-win32.whl", hash = "sha256:1410155d0e764a4615774e5c2c6fc516259fe3eca5882f034eb9bfdbee056259"}, - {file = "fonttools-4.60.1-cp314-cp314t-win_amd64.whl", hash = "sha256:022beaea4b73a70295b688f817ddc24ed3e3418b5036ffcd5658141184ef0d0c"}, - {file = "fonttools-4.60.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:122e1a8ada290423c493491d002f622b1992b1ab0b488c68e31c413390dc7eb2"}, - {file = "fonttools-4.60.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a140761c4ff63d0cb9256ac752f230460ee225ccef4ad8f68affc723c88e2036"}, - {file = "fonttools-4.60.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0eae96373e4b7c9e45d099d7a523444e3554360927225c1cdae221a58a45b856"}, - {file = "fonttools-4.60.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:596ecaca36367027d525b3b426d8a8208169d09edcf8c7506aceb3a38bfb55c7"}, - {file = "fonttools-4.60.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2ee06fc57512144d8b0445194c2da9f190f61ad51e230f14836286470c99f854"}, - {file = "fonttools-4.60.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b42d86938e8dda1cd9a1a87a6d82f1818eaf933348429653559a458d027446da"}, - {file = "fonttools-4.60.1-cp39-cp39-win32.whl", hash = "sha256:8b4eb332f9501cb1cd3d4d099374a1e1306783ff95489a1026bde9eb02ccc34a"}, - {file = "fonttools-4.60.1-cp39-cp39-win_amd64.whl", hash = "sha256:7473a8ed9ed09aeaa191301244a5a9dbe46fe0bf54f9d6cd21d83044c3321217"}, - {file = "fonttools-4.60.1-py3-none-any.whl", hash = "sha256:906306ac7afe2156fcf0042173d6ebbb05416af70f6b370967b47f8f00103bbb"}, - {file = "fonttools-4.60.1.tar.gz", hash = "sha256:ef00af0439ebfee806b25f24c8f92109157ff3fac5731dc7867957812e87b8d9"}, + {file = "fonttools-4.61.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c7db70d57e5e1089a274cbb2b1fd635c9a24de809a231b154965d415d6c6d24"}, + {file = "fonttools-4.61.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5fe9fd43882620017add5eabb781ebfbc6998ee49b35bd7f8f79af1f9f99a958"}, + {file = "fonttools-4.61.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8db08051fc9e7d8bc622f2112511b8107d8f27cd89e2f64ec45e9825e8288da"}, + {file = "fonttools-4.61.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a76d4cb80f41ba94a6691264be76435e5f72f2cb3cab0b092a6212855f71c2f6"}, + {file = "fonttools-4.61.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a13fc8aeb24bad755eea8f7f9d409438eb94e82cf86b08fe77a03fbc8f6a96b1"}, + {file = "fonttools-4.61.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b846a1fcf8beadeb9ea4f44ec5bdde393e2f1569e17d700bfc49cd69bde75881"}, + {file = "fonttools-4.61.1-cp310-cp310-win32.whl", hash = "sha256:78a7d3ab09dc47ac1a363a493e6112d8cabed7ba7caad5f54dbe2f08676d1b47"}, + {file = "fonttools-4.61.1-cp310-cp310-win_amd64.whl", hash = "sha256:eff1ac3cc66c2ac7cda1e64b4e2f3ffef474b7335f92fc3833fc632d595fcee6"}, + {file = "fonttools-4.61.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c6604b735bb12fef8e0efd5578c9fb5d3d8532d5001ea13a19cddf295673ee09"}, + {file = "fonttools-4.61.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5ce02f38a754f207f2f06557523cd39a06438ba3aafc0639c477ac409fc64e37"}, + {file = "fonttools-4.61.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77efb033d8d7ff233385f30c62c7c79271c8885d5c9657d967ede124671bbdfb"}, + {file = "fonttools-4.61.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:75c1a6dfac6abd407634420c93864a1e274ebc1c7531346d9254c0d8f6ca00f9"}, + {file = "fonttools-4.61.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0de30bfe7745c0d1ffa2b0b7048fb7123ad0d71107e10ee090fa0b16b9452e87"}, + {file = "fonttools-4.61.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:58b0ee0ab5b1fc9921eccfe11d1435added19d6494dde14e323f25ad2bc30c56"}, + {file = "fonttools-4.61.1-cp311-cp311-win32.whl", hash = "sha256:f79b168428351d11e10c5aeb61a74e1851ec221081299f4cf56036a95431c43a"}, + {file = "fonttools-4.61.1-cp311-cp311-win_amd64.whl", hash = "sha256:fe2efccb324948a11dd09d22136fe2ac8a97d6c1347cf0b58a911dcd529f66b7"}, + {file = "fonttools-4.61.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f3cb4a569029b9f291f88aafc927dd53683757e640081ca8c412781ea144565e"}, + {file = "fonttools-4.61.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41a7170d042e8c0024703ed13b71893519a1a6d6e18e933e3ec7507a2c26a4b2"}, + {file = "fonttools-4.61.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:10d88e55330e092940584774ee5e8a6971b01fc2f4d3466a1d6c158230880796"}, + {file = "fonttools-4.61.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15acc09befd16a0fb8a8f62bc147e1a82817542d72184acca9ce6e0aeda9fa6d"}, + {file = "fonttools-4.61.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e6bcdf33aec38d16508ce61fd81838f24c83c90a1d1b8c68982857038673d6b8"}, + {file = "fonttools-4.61.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5fade934607a523614726119164ff621e8c30e8fa1ffffbbd358662056ba69f0"}, + {file = "fonttools-4.61.1-cp312-cp312-win32.whl", hash = "sha256:75da8f28eff26defba42c52986de97b22106cb8f26515b7c22443ebc9c2d3261"}, + {file = "fonttools-4.61.1-cp312-cp312-win_amd64.whl", hash = "sha256:497c31ce314219888c0e2fce5ad9178ca83fe5230b01a5006726cdf3ac9f24d9"}, + {file = "fonttools-4.61.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c56c488ab471628ff3bfa80964372fc13504ece601e0d97a78ee74126b2045c"}, + {file = "fonttools-4.61.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dc492779501fa723b04d0ab1f5be046797fee17d27700476edc7ee9ae535a61e"}, + {file = "fonttools-4.61.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:64102ca87e84261419c3747a0d20f396eb024bdbeb04c2bfb37e2891f5fadcb5"}, + {file = "fonttools-4.61.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c1b526c8d3f615a7b1867f38a9410849c8f4aef078535742198e942fba0e9bd"}, + {file = "fonttools-4.61.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41ed4b5ec103bd306bb68f81dc166e77409e5209443e5773cb4ed837bcc9b0d3"}, + {file = "fonttools-4.61.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b501c862d4901792adaec7c25b1ecc749e2662543f68bb194c42ba18d6eec98d"}, + {file = "fonttools-4.61.1-cp313-cp313-win32.whl", hash = "sha256:4d7092bb38c53bbc78e9255a59158b150bcdc115a1e3b3ce0b5f267dc35dd63c"}, + {file = "fonttools-4.61.1-cp313-cp313-win_amd64.whl", hash = "sha256:21e7c8d76f62ab13c9472ccf74515ca5b9a761d1bde3265152a6dc58700d895b"}, + {file = "fonttools-4.61.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fff4f534200a04b4a36e7ae3cb74493afe807b517a09e99cb4faa89a34ed6ecd"}, + {file = "fonttools-4.61.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d9203500f7c63545b4ce3799319fe4d9feb1a1b89b28d3cb5abd11b9dd64147e"}, + {file = "fonttools-4.61.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fa646ecec9528bef693415c79a86e733c70a4965dd938e9a226b0fc64c9d2e6c"}, + {file = "fonttools-4.61.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11f35ad7805edba3aac1a3710d104592df59f4b957e30108ae0ba6c10b11dd75"}, + {file = "fonttools-4.61.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b931ae8f62db78861b0ff1ac017851764602288575d65b8e8ff1963fed419063"}, + {file = "fonttools-4.61.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b148b56f5de675ee16d45e769e69f87623a4944f7443850bf9a9376e628a89d2"}, + {file = "fonttools-4.61.1-cp314-cp314-win32.whl", hash = "sha256:9b666a475a65f4e839d3d10473fad6d47e0a9db14a2f4a224029c5bfde58ad2c"}, + {file = "fonttools-4.61.1-cp314-cp314-win_amd64.whl", hash = "sha256:4f5686e1fe5fce75d82d93c47a438a25bf0d1319d2843a926f741140b2b16e0c"}, + {file = "fonttools-4.61.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e76ce097e3c57c4bcb67c5aa24a0ecdbd9f74ea9219997a707a4061fbe2707aa"}, + {file = "fonttools-4.61.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9cfef3ab326780c04d6646f68d4b4742aae222e8b8ea1d627c74e38afcbc9d91"}, + {file = "fonttools-4.61.1-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a75c301f96db737e1c5ed5fd7d77d9c34466de16095a266509e13da09751bd19"}, + {file = "fonttools-4.61.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:91669ccac46bbc1d09e9273546181919064e8df73488ea087dcac3e2968df9ba"}, + {file = "fonttools-4.61.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c33ab3ca9d3ccd581d58e989d67554e42d8d4ded94ab3ade3508455fe70e65f7"}, + {file = "fonttools-4.61.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:664c5a68ec406f6b1547946683008576ef8b38275608e1cee6c061828171c118"}, + {file = "fonttools-4.61.1-cp314-cp314t-win32.whl", hash = "sha256:aed04cabe26f30c1647ef0e8fbb207516fd40fe9472e9439695f5c6998e60ac5"}, + {file = "fonttools-4.61.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2180f14c141d2f0f3da43f3a81bc8aa4684860f6b0e6f9e165a4831f24e6a23b"}, + {file = "fonttools-4.61.1-py3-none-any.whl", hash = "sha256:17d2bf5d541add43822bcf0c43d7d847b160c9bb01d15d5007d84e2217aaa371"}, + {file = "fonttools-4.61.1.tar.gz", hash = "sha256:6675329885c44657f826ef01d9e4fb33b9158e9d93c537d84ad8399539bc6f69"}, ] [package.extras] -all = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\"", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0) ; python_version <= \"3.12\"", "xattr ; sys_platform == \"darwin\"", "zopfli (>=0.1.4)"] +all = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\"", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.45.0)", "unicodedata2 (>=17.0.0) ; python_version <= \"3.14\"", "xattr ; sys_platform == \"darwin\"", "zopfli (>=0.1.4)"] graphite = ["lz4 (>=1.7.4.2)"] interpolatable = ["munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\""] lxml = ["lxml (>=4.0)"] pathops = ["skia-pathops (>=0.5.0)"] plot = ["matplotlib"] -repacker = ["uharfbuzz (>=0.23.0)"] +repacker = ["uharfbuzz (>=0.45.0)"] symfont = ["sympy"] type1 = ["xattr ; sys_platform == \"darwin\""] -unicode = ["unicodedata2 (>=15.1.0) ; python_version <= \"3.12\""] +unicode = ["unicodedata2 (>=17.0.0) ; python_version <= \"3.14\""] woff = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "zopfli (>=0.1.4)"] [[package]] @@ -2841,6 +2884,72 @@ files = [ {file = "frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f"}, ] +[[package]] +name = "gevent" +version = "25.9.1" +description = "Coroutine-based network library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "gevent-25.9.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:856b990be5590e44c3a3dc6c8d48a40eaccbb42e99d2b791d11d1e7711a4297e"}, + {file = "gevent-25.9.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fe1599d0b30e6093eb3213551751b24feeb43db79f07e89d98dd2f3330c9063e"}, + {file = "gevent-25.9.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:f0d8b64057b4bf1529b9ef9bd2259495747fba93d1f836c77bfeaacfec373fd0"}, + {file = "gevent-25.9.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b56cbc820e3136ba52cd690bdf77e47a4c239964d5f80dc657c1068e0fe9521c"}, + {file = "gevent-25.9.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c5fa9ce5122c085983e33e0dc058f81f5264cebe746de5c401654ab96dddfca8"}, + {file = "gevent-25.9.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:03c74fec58eda4b4edc043311fca8ba4f8744ad1632eb0a41d5ec25413581975"}, + {file = "gevent-25.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:a8ae9f895e8651d10b0a8328a61c9c53da11ea51b666388aa99b0ce90f9fdc27"}, + {file = "gevent-25.9.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:18e5aff9e8342dc954adb9c9c524db56c2f3557999463445ba3d9cbe3dada7b7"}, + {file = "gevent-25.9.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1cdf6db28f050ee103441caa8b0448ace545364f775059d5e2de089da975c457"}, + {file = "gevent-25.9.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:812debe235a8295be3b2a63b136c2474241fa5c58af55e6a0f8cfc29d4936235"}, + {file = "gevent-25.9.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b28b61ff9216a3d73fe8f35669eefcafa957f143ac534faf77e8a19eb9e6883a"}, + {file = "gevent-25.9.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5e4b6278b37373306fc6b1e5f0f1cf56339a1377f67c35972775143d8d7776ff"}, + {file = "gevent-25.9.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d99f0cb2ce43c2e8305bf75bee61a8bde06619d21b9d0316ea190fc7a0620a56"}, + {file = "gevent-25.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:72152517ecf548e2f838c61b4be76637d99279dbaa7e01b3924df040aa996586"}, + {file = "gevent-25.9.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:46b188248c84ffdec18a686fcac5dbb32365d76912e14fda350db5dc0bfd4f86"}, + {file = "gevent-25.9.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f2b54ea3ca6f0c763281cd3f96010ac7e98c2e267feb1221b5a26e2ca0b9a692"}, + {file = "gevent-25.9.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7a834804ac00ed8a92a69d3826342c677be651b1c3cd66cc35df8bc711057aa2"}, + {file = "gevent-25.9.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:323a27192ec4da6b22a9e51c3d9d896ff20bc53fdc9e45e56eaab76d1c39dd74"}, + {file = "gevent-25.9.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6ea78b39a2c51d47ff0f130f4c755a9a4bbb2dd9721149420ad4712743911a51"}, + {file = "gevent-25.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:dc45cd3e1cc07514a419960af932a62eb8515552ed004e56755e4bf20bad30c5"}, + {file = "gevent-25.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34e01e50c71eaf67e92c186ee0196a039d6e4f4b35670396baed4a2d8f1b347f"}, + {file = "gevent-25.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:4acd6bcd5feabf22c7c5174bd3b9535ee9f088d2bbce789f740ad8d6554b18f3"}, + {file = "gevent-25.9.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:4f84591d13845ee31c13f44bdf6bd6c3dbf385b5af98b2f25ec328213775f2ed"}, + {file = "gevent-25.9.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9cdbb24c276a2d0110ad5c978e49daf620b153719ac8a548ce1250a7eb1b9245"}, + {file = "gevent-25.9.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:88b6c07169468af631dcf0fdd3658f9246d6822cc51461d43f7c44f28b0abb82"}, + {file = "gevent-25.9.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b7bb0e29a7b3e6ca9bed2394aa820244069982c36dc30b70eb1004dd67851a48"}, + {file = "gevent-25.9.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2951bb070c0ee37b632ac9134e4fdaad70d2e660c931bb792983a0837fe5b7d7"}, + {file = "gevent-25.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4e17c2d57e9a42e25f2a73d297b22b60b2470a74be5a515b36c984e1a246d47"}, + {file = "gevent-25.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8d94936f8f8b23d9de2251798fcb603b84f083fdf0d7f427183c1828fb64f117"}, + {file = "gevent-25.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:eb51c5f9537b07da673258b4832f6635014fee31690c3f0944d34741b69f92fa"}, + {file = "gevent-25.9.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:1a3fe4ea1c312dbf6b375b416925036fe79a40054e6bf6248ee46526ea628be1"}, + {file = "gevent-25.9.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0adb937f13e5fb90cca2edf66d8d7e99d62a299687400ce2edee3f3504009356"}, + {file = "gevent-25.9.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:427f869a2050a4202d93cf7fd6ab5cffb06d3e9113c10c967b6e2a0d45237cb8"}, + {file = "gevent-25.9.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c049880175e8c93124188f9d926af0a62826a3b81aa6d3074928345f8238279e"}, + {file = "gevent-25.9.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b5a67a0974ad9f24721034d1e008856111e0535f1541499f72a733a73d658d1c"}, + {file = "gevent-25.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1d0f5d8d73f97e24ea8d24d8be0f51e0cf7c54b8021c1fddb580bf239474690f"}, + {file = "gevent-25.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddd3ff26e5c4240d3fbf5516c2d9d5f2a998ef87cfb73e1429cfaeaaec860fa6"}, + {file = "gevent-25.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:bb63c0d6cb9950cc94036a4995b9cc4667b8915366613449236970f4394f94d7"}, + {file = "gevent-25.9.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f18f80aef6b1f6907219affe15b36677904f7cfeed1f6a6bc198616e507ae2d7"}, + {file = "gevent-25.9.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b274a53e818124a281540ebb4e7a2c524778f745b7a99b01bdecf0ca3ac0ddb0"}, + {file = "gevent-25.9.1-cp39-cp39-win32.whl", hash = "sha256:c6c91f7e33c7f01237755884316110ee7ea076f5bdb9aa0982b6dc63243c0a38"}, + {file = "gevent-25.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:012a44b0121f3d7c800740ff80351c897e85e76a7e4764690f35c5ad9ec17de5"}, + {file = "gevent-25.9.1.tar.gz", hash = "sha256:adf9cd552de44a4e6754c51ff2e78d9193b7fa6eab123db9578a210e657235dd"}, +] + +[package.dependencies] +cffi = {version = ">=1.17.1", markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\""} +greenlet = {version = ">=3.2.2", markers = "platform_python_implementation == \"CPython\""} +"zope.event" = "*" +"zope.interface" = "*" + +[package.extras] +dnspython = ["dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\""] +docs = ["furo", "repoze.sphinx.autointerface", "sphinx", "sphinxcontrib-programoutput", "zope.schema"] +monitor = ["psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""] +recommended = ["cffi (>=1.17.1) ; platform_python_implementation == \"CPython\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""] +test = ["cffi (>=1.17.1) ; platform_python_implementation == \"CPython\"", "coverage (>=5.0) ; sys_platform != \"win32\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "objgraph", "psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\"", "requests"] + [[package]] name = "google-api-core" version = "2.25.1" @@ -3074,6 +3183,87 @@ files = [ dev = ["pytest"] docs = ["sphinx", "sphinx-autobuild"] +[[package]] +name = "greenlet" +version = "3.2.4" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\"" +files = [ + {file = "greenlet-3.2.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:8c68325b0d0acf8d91dde4e6f930967dd52a5302cd4062932a6b2e7c2969f47c"}, + {file = "greenlet-3.2.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:94385f101946790ae13da500603491f04a76b6e4c059dab271b3ce2e283b2590"}, + {file = "greenlet-3.2.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f10fd42b5ee276335863712fa3da6608e93f70629c631bf77145021600abc23c"}, + {file = "greenlet-3.2.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c8c9e331e58180d0d83c5b7999255721b725913ff6bc6cf39fa2a45841a4fd4b"}, + {file = "greenlet-3.2.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58b97143c9cc7b86fc458f215bd0932f1757ce649e05b640fea2e79b54cedb31"}, + {file = "greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8854167e06950ca75b898b104b63cc646573aa5fef1353d4508ecdd1ee76254f"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f47617f698838ba98f4ff4189aef02e7343952df3a615f847bb575c3feb177a7"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af41be48a4f60429d5cad9d22175217805098a9ef7c40bfef44f7669fb9d74d8"}, + {file = "greenlet-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:73f49b5368b5359d04e18d15828eecc1806033db5233397748f4ca813ff1056c"}, + {file = "greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2"}, + {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246"}, + {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3"}, + {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633"}, + {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079"}, + {file = "greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5"}, + {file = "greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9"}, + {file = "greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd"}, + {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb"}, + {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968"}, + {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9"}, + {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6"}, + {file = "greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d"}, + {file = "greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02"}, + {file = "greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31"}, + {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945"}, + {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc"}, + {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a"}, + {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504"}, + {file = "greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929"}, + {file = "greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b"}, + {file = "greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0"}, + {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f"}, + {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c17b6b34111ea72fc5a4e4beec9711d2226285f0386ea83477cbb97c30a3f3a5"}, + {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1"}, + {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735"}, + {file = "greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337"}, + {file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269"}, + {file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:015d48959d4add5d6c9f6c5210ee3803a830dce46356e3bc326d6776bde54681"}, + {file = "greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01"}, + {file = "greenlet-3.2.4-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:b6a7c19cf0d2742d0809a4c05975db036fdff50cd294a93632d6a310bf9ac02c"}, + {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:27890167f55d2387576d1f41d9487ef171849ea0359ce1510ca6e06c8bece11d"}, + {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:18d9260df2b5fbf41ae5139e1be4e796d99655f023a636cd0e11e6406cca7d58"}, + {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:671df96c1f23c4a0d4077a325483c1503c96a1b7d9db26592ae770daa41233d4"}, + {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:16458c245a38991aa19676900d48bd1a6f2ce3e16595051a4db9d012154e8433"}, + {file = "greenlet-3.2.4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9913f1a30e4526f432991f89ae263459b1c64d1608c0d22a5c79c287b3c70df"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b90654e092f928f110e0007f572007c9727b5265f7632c2fa7415b4689351594"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81701fd84f26330f0d5f4944d4e92e61afe6319dcd9775e39396e39d7c3e5f98"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:28a3c6b7cd72a96f61b0e4b2a36f681025b60ae4779cc73c1535eb5f29560b10"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:52206cd642670b0b320a1fd1cbfd95bca0e043179c1d8a045f2c6109dfe973be"}, + {file = "greenlet-3.2.4-cp39-cp39-win32.whl", hash = "sha256:65458b409c1ed459ea899e939f0e1cdb14f58dbc803f2f93c5eab5694d32671b"}, + {file = "greenlet-3.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:d2e685ade4dafd447ede19c31277a224a239a0a1a4eca4e6390efedf20260cfb"}, + {file = "greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil", "setuptools"] + [[package]] name = "grpc-google-iam-v1" version = "0.14.3" @@ -4087,14 +4277,14 @@ files = [ [[package]] name = "marshmallow" -version = "3.26.1" +version = "3.26.2" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c"}, - {file = "marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6"}, + {file = "marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73"}, + {file = "marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57"}, ] [package.dependencies] @@ -5479,7 +5669,7 @@ tzlocal = "5.3.1" type = "git" url = "https://github.com/prowler-cloud/prowler.git" reference = "attack-paths-demo" -resolved_reference = "95d9e9a59f8e52e4096a5c17bf839e515b918239" +resolved_reference = "b105c3a6e1df07903552c63e676e497a5ec242e1" [[package]] name = "psutil" @@ -5683,11 +5873,11 @@ description = "C parser in Python" optional = false python-versions = ">=3.8" groups = ["main", "dev"] +markers = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"" files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] -markers = {dev = "platform_python_implementation != \"PyPy\""} [[package]] name = "pycurl" @@ -5992,30 +6182,45 @@ files = [ [[package]] name = "pynacl" -version = "1.5.0" +version = "1.6.2" description = "Python binding to the Networking and Cryptography (NaCl) library" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" groups = ["main"] files = [ - {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, - {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, - {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"}, - {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"}, - {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"}, - {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"}, - {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"}, - {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"}, - {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"}, - {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"}, + {file = "pynacl-1.6.2-cp314-cp314t-macosx_10_10_universal2.whl", hash = "sha256:622d7b07cc5c02c666795792931b50c91f3ce3c2649762efb1ef0d5684c81594"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d071c6a9a4c94d79eb665db4ce5cedc537faf74f2355e4d502591d850d3913c0"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe9847ca47d287af41e82be1dd5e23023d3c31a951da134121ab02e42ac218c9"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04316d1fc625d860b6c162fff704eb8426b1a8bcd3abacea11142cbd99a6b574"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44081faff368d6c5553ccf55322ef2819abb40e25afaec7e740f159f74813634"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:a9f9932d8d2811ce1a8ffa79dcbdf3970e7355b5c8eb0c1a881a57e7f7d96e88"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:bc4a36b28dd72fb4845e5d8f9760610588a96d5a51f01d84d8c6ff9849968c14"}, + {file = "pynacl-1.6.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bffb6d0f6becacb6526f8f42adfb5efb26337056ee0831fb9a7044d1a964444"}, + {file = "pynacl-1.6.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fef529ef3ee487ad8113d287a593fa26f48ee3620d92ecc6f1d09ea38e0709b"}, + {file = "pynacl-1.6.2-cp314-cp314t-win32.whl", hash = "sha256:a84bf1c20339d06dc0c85d9aea9637a24f718f375d861b2668b2f9f96fa51145"}, + {file = "pynacl-1.6.2-cp314-cp314t-win_amd64.whl", hash = "sha256:320ef68a41c87547c91a8b58903c9caa641ab01e8512ce291085b5fe2fcb7590"}, + {file = "pynacl-1.6.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d29bfe37e20e015a7d8b23cfc8bd6aa7909c92a1b8f41ee416bbb3e79ef182b2"}, + {file = "pynacl-1.6.2-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6"}, + {file = "pynacl-1.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e"}, + {file = "pynacl-1.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577"}, + {file = "pynacl-1.6.2-cp38-abi3-win32.whl", hash = "sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa"}, + {file = "pynacl-1.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0"}, + {file = "pynacl-1.6.2-cp38-abi3-win_arm64.whl", hash = "sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c"}, + {file = "pynacl-1.6.2.tar.gz", hash = "sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c"}, ] [package.dependencies] -cffi = ">=1.4.1" +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\" and python_version >= \"3.9\""} [package.extras] -docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] -tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] +docs = ["sphinx (<7)", "sphinx_rtd_theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=7.4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] [[package]] name = "pyopenssl" @@ -7131,18 +7336,18 @@ files = [ [[package]] name = "sqlparse" -version = "0.5.3" +version = "0.5.5" description = "A non-validating SQL parser." optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "sqlparse-0.5.3-py3-none-any.whl", hash = "sha256:cf2196ed3418f3ba5de6af7e82c694a9fbdbfecccdfc72e281548517081f16ca"}, - {file = "sqlparse-0.5.3.tar.gz", hash = "sha256:09f67787f56a0b16ecdbde1bfc7f5d9c3371ca683cfeaa8e6ff60b4807ec9272"}, + {file = "sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba"}, + {file = "sqlparse-0.5.5.tar.gz", hash = "sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e"}, ] [package.extras] -dev = ["build", "hatch"] +dev = ["build"] doc = ["sphinx"] [[package]] @@ -7376,21 +7581,21 @@ files = [ [[package]] name = "urllib3" -version = "2.5.0" +version = "2.6.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, + {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "uuid6" @@ -7459,18 +7664,18 @@ test = ["websockets"] [[package]] name = "werkzeug" -version = "3.1.3" +version = "3.1.5" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, - {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, + {file = "werkzeug-3.1.5-py3-none-any.whl", hash = "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc"}, + {file = "werkzeug-3.1.5.tar.gz", hash = "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67"}, ] [package.dependencies] -MarkupSafe = ">=2.1.1" +markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] @@ -7803,7 +8008,69 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] +[[package]] +name = "zope-event" +version = "6.1" +description = "Very basic event publishing system" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "zope_event-6.1-py3-none-any.whl", hash = "sha256:0ca78b6391b694272b23ec1335c0294cc471065ed10f7f606858fc54566c25a0"}, + {file = "zope_event-6.1.tar.gz", hash = "sha256:6052a3e0cb8565d3d4ef1a3a7809336ac519bc4fe38398cb8d466db09adef4f0"}, +] + +[package.extras] +docs = ["Sphinx"] +test = ["zope.testrunner (>=6.4)"] + +[[package]] +name = "zope-interface" +version = "8.1.1" +description = "Interfaces for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "zope_interface-8.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5c6b12b656c7d7e3d79cad8e2afc4a37eae6b6076e2c209a33345143148e435e"}, + {file = "zope_interface-8.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:557c0f1363c300db406e9eeaae8ab6d1ba429d4fed60d8ab7dadab5ca66ccd35"}, + {file = "zope_interface-8.1.1-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:127b0e4c873752b777721543cf8525b3db5e76b88bd33bab807f03c568e9003f"}, + {file = "zope_interface-8.1.1-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e0892c9d2dd47b45f62d1861bcae8b427fcc49b4a04fff67f12c5c55e56654d7"}, + {file = "zope_interface-8.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff8a92dc8c8a2c605074e464984e25b9b5a8ac9b2a0238dd73a0f374df59a77e"}, + {file = "zope_interface-8.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:54627ddf6034aab1f506ba750dd093f67d353be6249467d720e9f278a578efe5"}, + {file = "zope_interface-8.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e8a0fdd5048c1bb733e4693eae9bc4145a19419ea6a1c95299318a93fe9f3d72"}, + {file = "zope_interface-8.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a4cb0ea75a26b606f5bc8524fbce7b7d8628161b6da002c80e6417ce5ec757c0"}, + {file = "zope_interface-8.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c267b00b5a49a12743f5e1d3b4beef45479d696dab090f11fe3faded078a5133"}, + {file = "zope_interface-8.1.1-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e25d3e2b9299e7ec54b626573673bdf0d740cf628c22aef0a3afef85b438aa54"}, + {file = "zope_interface-8.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:63db1241804417aff95ac229c13376c8c12752b83cc06964d62581b493e6551b"}, + {file = "zope_interface-8.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:9639bf4ed07b5277fb231e54109117c30d608254685e48a7104a34618bcbfc83"}, + {file = "zope_interface-8.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a16715808408db7252b8c1597ed9008bdad7bf378ed48eb9b0595fad4170e49d"}, + {file = "zope_interface-8.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce6b58752acc3352c4aa0b55bbeae2a941d61537e6afdad2467a624219025aae"}, + {file = "zope_interface-8.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:807778883d07177713136479de7fd566f9056a13aef63b686f0ab4807c6be259"}, + {file = "zope_interface-8.1.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50e5eb3b504a7d63dc25211b9298071d5b10a3eb754d6bf2f8ef06cb49f807ab"}, + {file = "zope_interface-8.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eee6f93b2512ec9466cf30c37548fd3ed7bc4436ab29cd5943d7a0b561f14f0f"}, + {file = "zope_interface-8.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:80edee6116d569883c58ff8efcecac3b737733d646802036dc337aa839a5f06b"}, + {file = "zope_interface-8.1.1-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:84f9be6d959640de9da5d14ac1f6a89148b16da766e88db37ed17e936160b0b1"}, + {file = "zope_interface-8.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:531fba91dcb97538f70cf4642a19d6574269460274e3f6004bba6fe684449c51"}, + {file = "zope_interface-8.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:fc65f5633d5a9583ee8d88d1f5de6b46cd42c62e47757cfe86be36fb7c8c4c9b"}, + {file = "zope_interface-8.1.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:efef80ddec4d7d99618ef71bc93b88859248075ca2e1ae1c78636654d3d55533"}, + {file = "zope_interface-8.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49aad83525eca3b4747ef51117d302e891f0042b06f32aa1c7023c62642f962b"}, + {file = "zope_interface-8.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:71cf329a21f98cb2bd9077340a589e316ac8a415cac900575a32544b3dffcb98"}, + {file = "zope_interface-8.1.1-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:da311e9d253991ca327601f47c4644d72359bac6950fbb22f971b24cd7850f8c"}, + {file = "zope_interface-8.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3fb25fca0442c7fb93c4ee40b42e3e033fef2f648730c4b7ae6d43222a3e8946"}, + {file = "zope_interface-8.1.1-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:bac588d0742b4e35efb7c7df1dacc0397b51ed37a17d4169a38019a1cebacf0a"}, + {file = "zope_interface-8.1.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d1f053d2d5e2b393e619bce1e55954885c2e63969159aa521839e719442db49"}, + {file = "zope_interface-8.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:64a1ad7f4cb17d948c6bdc525a1d60c0e567b2526feb4fa38b38f249961306b8"}, + {file = "zope_interface-8.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:169214da1b82b7695d1a36f92d70b11166d66b6b09d03df35d150cc62ac52276"}, + {file = "zope_interface-8.1.1.tar.gz", hash = "sha256:51b10e6e8e238d719636a401f44f1e366146912407b58453936b781a19be19ec"}, +] + +[package.extras] +docs = ["Sphinx", "furo", "repoze.sphinx.autointerface"] +test = ["coverage[toml]", "zope.event", "zope.testing"] +testing = ["coverage[toml]", "zope.event", "zope.testing"] + [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "de57b503d0f96c22ac3f9b1e9845aefac0a5b32089c9d90928a357f991384db3" +content-hash = "61fb4f6e5a99a2cb6e94f94fd96ed5ef086ccd3e57b99c1a5f2d47688508c78e" diff --git a/api/pyproject.toml b/api/pyproject.toml index f96fd1f54b..711b20092a 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -7,7 +7,7 @@ authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}] dependencies = [ "celery[pytest] (>=5.4.0,<6.0.0)", "dj-rest-auth[with_social,jwt] (==7.0.1)", - "django (==5.1.14)", + "django (==5.1.15)", "django-allauth[saml] (>=65.8.0,<66.0.0)", "django-celery-beat (>=2.7.0,<3.0.0)", "django-celery-results (>=2.5.1,<3.0.0)", @@ -24,7 +24,7 @@ dependencies = [ "drf-spectacular-jsonapi==0.5.1", "gunicorn==23.0.0", "lxml==5.3.2", - "prowler @ git+https://github.com/prowler-cloud/prowler.git@attack-paths-demo", + "prowler @ git+https://github.com/prowler-cloud/prowler.git@master", "psycopg2-binary==2.9.9", "pytest-celery[redis] (>=1.0.1,<2.0.0)", "sentry-sdk[django] (>=2.20.0,<3.0.0)", @@ -38,6 +38,10 @@ dependencies = [ "reportlab (>=4.4.4,<5.0.0)", "neo4j (<6.0.0)", "cartography @ git+https://github.com/prowler-cloud/cartography@master", + "gevent (>=25.9.1,<26.0.0)", + "werkzeug (>=3.1.4)", + "sqlparse (>=0.5.4)", + "fonttools (>=4.60.2)" ] description = "Prowler's API (Django/DRF)" license = "Apache-2.0" @@ -45,7 +49,7 @@ name = "prowler-api" package-mode = false # Needed for the SDK compatibility requires-python = ">=3.11,<3.13" -version = "1.15.0" +version = "1.18.0" [project.scripts] celery = "src.backend.config.settings.celery" diff --git a/api/src/backend/api/db_router.py b/api/src/backend/api/db_router.py index 2a9085746e..4ae36cc1a5 100644 --- a/api/src/backend/api/db_router.py +++ b/api/src/backend/api/db_router.py @@ -26,6 +26,7 @@ class MainRouter: default_db = "default" admin_db = "admin" replica_db = "replica" + admin_replica_db = "admin_replica" def db_for_read(self, model, **hints): # noqa: F841 model_table_name = model._meta.db_table @@ -49,7 +50,12 @@ class MainRouter: def allow_relation(self, obj1, obj2, **hints): # noqa: F841 # Allow relations when both objects originate from allowed connectors - allowed_dbs = {self.default_db, self.admin_db, self.replica_db} + allowed_dbs = { + self.default_db, + self.admin_db, + self.replica_db, + self.admin_replica_db, + } if {obj1._state.db, obj2._state.db} <= allowed_dbs: return True return None diff --git a/api/src/backend/api/decorators.py b/api/src/backend/api/decorators.py index b74d12df5d..d2330a6a06 100644 --- a/api/src/backend/api/decorators.py +++ b/api/src/backend/api/decorators.py @@ -1,10 +1,14 @@ import uuid from functools import wraps -from django.db import connection, transaction +from django.core.exceptions import ObjectDoesNotExist +from django.db import IntegrityError, connection, transaction from rest_framework_json_api.serializers import ValidationError -from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY +from api.db_router import READ_REPLICA_ALIAS +from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY, rls_transaction +from api.exceptions import ProviderDeletedException +from api.models import Provider, Scan def set_tenant(func=None, *, keep_tenant=False): @@ -66,3 +70,49 @@ def set_tenant(func=None, *, keep_tenant=False): return decorator else: return decorator(func) + + +def handle_provider_deletion(func): + """ + Decorator that raises ProviderDeletedException if provider was deleted during execution. + + Catches ObjectDoesNotExist and IntegrityError, checks if provider still exists, + and raises ProviderDeletedException if not. Otherwise, re-raises original exception. + + Requires tenant_id and provider_id in kwargs. + + Example: + @shared_task + @handle_provider_deletion + def scan_task(scan_id, tenant_id, provider_id): + ... + """ + + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except (ObjectDoesNotExist, IntegrityError): + tenant_id = kwargs.get("tenant_id") + provider_id = kwargs.get("provider_id") + + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + if provider_id is None: + scan_id = kwargs.get("scan_id") + if scan_id is None: + raise AssertionError( + "This task does not have provider or scan in the kwargs" + ) + scan = Scan.objects.filter(pk=scan_id).first() + if scan is None: + raise ProviderDeletedException( + f"Provider for scan '{scan_id}' was deleted during the scan" + ) from None + provider_id = str(scan.provider_id) + if not Provider.objects.filter(pk=provider_id).exists(): + raise ProviderDeletedException( + f"Provider '{provider_id}' was deleted during the scan" + ) from None + raise + + return wrapper diff --git a/api/src/backend/api/exceptions.py b/api/src/backend/api/exceptions.py index d1cf7d78ce..73af170f94 100644 --- a/api/src/backend/api/exceptions.py +++ b/api/src/backend/api/exceptions.py @@ -66,6 +66,10 @@ class ProviderConnectionError(Exception): """Base exception for provider connection errors.""" +class ProviderDeletedException(Exception): + """Raised when a provider has been deleted during scan/task execution.""" + + def custom_exception_handler(exc, context): if isinstance(exc, django_validation_error): if hasattr(exc, "error_dict"): diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index c11ca193b8..429101b316 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -23,7 +23,9 @@ from api.db_utils import ( StatusEnumField, ) from api.models import ( + AttackSurfaceOverview, ComplianceRequirementOverview, + DailySeveritySummary, Finding, Integration, Invitation, @@ -36,12 +38,14 @@ from api.models import ( PermissionChoices, Processor, Provider, + ProviderComplianceScore, ProviderGroup, ProviderSecret, Resource, ResourceTag, Role, Scan, + ScanCategorySummary, ScanSummary, SeverityChoices, StateChoices, @@ -90,10 +94,62 @@ class ChoiceInFilter(BaseInFilter, ChoiceFilter): pass +class BaseProviderFilter(FilterSet): + """ + Abstract base filter for models with direct FK to Provider. + + Provides standard provider_id and provider_type filters. + Subclasses must define Meta.model. + """ + + provider_id = UUIDFilter(field_name="provider__id", lookup_expr="exact") + provider_id__in = UUIDInFilter(field_name="provider__id", lookup_expr="in") + provider_type = ChoiceFilter( + field_name="provider__provider", choices=Provider.ProviderChoices.choices + ) + provider_type__in = ChoiceInFilter( + field_name="provider__provider", + choices=Provider.ProviderChoices.choices, + lookup_expr="in", + ) + + class Meta: + abstract = True + fields = {} + + +class BaseScanProviderFilter(FilterSet): + """ + Abstract base filter for models with FK to Scan (and Scan has FK to Provider). + + Provides standard provider_id and provider_type filters via scan relationship. + Subclasses must define Meta.model. + """ + + provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") + provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in") + provider_type = ChoiceFilter( + field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices + ) + provider_type__in = ChoiceInFilter( + field_name="scan__provider__provider", + choices=Provider.ProviderChoices.choices, + lookup_expr="in", + ) + + class Meta: + abstract = True + fields = {} + + class CommonFindingFilters(FilterSet): # We filter providers from the scan in findings + # Both 'provider' and 'provider_id' parameters are supported for API consistency + # Frontend uses 'provider_id' uniformly across all endpoints provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in") + provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") + provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in") provider_type = ChoiceFilter( choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider" ) @@ -156,6 +212,9 @@ class CommonFindingFilters(FilterSet): field_name="resources__type", lookup_expr="icontains" ) + category = CharFilter(method="filter_category") + category__in = CharInFilter(field_name="categories", lookup_expr="overlap") + # Temporarily disabled until we implement tag filtering in the UI # resource_tag_key = CharFilter(field_name="resources__tags__key") # resource_tag_key__in = CharInFilter( @@ -187,6 +246,9 @@ class CommonFindingFilters(FilterSet): def filter_resource_type(self, queryset, name, value): return queryset.filter(resource_types__contains=[value]) + def filter_category(self, queryset, name, value): + return queryset.filter(categories__contains=[value]) + def filter_resource_tag(self, queryset, name, value): overall_query = Q() for key_value_pair in value: @@ -778,7 +840,7 @@ class RoleFilter(FilterSet): class ComplianceOverviewFilter(FilterSet): inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") - scan_id = UUIDFilter(field_name="scan_id") + scan_id = UUIDFilter(field_name="scan_id", required=True) region = CharFilter(field_name="region") class Meta: @@ -812,6 +874,68 @@ class ScanSummaryFilter(FilterSet): } +class DailySeveritySummaryFilter(FilterSet): + """Filter for findings_severity/timeseries endpoint.""" + + MAX_DATE_RANGE_DAYS = 365 + + provider_id = UUIDFilter(field_name="provider_id", lookup_expr="exact") + provider_id__in = UUIDInFilter(field_name="provider_id", lookup_expr="in") + provider_type = ChoiceFilter( + field_name="provider__provider", choices=Provider.ProviderChoices.choices + ) + provider_type__in = ChoiceInFilter( + field_name="provider__provider", choices=Provider.ProviderChoices.choices + ) + date_from = DateFilter(method="filter_noop") + date_to = DateFilter(method="filter_noop") + + class Meta: + model = DailySeveritySummary + fields = ["provider_id"] + + def filter_noop(self, queryset, name, value): + return queryset + + def filter_queryset(self, queryset): + if not self.data.get("date_from"): + raise ValidationError( + [ + { + "detail": "This query parameter is required.", + "status": "400", + "source": {"pointer": "filter[date_from]"}, + "code": "required", + } + ] + ) + + today = date.today() + date_from = self.form.cleaned_data.get("date_from") + date_to = min(self.form.cleaned_data.get("date_to") or today, today) + + if (date_to - date_from).days > self.MAX_DATE_RANGE_DAYS: + raise ValidationError( + [ + { + "detail": f"Date range cannot exceed {self.MAX_DATE_RANGE_DAYS} days.", + "status": "400", + "source": {"pointer": "filter[date_from]"}, + "code": "invalid", + } + ] + ) + + # View access + self.request._date_from = date_from + self.request._date_to = date_to + + # Apply date filter (only lte for fill-forward logic) + queryset = queryset.filter(date__lte=date_to) + + return super().filter_queryset(queryset) + + class ScanSummarySeverityFilter(ScanSummaryFilter): """Filter for findings_severity ScanSummary endpoint - includes status filters""" @@ -1031,3 +1155,27 @@ class ThreatScoreSnapshotFilter(FilterSet): "inserted_at": ["date", "gte", "lte"], "overall_score": ["exact", "gte", "lte"], } + + +class AttackSurfaceOverviewFilter(BaseScanProviderFilter): + """Filter for attack surface overview aggregations by provider.""" + + class Meta(BaseScanProviderFilter.Meta): + model = AttackSurfaceOverview + + +class CategoryOverviewFilter(BaseScanProviderFilter): + """Filter for category overview aggregations by provider.""" + + category = CharFilter(field_name="category", lookup_expr="exact") + category__in = CharInFilter(field_name="category", lookup_expr="in") + + class Meta(BaseScanProviderFilter.Meta): + model = ScanCategorySummary + + +class ComplianceWatchlistFilter(BaseProviderFilter): + """Filter for compliance watchlist overview by provider.""" + + class Meta(BaseProviderFilter.Meta): + model = ProviderComplianceScore diff --git a/api/src/backend/api/migrations/0060_attack_surface_overview.py b/api/src/backend/api/migrations/0060_attack_surface_overview.py new file mode 100644 index 0000000000..8007d49a70 --- /dev/null +++ b/api/src/backend/api/migrations/0060_attack_surface_overview.py @@ -0,0 +1,89 @@ +# Generated by Django 5.1.14 on 2025-11-19 13:03 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0059_compliance_overview_summary"), + ] + + operations = [ + migrations.CreateModel( + name="AttackSurfaceOverview", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ( + "attack_surface_type", + models.CharField( + choices=[ + ("internet-exposed", "Internet Exposed"), + ("secrets", "Exposed Secrets"), + ("privilege-escalation", "Privilege Escalation"), + ("ec2-imdsv1", "EC2 IMDSv1 Enabled"), + ], + max_length=50, + ), + ), + ("total_findings", models.IntegerField(default=0)), + ("failed_findings", models.IntegerField(default=0)), + ("muted_failed_findings", models.IntegerField(default=0)), + ], + options={ + "db_table": "attack_surface_overviews", + "abstract": False, + }, + ), + migrations.AddField( + model_name="attacksurfaceoverview", + name="scan", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="attack_surface_overviews", + related_query_name="attack_surface_overview", + to="api.scan", + ), + ), + migrations.AddField( + model_name="attacksurfaceoverview", + name="tenant", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + migrations.AddIndex( + model_name="attacksurfaceoverview", + index=models.Index( + fields=["tenant_id", "scan_id"], name="attack_surf_tenant_scan_idx" + ), + ), + migrations.AddConstraint( + model_name="attacksurfaceoverview", + constraint=models.UniqueConstraint( + fields=("tenant_id", "scan_id", "attack_surface_type"), + name="unique_attack_surface_per_scan", + ), + ), + migrations.AddConstraint( + model_name="attacksurfaceoverview", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_attacksurfaceoverview", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + ] diff --git a/api/src/backend/api/migrations/0061_daily_severity_summary.py b/api/src/backend/api/migrations/0061_daily_severity_summary.py new file mode 100644 index 0000000000..7e4074cf7f --- /dev/null +++ b/api/src/backend/api/migrations/0061_daily_severity_summary.py @@ -0,0 +1,96 @@ +# Generated by Django 5.1.14 on 2025-12-03 13:38 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0060_attack_surface_overview"), + ] + + operations = [ + migrations.CreateModel( + name="DailySeveritySummary", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("date", models.DateField()), + ("critical", models.IntegerField(default=0)), + ("high", models.IntegerField(default=0)), + ("medium", models.IntegerField(default=0)), + ("low", models.IntegerField(default=0)), + ("informational", models.IntegerField(default=0)), + ("muted", models.IntegerField(default=0)), + ( + "provider", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="daily_severity_summaries", + related_query_name="daily_severity_summary", + to="api.provider", + ), + ), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="daily_severity_summaries", + related_query_name="daily_severity_summary", + to="api.scan", + ), + ), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="api.tenant", + ), + ), + ], + options={ + "db_table": "daily_severity_summaries", + "abstract": False, + }, + ), + migrations.AddIndex( + model_name="dailyseveritysummary", + index=models.Index( + fields=["tenant_id", "id"], + name="dss_tenant_id_idx", + ), + ), + migrations.AddIndex( + model_name="dailyseveritysummary", + index=models.Index( + fields=["tenant_id", "provider_id"], + name="dss_tenant_provider_idx", + ), + ), + migrations.AddConstraint( + model_name="dailyseveritysummary", + constraint=models.UniqueConstraint( + fields=("tenant_id", "provider", "date"), + name="unique_daily_severity_summary", + ), + ), + migrations.AddConstraint( + model_name="dailyseveritysummary", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_dailyseveritysummary", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + ] diff --git a/api/src/backend/api/migrations/0062_backfill_daily_severity_summaries.py b/api/src/backend/api/migrations/0062_backfill_daily_severity_summaries.py new file mode 100644 index 0000000000..f893ac4305 --- /dev/null +++ b/api/src/backend/api/migrations/0062_backfill_daily_severity_summaries.py @@ -0,0 +1,30 @@ +# Generated by Django 5.1.14 on 2025-12-10 + +from django.db import migrations +from tasks.tasks import backfill_daily_severity_summaries_task + +from api.db_router import MainRouter +from api.rls import Tenant + + +def trigger_backfill_task(apps, schema_editor): + """ + Trigger the backfill task for all tenants. + + This dispatches backfill_daily_severity_summaries_task for each tenant + in the system to populate DailySeveritySummary records from historical scans. + """ + tenant_ids = Tenant.objects.using(MainRouter.admin_db).values_list("id", flat=True) + + for tenant_id in tenant_ids: + backfill_daily_severity_summaries_task.delay(tenant_id=str(tenant_id), days=90) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0061_daily_severity_summary"), + ] + + operations = [ + migrations.RunPython(trigger_backfill_task, migrations.RunPython.noop), + ] diff --git a/api/src/backend/api/migrations/0063_scan_category_summary.py b/api/src/backend/api/migrations/0063_scan_category_summary.py new file mode 100644 index 0000000000..6ee67bf4db --- /dev/null +++ b/api/src/backend/api/migrations/0063_scan_category_summary.py @@ -0,0 +1,111 @@ +import uuid + +import django.db.models.deletion +from django.db import migrations, models + +import api.db_utils +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0062_backfill_daily_severity_summaries"), + ] + + operations = [ + migrations.CreateModel( + name="ScanCategorySummary", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="api.tenant", + ), + ), + ( + "inserted_at", + models.DateTimeField(auto_now_add=True), + ), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="category_summaries", + related_query_name="category_summary", + to="api.scan", + ), + ), + ( + "category", + models.CharField(max_length=100), + ), + ( + "severity", + api.db_utils.SeverityEnumField( + choices=[ + ("critical", "Critical"), + ("high", "High"), + ("medium", "Medium"), + ("low", "Low"), + ("informational", "Informational"), + ], + ), + ), + ( + "total_findings", + models.IntegerField( + default=0, help_text="Non-muted findings (PASS + FAIL)" + ), + ), + ( + "failed_findings", + models.IntegerField( + default=0, + help_text="Non-muted FAIL findings (subset of total_findings)", + ), + ), + ( + "new_failed_findings", + models.IntegerField( + default=0, + help_text="Non-muted FAIL with delta='new' (subset of failed_findings)", + ), + ), + ], + options={ + "db_table": "scan_category_summaries", + "abstract": False, + }, + ), + migrations.AddIndex( + model_name="scancategorysummary", + index=models.Index( + fields=["tenant_id", "scan"], name="scs_tenant_scan_idx" + ), + ), + migrations.AddConstraint( + model_name="scancategorysummary", + constraint=models.UniqueConstraint( + fields=("tenant_id", "scan_id", "category", "severity"), + name="unique_category_severity_per_scan", + ), + ), + migrations.AddConstraint( + model_name="scancategorysummary", + constraint=api.rls.RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_scancategorysummary", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + ] diff --git a/api/src/backend/api/migrations/0064_finding_categories.py b/api/src/backend/api/migrations/0064_finding_categories.py new file mode 100644 index 0000000000..8a0fc1df3a --- /dev/null +++ b/api/src/backend/api/migrations/0064_finding_categories.py @@ -0,0 +1,22 @@ +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0063_scan_category_summary"), + ] + + operations = [ + migrations.AddField( + model_name="finding", + name="categories", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=100), + blank=True, + null=True, + size=None, + help_text="Categories from check metadata for efficient filtering", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0065_alibabacloud_provider.py b/api/src/backend/api/migrations/0065_alibabacloud_provider.py new file mode 100644 index 0000000000..6ad542b643 --- /dev/null +++ b/api/src/backend/api/migrations/0065_alibabacloud_provider.py @@ -0,0 +1,37 @@ +# Generated by Django migration for Alibaba Cloud provider support + +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0064_finding_categories"), + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("mongodbatlas", "MongoDB Atlas"), + ("iac", "IaC"), + ("oraclecloud", "Oracle Cloud Infrastructure"), + ("alibabacloud", "Alibaba Cloud"), + ], + default="aws", + ), + ), + migrations.RunSQL( + "ALTER TYPE provider ADD VALUE IF NOT EXISTS 'alibabacloud';", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/api/src/backend/api/migrations/0066_provider_compliance_score.py b/api/src/backend/api/migrations/0066_provider_compliance_score.py new file mode 100644 index 0000000000..f9a6483e4f --- /dev/null +++ b/api/src/backend/api/migrations/0066_provider_compliance_score.py @@ -0,0 +1,94 @@ +import uuid + +import django.db.models.deletion +from django.db import migrations, models + +import api.db_utils +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0065_alibabacloud_provider"), + ] + + operations = [ + migrations.CreateModel( + name="ProviderComplianceScore", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("compliance_id", models.TextField()), + ("requirement_id", models.TextField()), + ( + "requirement_status", + api.db_utils.StatusEnumField( + choices=[ + ("FAIL", "Fail"), + ("PASS", "Pass"), + ("MANUAL", "Manual"), + ] + ), + ), + ("scan_completed_at", models.DateTimeField()), + ( + "provider", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="compliance_scores", + related_query_name="compliance_score", + to="api.provider", + ), + ), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="compliance_scores", + related_query_name="compliance_score", + to="api.scan", + ), + ), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="api.tenant", + ), + ), + ], + options={ + "db_table": "provider_compliance_scores", + "abstract": False, + }, + ), + migrations.AddConstraint( + model_name="providercompliancescore", + constraint=models.UniqueConstraint( + fields=("tenant_id", "provider_id", "compliance_id", "requirement_id"), + name="unique_provider_compliance_req", + ), + ), + migrations.AddConstraint( + model_name="providercompliancescore", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_providercompliancescore", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + migrations.AddIndex( + model_name="providercompliancescore", + index=models.Index( + fields=["tenant_id", "provider_id", "compliance_id"], + name="pcs_tenant_prov_comp_idx", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0067_tenant_compliance_summary.py b/api/src/backend/api/migrations/0067_tenant_compliance_summary.py new file mode 100644 index 0000000000..bd753ca575 --- /dev/null +++ b/api/src/backend/api/migrations/0067_tenant_compliance_summary.py @@ -0,0 +1,61 @@ +import uuid + +import django.db.models.deletion +from django.db import migrations, models + +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0066_provider_compliance_score"), + ] + + operations = [ + migrations.CreateModel( + name="TenantComplianceSummary", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("compliance_id", models.TextField()), + ("requirements_passed", models.IntegerField(default=0)), + ("requirements_failed", models.IntegerField(default=0)), + ("requirements_manual", models.IntegerField(default=0)), + ("total_requirements", models.IntegerField(default=0)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="api.tenant", + ), + ), + ], + options={ + "db_table": "tenant_compliance_summaries", + "abstract": False, + }, + ), + migrations.AddConstraint( + model_name="tenantcompliancesummary", + constraint=models.UniqueConstraint( + fields=("tenant_id", "compliance_id"), + name="unique_tenant_compliance_summary", + ), + ), + migrations.AddConstraint( + model_name="tenantcompliancesummary", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_tenantcompliancesummary", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index 5f00c578e2..e2436b3fae 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -287,6 +287,7 @@ class Provider(RowLevelSecurityProtectedModel): MONGODBATLAS = "mongodbatlas", _("MongoDB Atlas") IAC = "iac", _("IaC") ORACLECLOUD = "oraclecloud", _("Oracle Cloud Infrastructure") + ALIBABACLOUD = "alibabacloud", _("Alibaba Cloud") @staticmethod def validate_aws_uid(value): @@ -391,6 +392,15 @@ class Provider(RowLevelSecurityProtectedModel): pointer="/data/attributes/uid", ) + @staticmethod + def validate_alibabacloud_uid(value): + if not re.match(r"^\d{16}$", value): + raise ModelValidationError( + detail="Alibaba Cloud account ID must be exactly 16 digits.", + code="alibabacloud-uid", + pointer="/data/attributes/uid", + ) + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) inserted_at = models.DateTimeField(auto_now_add=True, editable=False) updated_at = models.DateTimeField(auto_now=True, editable=False) @@ -811,14 +821,19 @@ class Resource(RowLevelSecurityProtectedModel): self.clear_tags() return - # Add new relationships with the tenant_id field + # Add new relationships with the tenant_id field; avoid touching the + # Resource row unless a mapping is actually created to prevent noisy + # updates during scans. + mapping_created = False for tag in tags: - ResourceTagMapping.objects.update_or_create( + _, created = ResourceTagMapping.objects.update_or_create( tag=tag, resource=self, tenant_id=self.tenant_id ) + mapping_created = mapping_created or created - # Save the instance - self.save() + if mapping_created: + # Only bump updated_at when the tag set truly changed + self.save(update_fields=["updated_at"]) class Meta(RowLevelSecurityProtectedModel.Meta): db_table = "resources" @@ -963,6 +978,14 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): null=True, ) + # Check metadata denormalization + categories = ArrayField( + models.CharField(max_length=100), + blank=True, + null=True, + help_text="Categories from check metadata for efficient filtering", + ) + # Relationships scan = models.ForeignKey(to=Scan, related_name="findings", on_delete=models.CASCADE) @@ -1595,6 +1618,65 @@ class ScanSummary(RowLevelSecurityProtectedModel): resource_name = "scan-summaries" +class DailySeveritySummary(RowLevelSecurityProtectedModel): + """ + Pre-aggregated daily severity counts per provider. + Used by findings_severity/timeseries endpoint for efficient queries. + """ + + objects = ActiveProviderManager() + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + date = models.DateField() + + provider = models.ForeignKey( + Provider, + on_delete=models.CASCADE, + related_name="daily_severity_summaries", + related_query_name="daily_severity_summary", + ) + scan = models.ForeignKey( + Scan, + on_delete=models.CASCADE, + related_name="daily_severity_summaries", + related_query_name="daily_severity_summary", + ) + + # Aggregated fail counts by severity + critical = models.IntegerField(default=0) + high = models.IntegerField(default=0) + medium = models.IntegerField(default=0) + low = models.IntegerField(default=0) + informational = models.IntegerField(default=0) + muted = models.IntegerField(default=0) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "daily_severity_summaries" + + constraints = [ + models.UniqueConstraint( + fields=("tenant_id", "provider", "date"), + name="unique_daily_severity_summary", + ), + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + + indexes = [ + models.Index( + fields=["tenant_id", "id"], + name="dss_tenant_id_idx", + ), + models.Index( + fields=["tenant_id", "provider_id"], + name="dss_tenant_provider_idx", + ), + ] + + class Integration(RowLevelSecurityProtectedModel): class IntegrationChoices(models.TextChoices): AMAZON_S3 = "amazon_s3", _("Amazon S3") @@ -1987,6 +2069,64 @@ class ResourceScanSummary(RowLevelSecurityProtectedModel): ] +class ScanCategorySummary(RowLevelSecurityProtectedModel): + """ + Pre-aggregated category metrics per scan by severity. + + Stores one row per (category, severity) combination per scan for efficient + overview queries. Categories come from check_metadata.categories. + + Count relationships (each is a subset of the previous): + - total_findings >= failed_findings >= new_failed_findings + """ + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + + scan = models.ForeignKey( + Scan, + on_delete=models.CASCADE, + related_name="category_summaries", + related_query_name="category_summary", + ) + + category = models.CharField(max_length=100) + severity = SeverityEnumField(choices=SeverityChoices) + + total_findings = models.IntegerField( + default=0, help_text="Non-muted findings (PASS + FAIL)" + ) + failed_findings = models.IntegerField( + default=0, help_text="Non-muted FAIL findings (subset of total_findings)" + ) + new_failed_findings = models.IntegerField( + default=0, + help_text="Non-muted FAIL with delta='new' (subset of failed_findings)", + ) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "scan_category_summaries" + + indexes = [ + models.Index(fields=["tenant_id", "scan"], name="scs_tenant_scan_idx"), + ] + + constraints = [ + models.UniqueConstraint( + fields=("tenant_id", "scan_id", "category", "severity"), + name="unique_category_severity_per_scan", + ), + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + + class JSONAPIMeta: + resource_name = "scan-category-summaries" + + class LighthouseConfiguration(RowLevelSecurityProtectedModel): """ Stores configuration and API keys for LLM services. @@ -2500,3 +2640,152 @@ class ThreatScoreSnapshot(RowLevelSecurityProtectedModel): class JSONAPIMeta: resource_name = "threatscore-snapshots" + + +class AttackSurfaceOverview(RowLevelSecurityProtectedModel): + """ + Pre-aggregated attack surface metrics per scan. + + Stores counts for each attack surface type (internet-exposed, secrets, + privilege-escalation, ec2-imdsv1) to enable fast overview queries. + """ + + class AttackSurfaceTypeChoices(models.TextChoices): + INTERNET_EXPOSED = "internet-exposed", _("Internet Exposed") + SECRETS = "secrets", _("Exposed Secrets") + PRIVILEGE_ESCALATION = "privilege-escalation", _("Privilege Escalation") + EC2_IMDSV1 = "ec2-imdsv1", _("EC2 IMDSv1 Enabled") + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + + scan = models.ForeignKey( + Scan, + on_delete=models.CASCADE, + related_name="attack_surface_overviews", + related_query_name="attack_surface_overview", + ) + + attack_surface_type = models.CharField( + max_length=50, + choices=AttackSurfaceTypeChoices.choices, + ) + + # Finding counts + total_findings = models.IntegerField(default=0) # All findings (PASS + FAIL) + failed_findings = models.IntegerField(default=0) # Non-muted failed findings + muted_failed_findings = models.IntegerField(default=0) # Muted failed findings + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "attack_surface_overviews" + + constraints = [ + models.UniqueConstraint( + fields=("tenant_id", "scan_id", "attack_surface_type"), + name="unique_attack_surface_per_scan", + ), + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + + indexes = [ + models.Index( + fields=["tenant_id", "scan_id"], + name="attack_surf_tenant_scan_idx", + ), + ] + + class JSONAPIMeta: + resource_name = "attack-surface-overviews" + + +class ProviderComplianceScore(RowLevelSecurityProtectedModel): + """ + Compliance requirement status from latest completed scan per provider. + + Used for efficient compliance watchlist queries with FAIL-dominant aggregation + across multiple providers. Updated via atomic upsert after each scan completion. + """ + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + + scan = models.ForeignKey( + Scan, + on_delete=models.CASCADE, + related_name="compliance_scores", + related_query_name="compliance_score", + ) + + provider = models.ForeignKey( + Provider, + on_delete=models.CASCADE, + related_name="compliance_scores", + related_query_name="compliance_score", + ) + + compliance_id = models.TextField() + requirement_id = models.TextField() + requirement_status = StatusEnumField(choices=StatusChoices) + + scan_completed_at = models.DateTimeField() + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "provider_compliance_scores" + + constraints = [ + models.UniqueConstraint( + fields=("tenant_id", "provider_id", "compliance_id", "requirement_id"), + name="unique_provider_compliance_req", + ), + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + + indexes = [ + models.Index( + fields=["tenant_id", "provider_id", "compliance_id"], + name="pcs_tenant_prov_comp_idx", + ), + ] + + +class TenantComplianceSummary(RowLevelSecurityProtectedModel): + """ + Pre-aggregated compliance counts per tenant with FAIL-dominant logic applied. + + One row per (tenant, compliance_id). Used for fast watchlist queries when + no provider filter is applied. Recalculated after each scan by aggregating + across all providers with FAIL-dominant logic at requirement level. + """ + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + + compliance_id = models.TextField() + + requirements_passed = models.IntegerField(default=0) + requirements_failed = models.IntegerField(default=0) + requirements_manual = models.IntegerField(default=0) + total_requirements = models.IntegerField(default=0) + + updated_at = models.DateTimeField(auto_now=True) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "tenant_compliance_summaries" + + constraints = [ + models.UniqueConstraint( + fields=("tenant_id", "compliance_id"), + name="unique_tenant_compliance_summary", + ), + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] diff --git a/api/src/backend/api/rbac/permissions.py b/api/src/backend/api/rbac/permissions.py index 6a95e82932..97d7d785e0 100644 --- a/api/src/backend/api/rbac/permissions.py +++ b/api/src/backend/api/rbac/permissions.py @@ -65,11 +65,11 @@ def get_providers(role: Role) -> QuerySet[Provider]: A QuerySet of Provider objects filtered by the role's provider groups. If the role has no provider groups, returns an empty queryset. """ - tenant = role.tenant + tenant_id = role.tenant_id provider_groups = role.provider_groups.all() if not provider_groups.exists(): return Provider.objects.none() return Provider.objects.filter( - tenant=tenant, provider_groups__in=provider_groups + tenant_id=tenant_id, provider_groups__in=provider_groups ).distinct() diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 4a1791e65b..0ef163a9b5 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.15.0 + version: 1.18.0 description: |- Prowler API specification. @@ -1140,6 +1140,7 @@ paths: - severity - check_id - check_metadata + - categories - raw_result - inserted_at - updated_at @@ -1152,6 +1153,19 @@ paths: description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false + - in: query + name: filter[category] + schema: + type: string + - in: query + name: filter[category__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[check_id] schema: @@ -1294,12 +1308,28 @@ paths: description: Multiple values may be separated by commas. explode: false style: form + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -1319,14 +1349,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -1348,6 +1380,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -1634,6 +1667,7 @@ paths: - severity - check_id - check_metadata + - categories - raw_result - inserted_at - updated_at @@ -1694,6 +1728,19 @@ paths: description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false + - in: query + name: filter[category] + schema: + type: string + - in: query + name: filter[category__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[check_id] schema: @@ -1833,12 +1880,28 @@ paths: description: Multiple values may be separated by commas. explode: false style: form + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -1858,14 +1921,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -1887,6 +1952,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -2151,6 +2217,7 @@ paths: - severity - check_id - check_metadata + - categories - raw_result - inserted_at - updated_at @@ -2163,6 +2230,19 @@ paths: description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false + - in: query + name: filter[category] + schema: + type: string + - in: query + name: filter[category__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[check_id] schema: @@ -2280,12 +2360,28 @@ paths: description: Multiple values may be separated by commas. explode: false style: form + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -2305,14 +2401,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -2334,6 +2432,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -2580,9 +2679,23 @@ paths: - services - regions - resource_types + - categories description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false + - in: query + name: filter[category] + schema: + type: string + - in: query + name: filter[category__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[check_id] schema: @@ -2725,12 +2838,28 @@ paths: description: Multiple values may be separated by commas. explode: false style: form + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -2750,14 +2879,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -2779,6 +2910,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -3038,9 +3170,23 @@ paths: - services - regions - resource_types + - categories description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false + - in: query + name: filter[category] + schema: + type: string + - in: query + name: filter[category__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[check_id] schema: @@ -3158,12 +3304,28 @@ paths: description: Multiple values may be separated by commas. explode: false style: form + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -3183,14 +3345,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -3212,6 +3376,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -4926,6 +5091,346 @@ paths: responses: '204': description: No response body + /api/v1/overviews/attack-surfaces: + get: + operationId: overviews_attack_surfaces_list + description: Retrieve aggregated attack surface metrics from latest completed + scans per provider. + summary: Get attack surface overview + parameters: + - in: query + name: fields[attack-surface-overviews] + schema: + type: array + items: + type: string + enum: + - id + - total_findings + - failed_findings + - muted_failed_findings + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + x-spec-enum-id: 684bf4173d2b754f + enum: + - alibabacloud + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + x-spec-enum-id: 684bf4173d2b754f + enum: + - alibabacloud + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - total_findings + - -total_findings + - failed_findings + - -failed_findings + - muted_failed_findings + - -muted_failed_findings + explode: false + tags: + - Overview + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedAttackSurfaceOverviewList' + description: '' + /api/v1/overviews/categories: + get: + operationId: overviews_categories_list + description: 'Retrieve aggregated category metrics from latest completed scans + per provider. Returns one row per category with total, failed, and new failed + findings counts, plus a severity breakdown showing failed findings per severity + level. ' + summary: Get category overview + parameters: + - in: query + name: fields[category-overviews] + schema: + type: array + items: + type: string + enum: + - id + - total_findings + - failed_findings + - new_failed_findings + - severity + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[category] + schema: + type: string + - in: query + name: filter[category__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + x-spec-enum-id: 684bf4173d2b754f + enum: + - alibabacloud + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + x-spec-enum-id: 684bf4173d2b754f + enum: + - alibabacloud + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - total_findings + - -total_findings + - failed_findings + - -failed_findings + - new_failed_findings + - -new_failed_findings + - severity + - -severity + explode: false + tags: + - Overview + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedCategoryOverviewList' + description: '' + /api/v1/overviews/compliance-watchlist: + get: + operationId: overviews_compliance_watchlist_retrieve + description: |- + Get compliance watchlist overview with FAIL-dominant aggregation. + + Without filters: uses pre-aggregated TenantComplianceSummary (~70 rows). + With provider filters: queries ProviderComplianceScore with FAIL-dominant logic. + parameters: + - in: query + name: fields[compliance-watchlist-overviews] + schema: + type: array + items: + type: string + enum: + - id + - compliance_id + - requirements_passed + - requirements_failed + - requirements_manual + - total_requirements + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + tags: + - Overview + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/ComplianceWatchlistOverviewResponse' + description: '' /api/v1/overviews/findings: get: operationId: overviews_findings_retrieve @@ -4999,8 +5504,9 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -5020,14 +5526,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -5049,6 +5557,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -5188,8 +5697,9 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -5209,14 +5719,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -5238,6 +5750,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -5315,6 +5828,160 @@ paths: schema: $ref: '#/components/schemas/OverviewSeverityResponse' description: '' + /api/v1/overviews/findings_severity/timeseries: + get: + operationId: overviews_findings_severity_timeseries_retrieve + description: Retrieve daily aggregated findings data grouped by severity levels + over a date range. Returns one data point per day with counts of failed findings + by severity (critical, high, medium, low, informational) and muted findings. + Days without scans are filled forward with the most recent known values. Use + date_from (required) and date_to filters to specify the range. + summary: Get findings severity data over time + parameters: + - in: query + name: fields[findings-severity-over-time] + schema: + type: array + items: + type: string + enum: + - id + - critical + - high + - medium + - low + - informational + - muted + - scan_ids + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[date_from] + schema: + type: string + format: date + - in: query + name: filter[date_to] + schema: + type: string + format: date + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - alibabacloud + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - alibabacloud + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - critical + - -critical + - high + - -high + - medium + - -medium + - low + - -low + - informational + - -informational + - muted + - -muted + - scan_ids + - -scan_ids + explode: false + tags: + - Overview + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/FindingsSeverityOverTimeResponse' + description: '' /api/v1/overviews/providers: get: operationId: overviews_providers_retrieve @@ -5445,8 +6112,9 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -5466,14 +6134,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -5495,6 +6165,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -5617,8 +6288,9 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -5638,14 +6310,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -5667,6 +6341,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -6436,8 +7111,9 @@ paths: name: filter[provider] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -6457,14 +7133,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -6486,14 +7164,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -6513,14 +7193,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -6542,6 +7224,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - name: filter[search] @@ -7142,8 +7825,9 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -7163,14 +7847,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -7192,6 +7878,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -7525,8 +8212,9 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -7546,14 +8234,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -7575,6 +8265,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -7803,8 +8494,9 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -7824,14 +8516,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -7853,6 +8547,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -8087,8 +8782,9 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -8108,14 +8804,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -8137,6 +8835,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -8934,8 +9633,9 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -8955,14 +9655,16 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f enum: + - alibabacloud - aws - azure - gcp @@ -8984,6 +9686,7 @@ paths: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud explode: false style: form - in: query @@ -11047,6 +11750,7 @@ paths: description: '' components: schemas: +<<<<<<< HEAD AttackPathsNode: type: object required: @@ -11077,6 +11781,9 @@ components: - labels - properties AttackPathsQuery: +======= + AttackSurfaceOverview: +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 type: object required: - type @@ -11089,13 +11796,18 @@ components: member is used to describe resource objects that share common attributes and relationships. enum: +<<<<<<< HEAD - attack-paths-query +======= + - attack-surface-overviews +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 id: {} attributes: type: object properties: id: type: string +<<<<<<< HEAD name: type: string description: @@ -11113,6 +11825,20 @@ components: - provider - parameters AttackPathsQueryParameter: +======= + total_findings: + type: integer + failed_findings: + type: integer + muted_failed_findings: + type: integer + required: + - id + - total_findings + - failed_findings + - muted_failed_findings + CategoryOverview: +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 type: object required: - type @@ -11125,11 +11851,16 @@ components: member is used to describe resource objects that share common attributes and relationships. enum: +<<<<<<< HEAD - attack-paths-query-parameter +======= + - category-overviews +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 id: {} attributes: type: object properties: +<<<<<<< HEAD name: type: string label: @@ -11390,6 +12121,25 @@ components: $ref: '#/components/schemas/AttackPathsScan' required: - data +======= + id: + type: string + total_findings: + type: integer + failed_findings: + type: integer + new_failed_findings: + type: integer + severity: + description: 'Severity breakdown: {informational, low, medium, high, + critical}' + required: + - id + - total_findings + - failed_findings + - new_failed_findings + - severity +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 ComplianceOverview: type: object required: @@ -11539,6 +12289,50 @@ components: type: string required: - regions + ComplianceWatchlistOverview: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - compliance-watchlist-overviews + id: {} + attributes: + type: object + properties: + id: + type: string + compliance_id: + type: string + requirements_passed: + type: integer + requirements_failed: + type: integer + requirements_manual: + type: integer + total_requirements: + type: integer + required: + - id + - compliance_id + - requirements_passed + - requirements_failed + - requirements_manual + - total_requirements + ComplianceWatchlistOverviewResponse: + type: object + properties: + data: + $ref: '#/components/schemas/ComplianceWatchlistOverview' + required: + - data Finding: type: object required: @@ -11606,6 +12400,13 @@ components: type: string maxLength: 100 check_metadata: {} + categories: + type: array + items: + type: string + maxLength: 100 + nullable: true + description: Categories from check metadata for efficient filtering raw_result: {} inserted_at: type: string @@ -11756,10 +12557,15 @@ components: type: array items: type: string + categories: + type: array + items: + type: string required: - services - regions - resource_types + - categories FindingMetadataResponse: type: object properties: @@ -11774,6 +12580,60 @@ components: $ref: '#/components/schemas/Finding' required: - data + FindingsSeverityOverTime: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - findings-severity-over-time + id: {} + attributes: + type: object + properties: + id: + type: string + format: date + critical: + type: integer + high: + type: integer + medium: + type: integer + low: + type: integer + informational: + type: integer + muted: + type: integer + scan_ids: + type: array + items: + type: string + format: uuid + required: + - id + - critical + - high + - medium + - low + - informational + - muted + - scan_ids + FindingsSeverityOverTimeResponse: + type: object + properties: + data: + $ref: '#/components/schemas/FindingsSeverityOverTime' + required: + - data Integration: type: object required: @@ -13330,26 +14190,46 @@ components: pattern: ^sk-[\w-]+$ required: - api_key - - type: object - title: AWS Bedrock Credentials - properties: - access_key_id: - type: string - description: AWS access key ID. - pattern: ^AKIA[0-9A-Z]{16}$ - secret_access_key: - type: string - description: AWS secret access key. - pattern: ^[A-Za-z0-9/+=]{40}$ - region: - type: string - description: 'AWS region identifier where Bedrock is available. - Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' - pattern: ^[a-z]{2}-[a-z]+-\d+$ - required: - - access_key_id - - secret_access_key - - region + - title: AWS Bedrock Credentials + oneOf: + - title: IAM Access Key Pair + type: object + description: Authenticate with AWS access key and secret key. Recommended + when you manage IAM users or roles. + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - title: Amazon Bedrock API Key + type: object + description: Authenticate with an Amazon Bedrock API key (bearer + token). Region is still required. + properties: + api_key: + type: string + description: Amazon Bedrock API key (bearer token). + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - api_key + - region - type: object title: OpenAI Compatible Credentials properties: @@ -13417,26 +14297,46 @@ components: pattern: ^sk-[\w-]+$ required: - api_key - - type: object - title: AWS Bedrock Credentials - properties: - access_key_id: - type: string - description: AWS access key ID. - pattern: ^AKIA[0-9A-Z]{16}$ - secret_access_key: - type: string - description: AWS secret access key. - pattern: ^[A-Za-z0-9/+=]{40}$ - region: - type: string - description: 'AWS region identifier where Bedrock is available. - Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' - pattern: ^[a-z]{2}-[a-z]+-\d+$ - required: - - access_key_id - - secret_access_key - - region + - title: AWS Bedrock Credentials + oneOf: + - title: IAM Access Key Pair + type: object + description: Authenticate with AWS access key and secret key. + Recommended when you manage IAM users or roles. + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - title: Amazon Bedrock API Key + type: object + description: Authenticate with an Amazon Bedrock API key (bearer + token). Region is still required. + properties: + api_key: + type: string + description: Amazon Bedrock API key (bearer token). + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - api_key + - region - type: object title: OpenAI Compatible Credentials properties: @@ -13522,26 +14422,46 @@ components: pattern: ^sk-[\w-]+$ required: - api_key - - type: object - title: AWS Bedrock Credentials - properties: - access_key_id: - type: string - description: AWS access key ID. - pattern: ^AKIA[0-9A-Z]{16}$ - secret_access_key: - type: string - description: AWS secret access key. - pattern: ^[A-Za-z0-9/+=]{40}$ - region: - type: string - description: 'AWS region identifier where Bedrock is available. - Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' - pattern: ^[a-z]{2}-[a-z]+-\d+$ - required: - - access_key_id - - secret_access_key - - region + - title: AWS Bedrock Credentials + oneOf: + - title: IAM Access Key Pair + type: object + description: Authenticate with AWS access key and secret key. Recommended + when you manage IAM users or roles. + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - title: Amazon Bedrock API Key + type: object + description: Authenticate with an Amazon Bedrock API key (bearer + token). Region is still required. + properties: + api_key: + type: string + description: Amazon Bedrock API key (bearer token). + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - api_key + - region - type: object title: OpenAI Compatible Credentials properties: @@ -14348,22 +15268,37 @@ components: $ref: '#/components/schemas/OverviewSeverity' required: - data +<<<<<<< HEAD PaginatedAttackPathsQueryList: +======= + PaginatedAttackSurfaceOverviewList: +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 type: object properties: data: type: array items: +<<<<<<< HEAD $ref: '#/components/schemas/AttackPathsQuery' required: - data PaginatedAttackPathsScanList: +======= + $ref: '#/components/schemas/AttackSurfaceOverview' + required: + - data + PaginatedCategoryOverviewList: +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 type: object properties: data: type: array items: +<<<<<<< HEAD $ref: '#/components/schemas/AttackPathsScan' +======= + $ref: '#/components/schemas/CategoryOverview' +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 required: - data PaginatedComplianceOverviewAttributesList: @@ -14982,26 +15917,46 @@ components: pattern: ^sk-[\w-]+$ required: - api_key - - type: object - title: AWS Bedrock Credentials - properties: - access_key_id: - type: string - description: AWS access key ID. - pattern: ^AKIA[0-9A-Z]{16}$ - secret_access_key: - type: string - description: AWS secret access key. - pattern: ^[A-Za-z0-9/+=]{40}$ - region: - type: string - description: 'AWS region identifier where Bedrock is available. - Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' - pattern: ^[a-z]{2}-[a-z]+-\d+$ - required: - - access_key_id - - secret_access_key - - region + - title: AWS Bedrock Credentials + oneOf: + - title: IAM Access Key Pair + type: object + description: Authenticate with AWS access key and secret key. + Recommended when you manage IAM users or roles. + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - title: Amazon Bedrock API Key + type: object + description: Authenticate with an Amazon Bedrock API key (bearer + token). Region is still required. + properties: + api_key: + type: string + description: Amazon Bedrock API key (bearer token). + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - api_key + - region - type: object title: OpenAI Compatible Credentials properties: @@ -15613,6 +16568,44 @@ components: required: - atlas_public_key - atlas_private_key + - type: object + title: Alibaba Cloud Static Credentials + properties: + access_key_id: + type: string + description: The Alibaba Cloud access key ID for authentication. + access_key_secret: + type: string + description: The Alibaba Cloud access key secret for authentication. + security_token: + type: string + description: The STS security token for temporary credentials + (optional). + required: + - access_key_id + - access_key_secret + - type: object + title: Alibaba Cloud RAM Role Assumption + properties: + role_arn: + type: string + description: The ARN of the RAM role to assume (e.g., acs:ram::1234567890123456:role/ProwlerRole). + access_key_id: + type: string + description: The Alibaba Cloud access key ID of the RAM user + that will assume the role. + access_key_secret: + type: string + description: The Alibaba Cloud access key secret of the RAM + user that will assume the role. + role_session_name: + type: string + description: An identifier for the role session (optional, + defaults to 'ProwlerSession'). + required: + - role_arn + - access_key_id + - access_key_secret writeOnly: true required: - secret @@ -16610,6 +17603,7 @@ components: - mongodbatlas - iac - oraclecloud + - alibabacloud type: string description: |- * `aws` - AWS @@ -16621,7 +17615,8 @@ components: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure - x-spec-enum-id: eca8c51e6bd28935 + * `alibabacloud` - Alibaba Cloud + x-spec-enum-id: 684bf4173d2b754f uid: type: string title: Unique identifier for the provider, set by the provider @@ -16736,8 +17731,9 @@ components: - mongodbatlas - iac - oraclecloud + - alibabacloud type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f description: |- Type of provider to create. @@ -16750,6 +17746,7 @@ components: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud uid: type: string title: Unique identifier for the provider, set by the provider @@ -16796,8 +17793,9 @@ components: - mongodbatlas - iac - oraclecloud + - alibabacloud type: string - x-spec-enum-id: eca8c51e6bd28935 + x-spec-enum-id: 684bf4173d2b754f description: |- Type of provider to create. @@ -16810,6 +17808,7 @@ components: * `mongodbatlas` - MongoDB Atlas * `iac` - IaC * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud uid: type: string minLength: 3 @@ -17572,6 +18571,44 @@ components: required: - atlas_public_key - atlas_private_key + - type: object + title: Alibaba Cloud Static Credentials + properties: + access_key_id: + type: string + description: The Alibaba Cloud access key ID for authentication. + access_key_secret: + type: string + description: The Alibaba Cloud access key secret for authentication. + security_token: + type: string + description: The STS security token for temporary credentials + (optional). + required: + - access_key_id + - access_key_secret + - type: object + title: Alibaba Cloud RAM Role Assumption + properties: + role_arn: + type: string + description: The ARN of the RAM role to assume (e.g., acs:ram::1234567890123456:role/ProwlerRole). + access_key_id: + type: string + description: The Alibaba Cloud access key ID of the RAM user that + will assume the role. + access_key_secret: + type: string + description: The Alibaba Cloud access key secret of the RAM user + that will assume the role. + role_session_name: + type: string + description: An identifier for the role session (optional, defaults + to 'ProwlerSession'). + required: + - role_arn + - access_key_id + - access_key_secret writeOnly: true required: - secret_type @@ -17897,6 +18934,44 @@ components: required: - atlas_public_key - atlas_private_key + - type: object + title: Alibaba Cloud Static Credentials + properties: + access_key_id: + type: string + description: The Alibaba Cloud access key ID for authentication. + access_key_secret: + type: string + description: The Alibaba Cloud access key secret for authentication. + security_token: + type: string + description: The STS security token for temporary credentials + (optional). + required: + - access_key_id + - access_key_secret + - type: object + title: Alibaba Cloud RAM Role Assumption + properties: + role_arn: + type: string + description: The ARN of the RAM role to assume (e.g., acs:ram::1234567890123456:role/ProwlerRole). + access_key_id: + type: string + description: The Alibaba Cloud access key ID of the RAM user + that will assume the role. + access_key_secret: + type: string + description: The Alibaba Cloud access key secret of the RAM + user that will assume the role. + role_session_name: + type: string + description: An identifier for the role session (optional, + defaults to 'ProwlerSession'). + required: + - role_arn + - access_key_id + - access_key_secret writeOnly: true required: - secret_type @@ -18236,6 +19311,44 @@ components: required: - atlas_public_key - atlas_private_key + - type: object + title: Alibaba Cloud Static Credentials + properties: + access_key_id: + type: string + description: The Alibaba Cloud access key ID for authentication. + access_key_secret: + type: string + description: The Alibaba Cloud access key secret for authentication. + security_token: + type: string + description: The STS security token for temporary credentials + (optional). + required: + - access_key_id + - access_key_secret + - type: object + title: Alibaba Cloud RAM Role Assumption + properties: + role_arn: + type: string + description: The ARN of the RAM role to assume (e.g., acs:ram::1234567890123456:role/ProwlerRole). + access_key_id: + type: string + description: The Alibaba Cloud access key ID of the RAM user that + will assume the role. + access_key_secret: + type: string + description: The Alibaba Cloud access key secret of the RAM user + that will assume the role. + role_session_name: + type: string + description: An identifier for the role session (optional, defaults + to 'ProwlerSession'). + required: + - role_arn + - access_key_id + - access_key_secret writeOnly: true required: - secret diff --git a/api/src/backend/api/tests/test_decorators.py b/api/src/backend/api/tests/test_decorators.py index 8ac31b1d49..9a113abad8 100644 --- a/api/src/backend/api/tests/test_decorators.py +++ b/api/src/backend/api/tests/test_decorators.py @@ -2,9 +2,12 @@ import uuid from unittest.mock import call, patch import pytest +from django.core.exceptions import ObjectDoesNotExist +from django.db import IntegrityError from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY -from api.decorators import set_tenant +from api.decorators import handle_provider_deletion, set_tenant +from api.exceptions import ProviderDeletedException @pytest.mark.django_db @@ -34,3 +37,142 @@ class TestSetTenantDecorator: with pytest.raises(KeyError): random_func("test_arg") + + +@pytest.mark.django_db +class TestHandleProviderDeletionDecorator: + def test_success_no_exception(self, tenants_fixture, providers_fixture): + """Decorated function runs normally when no exception is raised.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + + @handle_provider_deletion + def task_func(**kwargs): + return "success" + + result = task_func( + tenant_id=str(tenant.id), + provider_id=str(provider.id), + ) + assert result == "success" + + @patch("api.decorators.rls_transaction") + @patch("api.decorators.Provider.objects.filter") + def test_provider_deleted_with_provider_id( + self, mock_filter, mock_rls, tenants_fixture + ): + """Raises ProviderDeletedException when provider_id provided and provider deleted.""" + tenant = tenants_fixture[0] + deleted_provider_id = str(uuid.uuid4()) + + mock_rls.return_value.__enter__ = lambda s: None + mock_rls.return_value.__exit__ = lambda s, *args: None + mock_filter.return_value.exists.return_value = False + + @handle_provider_deletion + def task_func(**kwargs): + raise ObjectDoesNotExist("Some object not found") + + with pytest.raises(ProviderDeletedException) as exc_info: + task_func(tenant_id=str(tenant.id), provider_id=deleted_provider_id) + + assert deleted_provider_id in str(exc_info.value) + + @patch("api.decorators.rls_transaction") + @patch("api.decorators.Provider.objects.filter") + @patch("api.decorators.Scan.objects.filter") + def test_provider_deleted_with_scan_id( + self, mock_scan_filter, mock_provider_filter, mock_rls, tenants_fixture + ): + """Raises ProviderDeletedException when scan exists but provider deleted.""" + tenant = tenants_fixture[0] + scan_id = str(uuid.uuid4()) + provider_id = str(uuid.uuid4()) + + mock_rls.return_value.__enter__ = lambda s: None + mock_rls.return_value.__exit__ = lambda s, *args: None + + mock_scan = type("MockScan", (), {"provider_id": provider_id})() + mock_scan_filter.return_value.first.return_value = mock_scan + mock_provider_filter.return_value.exists.return_value = False + + @handle_provider_deletion + def task_func(**kwargs): + raise ObjectDoesNotExist("Some object not found") + + with pytest.raises(ProviderDeletedException) as exc_info: + task_func(tenant_id=str(tenant.id), scan_id=scan_id) + + assert provider_id in str(exc_info.value) + + @patch("api.decorators.rls_transaction") + @patch("api.decorators.Scan.objects.filter") + def test_scan_deleted_cascade(self, mock_scan_filter, mock_rls, tenants_fixture): + """Raises ProviderDeletedException when scan was deleted (CASCADE from provider).""" + tenant = tenants_fixture[0] + scan_id = str(uuid.uuid4()) + + mock_rls.return_value.__enter__ = lambda s: None + mock_rls.return_value.__exit__ = lambda s, *args: None + mock_scan_filter.return_value.first.return_value = None + + @handle_provider_deletion + def task_func(**kwargs): + raise ObjectDoesNotExist("Some object not found") + + with pytest.raises(ProviderDeletedException) as exc_info: + task_func(tenant_id=str(tenant.id), scan_id=scan_id) + + assert scan_id in str(exc_info.value) + + @patch("api.decorators.rls_transaction") + @patch("api.decorators.Provider.objects.filter") + def test_provider_exists_reraises_original( + self, mock_filter, mock_rls, tenants_fixture, providers_fixture + ): + """Re-raises original exception when provider still exists.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + + mock_rls.return_value.__enter__ = lambda s: None + mock_rls.return_value.__exit__ = lambda s, *args: None + mock_filter.return_value.exists.return_value = True + + @handle_provider_deletion + def task_func(**kwargs): + raise ObjectDoesNotExist("Actual object missing") + + with pytest.raises(ObjectDoesNotExist): + task_func(tenant_id=str(tenant.id), provider_id=str(provider.id)) + + @patch("api.decorators.rls_transaction") + @patch("api.decorators.Provider.objects.filter") + def test_integrity_error_provider_deleted( + self, mock_filter, mock_rls, tenants_fixture + ): + """Raises ProviderDeletedException on IntegrityError when provider deleted.""" + tenant = tenants_fixture[0] + deleted_provider_id = str(uuid.uuid4()) + + mock_rls.return_value.__enter__ = lambda s: None + mock_rls.return_value.__exit__ = lambda s, *args: None + mock_filter.return_value.exists.return_value = False + + @handle_provider_deletion + def task_func(**kwargs): + raise IntegrityError("FK constraint violation") + + with pytest.raises(ProviderDeletedException): + task_func(tenant_id=str(tenant.id), provider_id=deleted_provider_id) + + def test_missing_provider_and_scan_raises_assertion(self, tenants_fixture): + """Raises AssertionError when neither provider_id nor scan_id in kwargs.""" + + @handle_provider_deletion + def task_func(**kwargs): + raise ObjectDoesNotExist("Some object not found") + + with pytest.raises(AssertionError) as exc_info: + task_func(tenant_id=str(tenants_fixture[0].id)) + + assert "provider or scan" in str(exc_info.value) diff --git a/api/src/backend/api/tests/test_models.py b/api/src/backend/api/tests/test_models.py index 79d405d4a7..13878794b1 100644 --- a/api/src/backend/api/tests/test_models.py +++ b/api/src/backend/api/tests/test_models.py @@ -1,9 +1,21 @@ +from datetime import datetime, timezone + import pytest from allauth.socialaccount.models import SocialApp from django.core.exceptions import ValidationError +from django.db import IntegrityError from api.db_router import MainRouter -from api.models import Resource, ResourceTag, SAMLConfiguration, SAMLDomainIndex +from api.models import ( + ProviderComplianceScore, + Resource, + ResourceTag, + SAMLConfiguration, + SAMLDomainIndex, + StateChoices, + StatusChoices, + TenantComplianceSummary, +) @pytest.mark.django_db @@ -324,3 +336,159 @@ class TestSAMLConfigurationModel: errors = exc_info.value.message_dict assert "metadata_xml" in errors assert "There is a problem with your metadata." in errors["metadata_xml"][0] + + +@pytest.mark.django_db +class TestProviderComplianceScoreModel: + def test_create_provider_compliance_score(self, providers_fixture, scans_fixture): + provider = providers_fixture[0] + scan = scans_fixture[0] + scan.completed_at = datetime.now(timezone.utc) + scan.save() + + score = ProviderComplianceScore.objects.create( + tenant_id=provider.tenant_id, + provider=provider, + scan=scan, + compliance_id="aws_cis_2.0", + requirement_id="req_1", + requirement_status=StatusChoices.PASS, + scan_completed_at=scan.completed_at, + ) + + assert score.compliance_id == "aws_cis_2.0" + assert score.requirement_id == "req_1" + assert score.requirement_status == StatusChoices.PASS + + def test_unique_constraint_per_provider_compliance_requirement( + self, providers_fixture, scans_fixture + ): + provider = providers_fixture[0] + scan = scans_fixture[0] + scan.completed_at = datetime.now(timezone.utc) + scan.save() + + ProviderComplianceScore.objects.create( + tenant_id=provider.tenant_id, + provider=provider, + scan=scan, + compliance_id="aws_cis_2.0", + requirement_id="req_1", + requirement_status=StatusChoices.PASS, + scan_completed_at=scan.completed_at, + ) + + with pytest.raises(IntegrityError): + ProviderComplianceScore.objects.create( + tenant_id=provider.tenant_id, + provider=provider, + scan=scan, + compliance_id="aws_cis_2.0", + requirement_id="req_1", + requirement_status=StatusChoices.FAIL, + scan_completed_at=scan.completed_at, + ) + + def test_different_providers_same_requirement_allowed( + self, providers_fixture, scans_fixture + ): + provider1, provider2, *_ = providers_fixture + scan1 = scans_fixture[0] + scan1.completed_at = datetime.now(timezone.utc) + scan1.save() + + scan2 = scans_fixture[2] + scan2.state = StateChoices.COMPLETED + scan2.completed_at = datetime.now(timezone.utc) + scan2.save() + + score1 = ProviderComplianceScore.objects.create( + tenant_id=provider1.tenant_id, + provider=provider1, + scan=scan1, + compliance_id="aws_cis_2.0", + requirement_id="req_1", + requirement_status=StatusChoices.PASS, + scan_completed_at=scan1.completed_at, + ) + + score2 = ProviderComplianceScore.objects.create( + tenant_id=provider2.tenant_id, + provider=provider2, + scan=scan2, + compliance_id="aws_cis_2.0", + requirement_id="req_1", + requirement_status=StatusChoices.FAIL, + scan_completed_at=scan2.completed_at, + ) + + assert score1.id != score2.id + assert score1.requirement_status != score2.requirement_status + + +@pytest.mark.django_db +class TestTenantComplianceSummaryModel: + def test_create_tenant_compliance_summary(self, tenants_fixture): + tenant = tenants_fixture[0] + + summary = TenantComplianceSummary.objects.create( + tenant_id=tenant.id, + compliance_id="aws_cis_2.0", + requirements_passed=5, + requirements_failed=2, + requirements_manual=1, + total_requirements=8, + ) + + assert summary.compliance_id == "aws_cis_2.0" + assert summary.requirements_passed == 5 + assert summary.requirements_failed == 2 + assert summary.requirements_manual == 1 + assert summary.total_requirements == 8 + assert summary.updated_at is not None + + def test_unique_constraint_per_tenant_compliance(self, tenants_fixture): + tenant = tenants_fixture[0] + + TenantComplianceSummary.objects.create( + tenant_id=tenant.id, + compliance_id="aws_cis_2.0", + requirements_passed=5, + requirements_failed=2, + requirements_manual=1, + total_requirements=8, + ) + + with pytest.raises(IntegrityError): + TenantComplianceSummary.objects.create( + tenant_id=tenant.id, + compliance_id="aws_cis_2.0", + requirements_passed=3, + requirements_failed=4, + requirements_manual=1, + total_requirements=8, + ) + + def test_different_tenants_same_compliance_allowed(self, tenants_fixture): + tenant1, tenant2, *_ = tenants_fixture + + summary1 = TenantComplianceSummary.objects.create( + tenant_id=tenant1.id, + compliance_id="aws_cis_2.0", + requirements_passed=5, + requirements_failed=2, + requirements_manual=1, + total_requirements=8, + ) + + summary2 = TenantComplianceSummary.objects.create( + tenant_id=tenant2.id, + compliance_id="aws_cis_2.0", + requirements_passed=3, + requirements_failed=4, + requirements_manual=1, + total_requirements=8, + ) + + assert summary1.id != summary2.id + assert summary1.requirements_passed != summary2.requirements_passed diff --git a/api/src/backend/api/tests/test_utils.py b/api/src/backend/api/tests/test_utils.py index 2229a2f98e..6f9146cd5c 100644 --- a/api/src/backend/api/tests/test_utils.py +++ b/api/src/backend/api/tests/test_utils.py @@ -16,6 +16,7 @@ from api.utils import ( return_prowler_provider, validate_invitation, ) +from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider from prowler.providers.aws.aws_provider import AwsProvider from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection from prowler.providers.azure.azure_provider import AzureProvider @@ -116,6 +117,7 @@ class TestReturnProwlerProvider: (Provider.ProviderChoices.MONGODBATLAS.value, MongodbatlasProvider), (Provider.ProviderChoices.ORACLECLOUD.value, OraclecloudProvider), (Provider.ProviderChoices.IAC.value, IacProvider), + (Provider.ProviderChoices.ALIBABACLOUD.value, AlibabacloudProvider), ], ) def test_return_prowler_provider(self, provider_type, expected_provider): diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 7ba0d7094b..291ab72d7e 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -3,7 +3,7 @@ import io import json import os import tempfile -from datetime import datetime, timedelta, timezone +from datetime import date, datetime, timedelta, timezone from decimal import Decimal from pathlib import Path from types import SimpleNamespace @@ -39,6 +39,10 @@ from api.attack_paths import ( from api.compliance import get_compliance_frameworks from api.db_router import MainRouter from api.models import ( + AttackSurfaceOverview, + ComplianceOverviewSummary, + ComplianceRequirementOverview, + DailySeveritySummary, Finding, Integration, Invitation, @@ -59,6 +63,7 @@ from api.models import ( Scan, ScanSummary, StateChoices, + StatusChoices, Task, TenantAPIKey, ThreatScoreSnapshot, @@ -1164,6 +1169,11 @@ class TestProviderViewSet: "uid": "64b1d3c0e4b03b1234567890", "alias": "Atlas Organization", }, + { + "provider": "alibabacloud", + "uid": "1234567890123456", + "alias": "Alibaba Cloud Account", + }, ] ), ) @@ -1513,6 +1523,36 @@ class TestProviderViewSet: "mongodbatlas-uid", "uid", ), + # Alibaba Cloud UID validation - too short (not 16 digits) + ( + { + "provider": "alibabacloud", + "uid": "123456789012345", + "alias": "test", + }, + "alibabacloud-uid", + "uid", + ), + # Alibaba Cloud UID validation - too long (not 16 digits) + ( + { + "provider": "alibabacloud", + "uid": "12345678901234567", + "alias": "test", + }, + "alibabacloud-uid", + "uid", + ), + # Alibaba Cloud UID validation - contains non-digits + ( + { + "provider": "alibabacloud", + "uid": "123456789012345a", + "alias": "test", + }, + "alibabacloud-uid", + "uid", + ), ] ), ) @@ -1686,21 +1726,21 @@ class TestProviderViewSet: ( "uid.icontains", "1", - 7, + 8, ), ("alias", "aws_testing_1", 1), ("alias.icontains", "aws", 2), - ("inserted_at", TODAY, 8), + ("inserted_at", TODAY, 9), ( "inserted_at.gte", "2024-01-01", - 8, + 9, ), ("inserted_at.lte", "2024-01-01", 0), ( "updated_at.gte", "2024-01-01", - 8, + 9, ), ("updated_at.lte", "2024-01-01", 0), ] @@ -2250,6 +2290,46 @@ class TestProviderSecretViewSet: "atlas_private_key": "private-key", }, ), + # Alibaba Cloud credentials (with access key only) + ( + Provider.ProviderChoices.ALIBABACLOUD.value, + ProviderSecret.TypeChoices.STATIC, + { + "access_key_id": "LTAI5t1234567890abcdef", + "access_key_secret": "my-secret-access-key", + }, + ), + # Alibaba Cloud credentials (with STS security token) + ( + Provider.ProviderChoices.ALIBABACLOUD.value, + ProviderSecret.TypeChoices.STATIC, + { + "access_key_id": "LTAI5t1234567890abcdef", + "access_key_secret": "my-secret-access-key", + "security_token": "my-security-token-for-sts", + }, + ), + # Alibaba Cloud RAM Role Assumption (minimal required fields) + ( + Provider.ProviderChoices.ALIBABACLOUD.value, + ProviderSecret.TypeChoices.ROLE, + { + "role_arn": "acs:ram::1234567890123456:role/ProwlerRole", + "access_key_id": "LTAI5t1234567890abcdef", + "access_key_secret": "my-secret-access-key", + }, + ), + # Alibaba Cloud RAM Role Assumption (with optional role_session_name) + ( + Provider.ProviderChoices.ALIBABACLOUD.value, + ProviderSecret.TypeChoices.ROLE, + { + "role_arn": "acs:ram::1234567890123456:role/ProwlerRole", + "access_key_id": "LTAI5t1234567890abcdef", + "access_key_secret": "my-secret-access-key", + "role_session_name": "ProwlerAuditSession", + }, + ), ], ) def test_provider_secrets_create_valid( @@ -4448,6 +4528,37 @@ class TestFindingViewSet: assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == 2 + def test_finding_filter_by_provider_id_alias( + self, authenticated_client, findings_fixture + ): + """Test that provider_id filter alias works identically to provider filter.""" + response = authenticated_client.get( + reverse("finding-list"), + { + "filter[provider_id]": findings_fixture[0].scan.provider.id, + "filter[inserted_at]": TODAY, + }, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 2 + + def test_finding_filter_by_provider_id_in_alias( + self, authenticated_client, findings_fixture + ): + """Test that provider_id__in filter alias works identically to provider__in filter.""" + response = authenticated_client.get( + reverse("finding-list"), + { + "filter[provider_id__in]": [ + findings_fixture[0].scan.provider.id, + findings_fixture[1].scan.provider.id, + ], + "filter[inserted_at]": TODAY, + }, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 2 + @pytest.mark.parametrize( "filter_name", ( @@ -4669,6 +4780,28 @@ class TestFindingViewSet: == latest_scan_finding.status ) + def test_findings_latest_filter_by_provider_id_alias( + self, authenticated_client, latest_scan_finding + ): + """Test that provider_id filter alias works on latest findings endpoint.""" + response = authenticated_client.get( + reverse("finding-latest"), + {"filter[provider_id]": latest_scan_finding.scan.provider.id}, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 1 + + def test_findings_latest_filter_by_provider_id_in_alias( + self, authenticated_client, latest_scan_finding + ): + """Test that provider_id__in filter alias works on latest findings endpoint.""" + response = authenticated_client.get( + reverse("finding-latest"), + {"filter[provider_id__in]": str(latest_scan_finding.scan.provider.id)}, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 1 + def test_findings_metadata_latest(self, authenticated_client, latest_scan_finding): response = authenticated_client.get( reverse("finding-metadata_latest"), @@ -4680,6 +4813,74 @@ class TestFindingViewSet: assert attributes["regions"] == latest_scan_finding.resource_regions assert attributes["resource_types"] == latest_scan_finding.resource_types + def test_findings_metadata_categories( + self, authenticated_client, findings_with_categories + ): + finding = findings_with_categories + response = authenticated_client.get( + reverse("finding-metadata"), + {"filter[inserted_at]": finding.inserted_at.strftime("%Y-%m-%d")}, + ) + assert response.status_code == status.HTTP_200_OK + attributes = response.json()["data"]["attributes"] + assert set(attributes["categories"]) == {"gen-ai", "security"} + + def test_findings_metadata_latest_categories( + self, authenticated_client, latest_scan_finding_with_categories + ): + response = authenticated_client.get( + reverse("finding-metadata_latest"), + ) + assert response.status_code == status.HTTP_200_OK + attributes = response.json()["data"]["attributes"] + assert set(attributes["categories"]) == {"gen-ai", "iam"} + + def test_findings_filter_by_category( + self, authenticated_client, findings_with_categories + ): + finding = findings_with_categories + response = authenticated_client.get( + reverse("finding-list"), + { + "filter[category]": "gen-ai", + "filter[inserted_at]": finding.inserted_at.strftime("%Y-%m-%d"), + }, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 1 + assert set(response.json()["data"][0]["attributes"]["categories"]) == { + "gen-ai", + "security", + } + + def test_findings_filter_by_category_in( + self, authenticated_client, findings_with_multiple_categories + ): + finding1, _ = findings_with_multiple_categories + response = authenticated_client.get( + reverse("finding-list"), + { + "filter[category__in]": "gen-ai,iam", + "filter[inserted_at]": finding1.inserted_at.strftime("%Y-%m-%d"), + }, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 2 + + def test_findings_filter_by_category_no_match( + self, authenticated_client, findings_with_categories + ): + finding = findings_with_categories + response = authenticated_client.get( + reverse("finding-list"), + { + "filter[category]": "nonexistent", + "filter[inserted_at]": finding.inserted_at.strftime("%Y-%m-%d"), + }, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 0 + @pytest.mark.django_db class TestJWTFields: @@ -6237,16 +6438,44 @@ class TestProviderGroupMembershipViewSet: @pytest.mark.django_db class TestComplianceOverviewViewSet: - def test_compliance_overview_list_none(self, authenticated_client): + @pytest.fixture(autouse=True) + def mock_backfill_task(self): + with patch("api.v1.views.backfill_compliance_summaries_task.delay") as mock: + yield mock + + def test_compliance_overview_list_none( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + mock_backfill_task, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + scan = Scan.objects.create( + name="empty-compliance-scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + response = authenticated_client.get( reverse("complianceoverview-list"), - {"filter[scan_id]": "8d20ac7d-4cbc-435e-85f4-359be37af821"}, + {"filter[scan_id]": str(scan.id)}, ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == 0 + mock_backfill_task.assert_called_once() + _, kwargs = mock_backfill_task.call_args + assert kwargs["scan_id"] == str(scan.id) + assert str(kwargs["tenant_id"]) == str(tenant.id) def test_compliance_overview_list( - self, authenticated_client, compliance_requirements_overviews_fixture + self, + authenticated_client, + compliance_requirements_overviews_fixture, + mock_backfill_task, ): # List compliance overviews with existing data requirement_overview1 = compliance_requirements_overviews_fixture[0] @@ -6276,6 +6505,90 @@ class TestComplianceOverviewViewSet: assert "requirements_failed" in attributes assert "requirements_manual" in attributes assert "total_requirements" in attributes + mock_backfill_task.assert_called_once() + _, kwargs = mock_backfill_task.call_args + assert kwargs["scan_id"] == scan_id + + def test_compliance_overview_list_uses_preaggregated_summaries( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + mock_backfill_task, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + scan = Scan.objects.create( + name="preaggregated-scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + ComplianceRequirementOverview.objects.create( + tenant=tenant, + scan=scan, + compliance_id="cis_1.4_aws", + framework="CIS-1.4-AWS", + version="1.4", + description="CIS AWS Foundations Benchmark v1.4.0", + region="eu-west-1", + requirement_id="framework-metadata", + requirement_status=StatusChoices.PASS, + passed_checks=1, + failed_checks=0, + total_checks=1, + ) + + ComplianceOverviewSummary.objects.create( + tenant=tenant, + scan=scan, + compliance_id="cis_1.4_aws", + requirements_passed=5, + requirements_failed=1, + requirements_manual=2, + total_requirements=8, + ) + + response = authenticated_client.get( + reverse("complianceoverview-list"), + {"filter[scan_id]": str(scan.id)}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + overview = data[0] + assert overview["id"] == "cis_1.4_aws" + assert overview["attributes"]["requirements_passed"] == 5 + assert overview["attributes"]["requirements_failed"] == 1 + assert overview["attributes"]["requirements_manual"] == 2 + assert overview["attributes"]["total_requirements"] == 8 + assert "framework" in overview["attributes"] + assert "version" in overview["attributes"] + mock_backfill_task.assert_not_called() + + def test_compliance_overview_region_filter_skips_backfill( + self, + authenticated_client, + compliance_requirements_overviews_fixture, + mock_backfill_task, + ): + requirement_overview = compliance_requirements_overviews_fixture[0] + scan_id = str(requirement_overview.scan.id) + + response = authenticated_client.get( + reverse("complianceoverview-list"), + { + "filter[scan_id]": scan_id, + "filter[region]": requirement_overview.region, + }, + ) + + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) >= 1 + mock_backfill_task.assert_not_called() def test_compliance_overview_metadata( self, authenticated_client, compliance_requirements_overviews_fixture @@ -6429,6 +6742,11 @@ class TestComplianceOverviewViewSet: requirement_overview1 = compliance_requirements_overviews_fixture[0] scan_id = str(requirement_overview1.scan.id) + # Remove existing compliance data so the view falls back to task checks + scan = requirement_overview1.scan + ComplianceOverviewSummary.objects.filter(scan=scan).delete() + ComplianceRequirementOverview.objects.filter(scan=scan).delete() + # Mock a running task with patch.object( ComplianceOverviewViewSet, "get_task_response_if_running" @@ -6456,6 +6774,11 @@ class TestComplianceOverviewViewSet: requirement_overview1 = compliance_requirements_overviews_fixture[0] scan_id = str(requirement_overview1.scan.id) + # Remove existing compliance data so the view falls back to task checks + scan = requirement_overview1.scan + ComplianceOverviewSummary.objects.filter(scan=scan).delete() + ComplianceRequirementOverview.objects.filter(scan=scan).delete() + # Mock a failed task with patch.object( ComplianceOverviewViewSet, "get_task_response_if_running" @@ -6479,6 +6802,8 @@ class TestComplianceOverviewViewSet: ("framework", "framework", 1), ("version", "version", 1), ("region", "region", 1), + ("region__in", "region", 1), + ("region.in", "region", 1), ], ) def test_compliance_overview_filters( @@ -7274,6 +7599,925 @@ class TestOverviewViewSet: assert combined_attributes["medium"] == 4 assert combined_attributes["critical"] == 3 + def test_overview_findings_severity_timeseries_requires_date_from( + self, authenticated_client + ): + response = authenticated_client.get( + reverse("overview-findings_severity_timeseries") + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "date_from" in response.json()["errors"][0]["source"]["pointer"] + + def test_overview_findings_severity_timeseries_invalid_date_format( + self, authenticated_client + ): + response = authenticated_client.get( + reverse("overview-findings_severity_timeseries"), + {"filter[date_from]": "invalid-date"}, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Enter a valid date." in response.json()["errors"][0]["detail"] + + def test_overview_findings_severity_timeseries_empty_data( + self, authenticated_client + ): + response = authenticated_client.get( + reverse("overview-findings_severity_timeseries"), + { + "filter[date_from]": "2024-01-01", + "filter[date_to]": "2024-01-03", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + # Should return 3 days with fill-forward (all zeros since no data) + assert len(data) == 3 + for item in data: + assert item["attributes"]["critical"] == 0 + assert item["attributes"]["high"] == 0 + assert item["attributes"]["medium"] == 0 + assert item["attributes"]["low"] == 0 + assert item["attributes"]["informational"] == 0 + assert item["attributes"]["muted"] == 0 + assert item["attributes"]["scan_ids"] == [] + + def test_overview_findings_severity_timeseries_with_data( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + # Create scan for day 1 + scan1 = Scan.objects.create( + name="severity-over-time-scan-1", + provider=provider1, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + completed_at=datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + ) + + # Create scan for day 3 + scan3 = Scan.objects.create( + name="severity-over-time-scan-3", + provider=provider1, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + completed_at=datetime(2024, 1, 3, 12, 0, 0, tzinfo=timezone.utc), + ) + + # Create DailySeveritySummary for day 1 + DailySeveritySummary.objects.create( + tenant=tenant, + provider=provider1, + scan=scan1, + date=date(2024, 1, 1), + critical=10, + high=20, + medium=30, + low=40, + informational=50, + muted=5, + ) + + # Create DailySeveritySummary for day 3 + DailySeveritySummary.objects.create( + tenant=tenant, + provider=provider1, + scan=scan3, + date=date(2024, 1, 3), + critical=15, + high=25, + medium=35, + low=45, + informational=55, + muted=10, + ) + + response = authenticated_client.get( + reverse("overview-findings_severity_timeseries"), + { + "filter[date_from]": "2024-01-01", + "filter[date_to]": "2024-01-03", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 3 + + # Day 1 - actual data (id is the date) + assert data[0]["id"] == "2024-01-01" + assert data[0]["attributes"]["critical"] == 10 + assert data[0]["attributes"]["high"] == 20 + assert data[0]["attributes"]["scan_ids"] == [str(scan1.id)] + + # Day 2 - fill forward from day 1 (no data for this day) + assert data[1]["id"] == "2024-01-02" + assert data[1]["attributes"]["critical"] == 10 + assert data[1]["attributes"]["high"] == 20 + assert data[1]["attributes"]["scan_ids"] == [str(scan1.id)] + + # Day 3 - actual data + assert data[2]["id"] == "2024-01-03" + assert data[2]["attributes"]["critical"] == 15 + assert data[2]["attributes"]["high"] == 25 + assert data[2]["attributes"]["scan_ids"] == [str(scan3.id)] + + def test_overview_findings_severity_timeseries_aggregates_providers( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + # Same day, different providers + scan1 = Scan.objects.create( + name="severity-over-time-scan-p1", + provider=provider1, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + completed_at=datetime(2024, 2, 1, 12, 0, 0, tzinfo=timezone.utc), + ) + scan2 = Scan.objects.create( + name="severity-over-time-scan-p2", + provider=provider2, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + completed_at=datetime(2024, 2, 1, 14, 0, 0, tzinfo=timezone.utc), + ) + + # Create DailySeveritySummary for provider1 + DailySeveritySummary.objects.create( + tenant=tenant, + provider=provider1, + scan=scan1, + date=date(2024, 2, 1), + critical=10, + high=20, + medium=30, + low=40, + informational=50, + muted=5, + ) + + # Create DailySeveritySummary for provider2 + DailySeveritySummary.objects.create( + tenant=tenant, + provider=provider2, + scan=scan2, + date=date(2024, 2, 1), + critical=5, + high=10, + medium=15, + low=20, + informational=25, + muted=3, + ) + + response = authenticated_client.get( + reverse("overview-findings_severity_timeseries"), + { + "filter[date_from]": "2024-02-01", + "filter[date_to]": "2024-02-01", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + + # Should aggregate both providers + assert data[0]["attributes"]["critical"] == 15 # 10 + 5 + assert data[0]["attributes"]["high"] == 30 # 20 + 10 + assert data[0]["attributes"]["medium"] == 45 # 30 + 15 + assert data[0]["attributes"]["low"] == 60 # 40 + 20 + assert data[0]["attributes"]["informational"] == 75 # 50 + 25 + assert data[0]["attributes"]["muted"] == 8 # 5 + 3 + # scan_ids should contain both scans (order may vary) + assert set(data[0]["attributes"]["scan_ids"]) == {str(scan1.id), str(scan2.id)} + + def test_overview_findings_severity_timeseries_provider_filter( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = Scan.objects.create( + name="severity-over-time-filter-scan-p1", + provider=provider1, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + completed_at=datetime(2024, 3, 1, 12, 0, 0, tzinfo=timezone.utc), + ) + scan2 = Scan.objects.create( + name="severity-over-time-filter-scan-p2", + provider=provider2, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + completed_at=datetime(2024, 3, 1, 14, 0, 0, tzinfo=timezone.utc), + ) + + # Provider 1 - critical=100 + DailySeveritySummary.objects.create( + tenant=tenant, + provider=provider1, + scan=scan1, + date=date(2024, 3, 1), + critical=100, + high=0, + medium=0, + low=0, + informational=0, + muted=0, + ) + + # Provider 2 - critical=50 + DailySeveritySummary.objects.create( + tenant=tenant, + provider=provider2, + scan=scan2, + date=date(2024, 3, 1), + critical=50, + high=0, + medium=0, + low=0, + informational=0, + muted=0, + ) + + # Filter by provider1 only + response = authenticated_client.get( + reverse("overview-findings_severity_timeseries"), + { + "filter[date_from]": "2024-03-01", + "filter[date_to]": "2024-03-01", + "filter[provider_id]": str(provider1.id), + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["attributes"]["critical"] == 100 # Only provider1 + assert data[0]["attributes"]["scan_ids"] == [str(scan1.id)] + + def test_overview_attack_surface_no_data(self, authenticated_client): + response = authenticated_client.get(reverse("overview-attack-surface")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 4 + for item in data: + assert item["attributes"]["total_findings"] == 0 + assert item["attributes"]["failed_findings"] == 0 + assert item["attributes"]["muted_failed_findings"] == 0 + + def test_overview_attack_surface_with_data( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + create_attack_surface_overview, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + + scan = Scan.objects.create( + name="attack-surface-scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + create_attack_surface_overview( + tenant, + scan, + AttackSurfaceOverview.AttackSurfaceTypeChoices.INTERNET_EXPOSED, + total=20, + failed=10, + muted_failed=3, + ) + create_attack_surface_overview( + tenant, + scan, + AttackSurfaceOverview.AttackSurfaceTypeChoices.SECRETS, + total=15, + failed=8, + muted_failed=2, + ) + + response = authenticated_client.get(reverse("overview-attack-surface")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 4 + + results_by_type = {item["id"]: item["attributes"] for item in data} + assert results_by_type["internet-exposed"]["total_findings"] == 20 + assert results_by_type["internet-exposed"]["failed_findings"] == 10 + assert results_by_type["secrets"]["total_findings"] == 15 + assert results_by_type["secrets"]["failed_findings"] == 8 + assert results_by_type["privilege-escalation"]["total_findings"] == 0 + assert results_by_type["ec2-imdsv1"]["total_findings"] == 0 + + def test_overview_attack_surface_provider_filter( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + create_attack_surface_overview, + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = Scan.objects.create( + name="attack-surface-scan-1", + provider=provider1, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + scan2 = Scan.objects.create( + name="attack-surface-scan-2", + provider=provider2, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + create_attack_surface_overview( + tenant, + scan1, + AttackSurfaceOverview.AttackSurfaceTypeChoices.INTERNET_EXPOSED, + total=10, + failed=5, + muted_failed=1, + ) + create_attack_surface_overview( + tenant, + scan2, + AttackSurfaceOverview.AttackSurfaceTypeChoices.INTERNET_EXPOSED, + total=20, + failed=15, + muted_failed=3, + ) + + response = authenticated_client.get( + reverse("overview-attack-surface"), + {"filter[provider_id]": str(provider1.id)}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + results_by_type = {item["id"]: item["attributes"] for item in data} + assert results_by_type["internet-exposed"]["total_findings"] == 10 + assert results_by_type["internet-exposed"]["failed_findings"] == 5 + + def test_overview_services_region_filter( + self, authenticated_client, scan_summaries_fixture + ): + response = authenticated_client.get( + reverse("overview-services"), + {"filter[region]": "region1"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + service_ids = {item["id"] for item in data} + assert service_ids == {"service1", "service2"} + + def test_overview_services_provider_type_filter( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + aws_provider, _, gcp_provider, *_ = providers_fixture + + aws_scan = Scan.objects.create( + name="aws-scan", + provider=aws_provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + gcp_scan = Scan.objects.create( + name="gcp-scan", + provider=gcp_provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + ScanSummary.objects.create( + tenant=tenant, + scan=aws_scan, + check_id="aws-check", + service="aws-service", + severity="high", + region="us-east-1", + _pass=5, + fail=2, + muted=1, + total=8, + ) + ScanSummary.objects.create( + tenant=tenant, + scan=gcp_scan, + check_id="gcp-check", + service="gcp-service", + severity="medium", + region="us-central1", + _pass=3, + fail=1, + muted=0, + total=4, + ) + + response = authenticated_client.get( + reverse("overview-services"), + {"filter[provider_type]": "aws"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + service_ids = [item["id"] for item in data] + assert "aws-service" in service_ids + assert "gcp-service" not in service_ids + + @pytest.mark.parametrize( + "status_filter,field_to_check", + [ + ("FAIL", "fail"), + ("PASS", "_pass"), + ], + ) + def test_overview_findings_severity_status_filter( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + status_filter, + field_to_check, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + + scan = Scan.objects.create( + name="status-filter-scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + ScanSummary.objects.create( + tenant=tenant, + scan=scan, + check_id="status-check-high", + service="service-a", + severity="high", + region="us-east-1", + _pass=10, + fail=5, + muted=3, + total=18, + ) + ScanSummary.objects.create( + tenant=tenant, + scan=scan, + check_id="status-check-medium", + service="service-a", + severity="medium", + region="us-east-1", + _pass=8, + fail=2, + muted=1, + total=11, + ) + + response = authenticated_client.get( + reverse("overview-findings_severity"), + { + "filter[provider_id]": str(provider.id), + "filter[status]": status_filter, + }, + ) + assert response.status_code == status.HTTP_200_OK + attrs = response.json()["data"]["attributes"] + if status_filter == "FAIL": + assert attrs["high"] == 5 + assert attrs["medium"] == 2 + else: + assert attrs["high"] == 10 + assert attrs["medium"] == 8 + + def test_overview_threatscore_compliance_id_filter( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + scan = self._create_scan(tenant, provider, "compliance-filter-scan") + + self._create_threatscore_snapshot( + tenant, + scan, + provider, + compliance_id="prowler_threatscore_aws", + overall_score="75.00", + score_delta="2.00", + section_scores={"1. IAM": "70.00"}, + critical_requirements=[], + total_requirements=50, + passed_requirements=35, + failed_requirements=15, + manual_requirements=0, + total_findings=30, + passed_findings=20, + failed_findings=10, + ) + self._create_threatscore_snapshot( + tenant, + scan, + provider, + compliance_id="cis_1.4_aws", + overall_score="65.00", + score_delta="1.00", + section_scores={"1. IAM": "60.00"}, + critical_requirements=[], + total_requirements=40, + passed_requirements=25, + failed_requirements=15, + manual_requirements=0, + total_findings=25, + passed_findings=15, + failed_findings=10, + ) + + response = authenticated_client.get( + reverse("overview-threatscore"), + {"filter[compliance_id]": "prowler_threatscore_aws"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["attributes"]["overall_score"] == "75.00" + assert data[0]["attributes"]["compliance_id"] == "prowler_threatscore_aws" + + def test_overview_threatscore_provider_type_filter( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + aws_provider, _, gcp_provider, *_ = providers_fixture + + aws_scan = self._create_scan(tenant, aws_provider, "aws-threatscore-scan") + gcp_scan = self._create_scan(tenant, gcp_provider, "gcp-threatscore-scan") + + self._create_threatscore_snapshot( + tenant, + aws_scan, + aws_provider, + compliance_id="prowler_threatscore_aws", + overall_score="80.00", + score_delta="3.00", + section_scores={"1. IAM": "75.00"}, + critical_requirements=[], + total_requirements=60, + passed_requirements=45, + failed_requirements=15, + manual_requirements=0, + total_findings=40, + passed_findings=30, + failed_findings=10, + ) + self._create_threatscore_snapshot( + tenant, + gcp_scan, + gcp_provider, + compliance_id="prowler_threatscore_gcp", + overall_score="70.00", + score_delta="2.00", + section_scores={"1. IAM": "65.00"}, + critical_requirements=[], + total_requirements=50, + passed_requirements=35, + failed_requirements=15, + manual_requirements=0, + total_findings=35, + passed_findings=25, + failed_findings=10, + ) + + response = authenticated_client.get( + reverse("overview-threatscore"), + {"filter[provider_type]": "aws"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["attributes"]["overall_score"] == "80.00" + + def test_overview_categories_no_data(self, authenticated_client): + response = authenticated_client.get(reverse("overview-categories")) + assert response.status_code == status.HTTP_200_OK + assert response.json()["data"] == [] + + def test_overview_categories_aggregates_by_category_with_severity( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + create_scan_category_summary, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + + scan = Scan.objects.create( + name="categories-scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + create_scan_category_summary( + tenant, + scan, + "iam", + "high", + total_findings=20, + failed_findings=10, + new_failed_findings=5, + ) + create_scan_category_summary( + tenant, + scan, + "iam", + "medium", + total_findings=15, + failed_findings=8, + new_failed_findings=3, + ) + create_scan_category_summary( + tenant, + scan, + "encryption", + "critical", + total_findings=5, + failed_findings=2, + new_failed_findings=1, + ) + + response = authenticated_client.get(reverse("overview-categories")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + + results_by_category = {item["id"]: item["attributes"] for item in data} + + assert results_by_category["iam"]["total_findings"] == 35 + assert results_by_category["iam"]["failed_findings"] == 18 + assert results_by_category["iam"]["new_failed_findings"] == 8 + assert results_by_category["iam"]["severity"]["high"] == 10 + assert results_by_category["iam"]["severity"]["medium"] == 8 + assert results_by_category["iam"]["severity"]["critical"] == 0 + + assert results_by_category["encryption"]["total_findings"] == 5 + assert results_by_category["encryption"]["failed_findings"] == 2 + assert results_by_category["encryption"]["severity"]["critical"] == 2 + + @pytest.mark.parametrize( + "filter_key,filter_value_fn,expected_total,expected_failed", + [ + ("filter[provider_id]", lambda p1, _: str(p1.id), 10, 5), + ("filter[provider_type]", lambda *_: "aws", 10, 5), + ("filter[provider_type__in]", lambda *_: "aws,gcp", 30, 20), + ], + ) + def test_overview_categories_filters( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + create_scan_category_summary, + filter_key, + filter_value_fn, + expected_total, + expected_failed, + ): + tenant = tenants_fixture[0] + provider1, _, gcp_provider, *_ = providers_fixture + + scan1 = Scan.objects.create( + name="categories-scan-1", + provider=provider1, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + scan2 = Scan.objects.create( + name="categories-scan-2", + provider=gcp_provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + create_scan_category_summary( + tenant, scan1, "iam", "high", total_findings=10, failed_findings=5 + ) + create_scan_category_summary( + tenant, scan2, "iam", "high", total_findings=20, failed_findings=15 + ) + + response = authenticated_client.get( + reverse("overview-categories"), + {filter_key: filter_value_fn(provider1, gcp_provider)}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["attributes"]["total_findings"] == expected_total + assert data[0]["attributes"]["failed_findings"] == expected_failed + + def test_overview_categories_category_filter( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + create_scan_category_summary, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + + scan = Scan.objects.create( + name="category-filter-scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + create_scan_category_summary( + tenant, scan, "iam", "high", total_findings=10, failed_findings=5 + ) + create_scan_category_summary( + tenant, scan, "encryption", "medium", total_findings=20, failed_findings=8 + ) + create_scan_category_summary( + tenant, scan, "logging", "low", total_findings=15, failed_findings=3 + ) + + response = authenticated_client.get( + reverse("overview-categories"), + {"filter[category__in]": "iam,encryption"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + category_ids = {item["id"] for item in data} + assert category_ids == {"iam", "encryption"} + + def test_overview_categories_aggregates_multiple_providers( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + create_scan_category_summary, + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = Scan.objects.create( + name="multi-provider-scan-1", + provider=provider1, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + scan2 = Scan.objects.create( + name="multi-provider-scan-2", + provider=provider2, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + create_scan_category_summary( + tenant, + scan1, + "iam", + "high", + total_findings=10, + failed_findings=5, + new_failed_findings=2, + ) + create_scan_category_summary( + tenant, + scan2, + "iam", + "high", + total_findings=15, + failed_findings=8, + new_failed_findings=3, + ) + + response = authenticated_client.get(reverse("overview-categories")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["id"] == "iam" + assert data[0]["attributes"]["total_findings"] == 25 + assert data[0]["attributes"]["failed_findings"] == 13 + assert data[0]["attributes"]["new_failed_findings"] == 5 + + def test_compliance_watchlist_no_filters_uses_tenant_summary( + self, authenticated_client, tenant_compliance_summary_fixture + ): + response = authenticated_client.get(reverse("overview-compliance-watchlist")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + + assert len(data) == 2 + + by_id = {item["id"]: item["attributes"] for item in data} + assert "aws_cis_2.0" in by_id + assert by_id["aws_cis_2.0"]["requirements_passed"] == 1 + assert by_id["aws_cis_2.0"]["requirements_failed"] == 2 + assert by_id["aws_cis_2.0"]["requirements_manual"] == 1 + assert by_id["aws_cis_2.0"]["total_requirements"] == 4 + + assert "gdpr_aws" in by_id + assert by_id["gdpr_aws"]["requirements_passed"] == 5 + assert by_id["gdpr_aws"]["requirements_failed"] == 0 + assert by_id["gdpr_aws"]["total_requirements"] == 7 + + def test_compliance_watchlist_with_provider_filter_uses_provider_scores( + self, + authenticated_client, + provider_compliance_scores_fixture, + providers_fixture, + ): + provider1 = providers_fixture[0] + url = f"{reverse('overview-compliance-watchlist')}?filter[provider_id]={provider1.id}" + response = authenticated_client.get(url) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + + assert len(data) == 2 + by_id = {item["id"]: item["attributes"] for item in data} + + assert by_id["aws_cis_2.0"]["requirements_passed"] == 1 + assert by_id["aws_cis_2.0"]["requirements_failed"] == 1 + assert by_id["aws_cis_2.0"]["requirements_manual"] == 1 + assert by_id["aws_cis_2.0"]["total_requirements"] == 3 + + def test_compliance_watchlist_fail_dominant_logic( + self, authenticated_client, provider_compliance_scores_fixture + ): + response = authenticated_client.get( + f"{reverse('overview-compliance-watchlist')}?filter[provider_type]=aws" + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + + by_id = {item["id"]: item["attributes"] for item in data} + aws_cis = by_id["aws_cis_2.0"] + + assert aws_cis["requirements_failed"] == 2 + assert aws_cis["requirements_passed"] == 0 + assert aws_cis["requirements_manual"] == 1 + assert aws_cis["total_requirements"] == 3 + + def test_compliance_watchlist_provider_id_in_filter( + self, + authenticated_client, + provider_compliance_scores_fixture, + providers_fixture, + ): + provider1, provider2, *_ = providers_fixture + url = ( + f"{reverse('overview-compliance-watchlist')}" + f"?filter[provider_id__in]={provider1.id},{provider2.id}" + ) + response = authenticated_client.get(url) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) >= 1 + + def test_compliance_watchlist_empty_result(self, authenticated_client): + response = authenticated_client.get(reverse("overview-compliance-watchlist")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data == [] + + @pytest.mark.parametrize( + "invalid_provider_type", + ["invalid", "not_a_provider", "AWS", "awss"], + ) + def test_compliance_watchlist_invalid_provider_type_filter( + self, authenticated_client, invalid_provider_type + ): + url = f"{reverse('overview-compliance-watchlist')}?filter[provider_type]={invalid_provider_type}" + response = authenticated_client.get(url) + assert response.status_code == status.HTTP_400_BAD_REQUEST + @pytest.mark.django_db class TestScheduleViewSet: @@ -10477,6 +11721,540 @@ class TestLighthouseProviderConfigViewSet: # Unrelated entries should remain untouched assert cfg.default_models.get("other") == "model-x" + @pytest.mark.parametrize( + "credentials", + [ + {}, # empty credentials + { + "access_key_id": "AKIAIOSFODNN7EXAMPLE" + }, # missing secret_access_key and region + { + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + }, # missing access_key_id and region + { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, # missing region + { # invalid access_key_id format (not starting with AKIA) + "access_key_id": "ABCD0123456789ABCDEF", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + }, + { # invalid access_key_id format (wrong length) + "access_key_id": "AKIAIOSFODNN7EXAMPL", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + }, + { # invalid secret_access_key format (wrong length) + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEK", + "region": "us-east-1", + }, + { # invalid region format + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "invalid-region", + }, + { # invalid region format (uppercase) + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "US-EAST-1", + }, + ], + ) + def test_bedrock_invalid_credentials(self, authenticated_client, credentials): + """Bedrock provider with invalid credentials should error""" + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": credentials, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + + def test_bedrock_valid_credentials_success(self, authenticated_client): + """Bedrock provider with valid AWS credentials should succeed and mask credentials""" + valid_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + } + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": valid_credentials, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_201_CREATED + data = resp.json()["data"] + + # Verify credentials are returned masked + masked_creds = data["attributes"].get("credentials") + assert masked_creds is not None + assert "access_key_id" in masked_creds + assert "secret_access_key" in masked_creds + assert "region" in masked_creds + # Verify all characters are masked with asterisks + assert all(c == "*" for c in masked_creds["access_key_id"]) + assert all(c == "*" for c in masked_creds["secret_access_key"]) + + def test_bedrock_provider_duplicate_per_tenant(self, authenticated_client): + """Creating a second Bedrock provider for same tenant should fail""" + valid_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-west-2", + } + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": valid_credentials, + }, + } + } + # First creation succeeds + resp1 = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp1.status_code == status.HTTP_201_CREATED + + # Second creation should fail with validation error + resp2 = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp2.status_code == status.HTTP_400_BAD_REQUEST + assert "already exists" in str(resp2.json()).lower() + + def test_bedrock_patch_credentials_and_fields_filter(self, authenticated_client): + """PATCH credentials and verify fields filter returns decrypted values""" + valid_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "eu-west-1", + } + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": valid_credentials, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Update credentials with new valid ones + new_credentials = { + "access_key_id": "AKIAZZZZZZZZZZZZZZZZ", + "secret_access_key": "aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789+/==", + "region": "ap-south-1", + } + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "credentials": new_credentials, + "is_active": False, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + updated = patch_resp.json()["data"]["attributes"] + assert updated["is_active"] is False + + # Default GET should return masked credentials + get_resp = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + ) + assert get_resp.status_code == status.HTTP_200_OK + masked = get_resp.json()["data"]["attributes"]["credentials"] + assert all(c == "*" for c in masked["access_key_id"]) + assert all(c == "*" for c in masked["secret_access_key"]) + + # Fields filter should return decrypted credentials + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + assert creds["access_key_id"] == new_credentials["access_key_id"] + assert creds["secret_access_key"] == new_credentials["secret_access_key"] + assert creds["region"] == new_credentials["region"] + + def test_bedrock_partial_credential_update(self, authenticated_client): + """Test partial update of Bedrock credentials (e.g., only region)""" + # Create provider with full credentials + initial_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + } + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": initial_credentials, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Update only the region field + partial_update = { + "region": "eu-west-1", + } + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "credentials": partial_update, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + + # Verify credentials with fields filter - region should be updated, keys preserved + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + + # Original keys should be preserved + assert creds["access_key_id"] == initial_credentials["access_key_id"] + assert creds["secret_access_key"] == initial_credentials["secret_access_key"] + # Region should be updated + assert creds["region"] == "eu-west-1" + + def test_bedrock_valid_api_key_credentials_success(self, authenticated_client): + """Bedrock provider with valid API key + region should succeed and return masked credentials""" + valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110) + api_credentials = { + "api_key": valid_api_key, + "region": "us-east-1", + } + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": api_credentials, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_201_CREATED + data = resp.json()["data"] + + # Verify credentials are returned masked + masked_creds = data["attributes"].get("credentials") + assert masked_creds is not None + assert "api_key" in masked_creds + assert "region" in masked_creds + assert all(c == "*" for c in masked_creds["api_key"]) + + def test_bedrock_mixed_api_key_and_access_keys_invalid_on_create( + self, authenticated_client + ): + """Bedrock provider with both API key and access keys should fail validation on create""" + valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110) + mixed_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "api_key": valid_api_key, + "region": "us-east-1", + } + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": mixed_credentials, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + error_body = str(resp.json()).lower() + assert "either access key + secret key or api key" in error_body + + def test_bedrock_cannot_switch_from_api_key_to_access_keys_on_update( + self, authenticated_client + ): + """If created with API key, switching to access keys via update should be rejected""" + valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110) + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": { + "api_key": valid_api_key, + "region": "us-east-1", + }, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Attempt to introduce access keys on update + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "credentials": { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_400_BAD_REQUEST + error_body = str(patch_resp.json()).lower() + assert "cannot change bedrock authentication method from api key" in error_body + + def test_bedrock_cannot_switch_from_access_keys_to_api_key_on_update( + self, authenticated_client + ): + """If created with access keys, switching to API key via update should be rejected""" + valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110) + initial_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + } + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": initial_credentials, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Attempt to introduce API key on update + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "credentials": { + "api_key": valid_api_key, + }, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_400_BAD_REQUEST + error_body = str(patch_resp.json()).lower() + assert ( + "cannot change bedrock authentication method from access key" in error_body + ) + + @pytest.mark.parametrize( + "attributes", + [ + pytest.param( + { + "provider_type": "openai_compatible", + "credentials": {"api_key": "compat-key"}, + }, + id="missing", + ), + pytest.param( + { + "provider_type": "openai_compatible", + "credentials": {"api_key": "compat-key"}, + "base_url": "", + }, + id="empty", + ), + ], + ) + def test_openai_compatible_missing_base_url(self, authenticated_client, attributes): + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": attributes, + } + } + + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + error_detail = str(resp.json()).lower() + assert "base_url" in error_detail + + def test_openai_compatible_invalid_credentials(self, authenticated_client): + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai_compatible", + "base_url": "https://compat.example/v1", + "credentials": {"api_key": ""}, + }, + } + } + + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + errors = resp.json().get("errors", []) + assert any( + error.get("source", {}).get("pointer") + == "/data/attributes/credentials/api_key" + for error in errors + ) + assert any( + "may not be blank" in error.get("detail", "").lower() for error in errors + ) + + def test_openai_compatible_patch_credentials_and_fields(self, authenticated_client): + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai_compatible", + "base_url": "https://compat.example/v1", + "credentials": {"api_key": "compat-key-123"}, + }, + } + } + + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + updated_base_url = "https://compat.example/v2" + updated_api_key = "compat-key-456" + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "base_url": updated_base_url, + "credentials": {"api_key": updated_api_key}, + }, + } + } + + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + updated_attrs = patch_resp.json()["data"]["attributes"] + assert updated_attrs["base_url"] == updated_base_url + assert updated_attrs["credentials"]["api_key"] == "*" * len(updated_api_key) + + get_resp = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + ) + assert get_resp.status_code == status.HTTP_200_OK + masked = get_resp.json()["data"]["attributes"]["credentials"]["api_key"] + assert masked == "*" * len(updated_api_key) + + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + assert creds["api_key"] == updated_api_key + @pytest.mark.django_db class TestMuteRuleViewSet: @@ -11030,380 +12808,3 @@ class TestMuteRuleViewSet: assert len(data) == len(mute_rules_fixture) for rule_data in data: assert rule_data["id"] != str(other_rule.id) - - @pytest.mark.parametrize( - "credentials", - [ - {}, # empty credentials - { - "access_key_id": "AKIAIOSFODNN7EXAMPLE" - }, # missing secret_access_key and region - { - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - }, # missing access_key_id and region - { - "access_key_id": "AKIAIOSFODNN7EXAMPLE", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - }, # missing region - { # invalid access_key_id format (not starting with AKIA) - "access_key_id": "ABCD0123456789ABCDEF", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "region": "us-east-1", - }, - { # invalid access_key_id format (wrong length) - "access_key_id": "AKIAIOSFODNN7EXAMPL", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "region": "us-east-1", - }, - { # invalid secret_access_key format (wrong length) - "access_key_id": "AKIAIOSFODNN7EXAMPLE", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEK", - "region": "us-east-1", - }, - { # invalid region format - "access_key_id": "AKIAIOSFODNN7EXAMPLE", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "region": "invalid-region", - }, - { # invalid region format (uppercase) - "access_key_id": "AKIAIOSFODNN7EXAMPLE", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "region": "US-EAST-1", - }, - ], - ) - def test_bedrock_invalid_credentials(self, authenticated_client, credentials): - """Bedrock provider with invalid credentials should error""" - payload = { - "data": { - "type": "lighthouse-providers", - "attributes": { - "provider_type": "bedrock", - "credentials": credentials, - }, - } - } - resp = authenticated_client.post( - reverse("lighthouse-providers-list"), - data=payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert resp.status_code == status.HTTP_400_BAD_REQUEST - - def test_bedrock_valid_credentials_success(self, authenticated_client): - """Bedrock provider with valid AWS credentials should succeed and mask credentials""" - valid_credentials = { - "access_key_id": "AKIAIOSFODNN7EXAMPLE", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "region": "us-east-1", - } - payload = { - "data": { - "type": "lighthouse-providers", - "attributes": { - "provider_type": "bedrock", - "credentials": valid_credentials, - }, - } - } - resp = authenticated_client.post( - reverse("lighthouse-providers-list"), - data=payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert resp.status_code == status.HTTP_201_CREATED - data = resp.json()["data"] - - # Verify credentials are returned masked - masked_creds = data["attributes"].get("credentials") - assert masked_creds is not None - assert "access_key_id" in masked_creds - assert "secret_access_key" in masked_creds - assert "region" in masked_creds - # Verify all characters are masked with asterisks - assert all(c == "*" for c in masked_creds["access_key_id"]) - assert all(c == "*" for c in masked_creds["secret_access_key"]) - - def test_bedrock_provider_duplicate_per_tenant(self, authenticated_client): - """Creating a second Bedrock provider for same tenant should fail""" - valid_credentials = { - "access_key_id": "AKIAIOSFODNN7EXAMPLE", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "region": "us-west-2", - } - payload = { - "data": { - "type": "lighthouse-providers", - "attributes": { - "provider_type": "bedrock", - "credentials": valid_credentials, - }, - } - } - # First creation succeeds - resp1 = authenticated_client.post( - reverse("lighthouse-providers-list"), - data=payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert resp1.status_code == status.HTTP_201_CREATED - - # Second creation should fail with validation error - resp2 = authenticated_client.post( - reverse("lighthouse-providers-list"), - data=payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert resp2.status_code == status.HTTP_400_BAD_REQUEST - assert "already exists" in str(resp2.json()).lower() - - def test_bedrock_patch_credentials_and_fields_filter(self, authenticated_client): - """PATCH credentials and verify fields filter returns decrypted values""" - valid_credentials = { - "access_key_id": "AKIAIOSFODNN7EXAMPLE", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "region": "eu-west-1", - } - create_payload = { - "data": { - "type": "lighthouse-providers", - "attributes": { - "provider_type": "bedrock", - "credentials": valid_credentials, - }, - } - } - create_resp = authenticated_client.post( - reverse("lighthouse-providers-list"), - data=create_payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert create_resp.status_code == status.HTTP_201_CREATED - provider_id = create_resp.json()["data"]["id"] - - # Update credentials with new valid ones - new_credentials = { - "access_key_id": "AKIAZZZZZZZZZZZZZZZZ", - "secret_access_key": "aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789+/==", - "region": "ap-south-1", - } - patch_payload = { - "data": { - "type": "lighthouse-providers", - "id": provider_id, - "attributes": { - "credentials": new_credentials, - "is_active": False, - }, - } - } - patch_resp = authenticated_client.patch( - reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), - data=patch_payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert patch_resp.status_code == status.HTTP_200_OK - updated = patch_resp.json()["data"]["attributes"] - assert updated["is_active"] is False - - # Default GET should return masked credentials - get_resp = authenticated_client.get( - reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) - ) - assert get_resp.status_code == status.HTTP_200_OK - masked = get_resp.json()["data"]["attributes"]["credentials"] - assert all(c == "*" for c in masked["access_key_id"]) - assert all(c == "*" for c in masked["secret_access_key"]) - - # Fields filter should return decrypted credentials - get_full = authenticated_client.get( - reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) - + "?fields[lighthouse-providers]=credentials" - ) - assert get_full.status_code == status.HTTP_200_OK - creds = get_full.json()["data"]["attributes"]["credentials"] - assert creds["access_key_id"] == new_credentials["access_key_id"] - assert creds["secret_access_key"] == new_credentials["secret_access_key"] - assert creds["region"] == new_credentials["region"] - - def test_bedrock_partial_credential_update(self, authenticated_client): - """Test partial update of Bedrock credentials (e.g., only region)""" - # Create provider with full credentials - initial_credentials = { - "access_key_id": "AKIAIOSFODNN7EXAMPLE", - "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "region": "us-east-1", - } - create_payload = { - "data": { - "type": "lighthouse-providers", - "attributes": { - "provider_type": "bedrock", - "credentials": initial_credentials, - }, - } - } - create_resp = authenticated_client.post( - reverse("lighthouse-providers-list"), - data=create_payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert create_resp.status_code == status.HTTP_201_CREATED - provider_id = create_resp.json()["data"]["id"] - - # Update only the region field - partial_update = { - "region": "eu-west-1", - } - patch_payload = { - "data": { - "type": "lighthouse-providers", - "id": provider_id, - "attributes": { - "credentials": partial_update, - }, - } - } - patch_resp = authenticated_client.patch( - reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), - data=patch_payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert patch_resp.status_code == status.HTTP_200_OK - - # Verify credentials with fields filter - region should be updated, keys preserved - get_full = authenticated_client.get( - reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) - + "?fields[lighthouse-providers]=credentials" - ) - assert get_full.status_code == status.HTTP_200_OK - creds = get_full.json()["data"]["attributes"]["credentials"] - - # Original keys should be preserved - assert creds["access_key_id"] == initial_credentials["access_key_id"] - assert creds["secret_access_key"] == initial_credentials["secret_access_key"] - # Region should be updated - assert creds["region"] == "eu-west-1" - - @pytest.mark.parametrize( - "attributes", - [ - pytest.param( - { - "provider_type": "openai_compatible", - "credentials": {"api_key": "compat-key"}, - }, - id="missing", - ), - pytest.param( - { - "provider_type": "openai_compatible", - "credentials": {"api_key": "compat-key"}, - "base_url": "", - }, - id="empty", - ), - ], - ) - def test_openai_compatible_missing_base_url(self, authenticated_client, attributes): - payload = { - "data": { - "type": "lighthouse-providers", - "attributes": attributes, - } - } - - resp = authenticated_client.post( - reverse("lighthouse-providers-list"), - data=payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert resp.status_code == status.HTTP_400_BAD_REQUEST - error_detail = str(resp.json()).lower() - assert "base_url" in error_detail - - def test_openai_compatible_invalid_credentials(self, authenticated_client): - payload = { - "data": { - "type": "lighthouse-providers", - "attributes": { - "provider_type": "openai_compatible", - "base_url": "https://compat.example/v1", - "credentials": {"api_key": ""}, - }, - } - } - - resp = authenticated_client.post( - reverse("lighthouse-providers-list"), - data=payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert resp.status_code == status.HTTP_400_BAD_REQUEST - errors = resp.json().get("errors", []) - assert any( - error.get("source", {}).get("pointer") - == "/data/attributes/credentials/api_key" - for error in errors - ) - assert any( - "may not be blank" in error.get("detail", "").lower() for error in errors - ) - - def test_openai_compatible_patch_credentials_and_fields(self, authenticated_client): - create_payload = { - "data": { - "type": "lighthouse-providers", - "attributes": { - "provider_type": "openai_compatible", - "base_url": "https://compat.example/v1", - "credentials": {"api_key": "compat-key-123"}, - }, - } - } - - create_resp = authenticated_client.post( - reverse("lighthouse-providers-list"), - data=create_payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert create_resp.status_code == status.HTTP_201_CREATED - provider_id = create_resp.json()["data"]["id"] - - updated_base_url = "https://compat.example/v2" - updated_api_key = "compat-key-456" - patch_payload = { - "data": { - "type": "lighthouse-providers", - "id": provider_id, - "attributes": { - "base_url": updated_base_url, - "credentials": {"api_key": updated_api_key}, - }, - } - } - - patch_resp = authenticated_client.patch( - reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), - data=patch_payload, - content_type=API_JSON_CONTENT_TYPE, - ) - assert patch_resp.status_code == status.HTTP_200_OK - updated_attrs = patch_resp.json()["data"]["attributes"] - assert updated_attrs["base_url"] == updated_base_url - assert updated_attrs["credentials"]["api_key"] == "*" * len(updated_api_key) - - get_resp = authenticated_client.get( - reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) - ) - assert get_resp.status_code == status.HTTP_200_OK - masked = get_resp.json()["data"]["attributes"]["credentials"]["api_key"] - assert masked == "*" * len(updated_api_key) - - get_full = authenticated_client.get( - reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) - + "?fields[lighthouse-providers]=credentials" - ) - assert get_full.status_code == status.HTTP_200_OK - creds = get_full.json()["data"]["attributes"]["credentials"] - assert creds["api_key"] == updated_api_key diff --git a/api/src/backend/api/utils.py b/api/src/backend/api/utils.py index 866088f64a..bc203c1584 100644 --- a/api/src/backend/api/utils.py +++ b/api/src/backend/api/utils.py @@ -11,6 +11,7 @@ from api.exceptions import InvitationTokenExpiredException from api.models import Integration, Invitation, Processor, Provider, Resource from api.v1.serializers import FindingMetadataSerializer from prowler.lib.outputs.jira.jira import Jira, JiraBasicAuthError +from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider from prowler.providers.aws.aws_provider import AwsProvider from prowler.providers.aws.lib.s3.s3 import S3 from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub @@ -63,8 +64,9 @@ def merge_dicts(default_dict: dict, replacement_dict: dict) -> dict: def return_prowler_provider( provider: Provider, -) -> [ - AwsProvider +) -> ( + AlibabacloudProvider + | AwsProvider | AzureProvider | GcpProvider | GithubProvider @@ -73,14 +75,14 @@ def return_prowler_provider( | M365Provider | MongodbatlasProvider | OraclecloudProvider -]: +): """Return the Prowler provider class based on the given provider type. Args: provider (Provider): The provider object containing the provider type and associated secrets. Returns: - AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: The corresponding provider class. + AlibabacloudProvider | AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: The corresponding provider class. Raises: ValueError: If the provider type specified in `provider.provider` is not supported. @@ -104,6 +106,8 @@ def return_prowler_provider( prowler_provider = IacProvider case Provider.ProviderChoices.ORACLECLOUD.value: prowler_provider = OraclecloudProvider + case Provider.ProviderChoices.ALIBABACLOUD.value: + prowler_provider = AlibabacloudProvider case _: raise ValueError(f"Provider type {provider.provider} not supported") return prowler_provider @@ -169,7 +173,8 @@ def initialize_prowler_provider( provider: Provider, mutelist_processor: Processor | None = None, ) -> ( - AwsProvider + AlibabacloudProvider + | AwsProvider | AzureProvider | GcpProvider | GithubProvider @@ -186,9 +191,8 @@ def initialize_prowler_provider( mutelist_processor (Processor): The mutelist processor object containing the mutelist configuration. Returns: - AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: An instance of the corresponding provider class - (`AwsProvider`, `AzureProvider`, `GcpProvider`, `GithubProvider`, `IacProvider`, `KubernetesProvider`, `M365Provider`, `OraclecloudProvider` or `MongodbatlasProvider`) initialized with the - provider's secrets. + AlibabacloudProvider | AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: An instance of the corresponding provider class + initialized with the provider's secrets. """ prowler_provider = return_prowler_provider(provider) prowler_provider_kwargs = get_prowler_provider_kwargs(provider, mutelist_processor) @@ -382,10 +386,18 @@ def get_findings_metadata_no_aggregations(tenant_id: str, filtered_queryset): regions = sorted({region for region in aggregation["regions"] or [] if region}) resource_types = sorted(set(aggregation["resource_types"] or [])) + # Aggregate categories from findings + categories_set = set() + for categories_list in filtered_queryset.values_list("categories", flat=True): + if categories_list: + categories_set.update(categories_list) + categories = sorted(categories_set) + result = { "services": services, "regions": regions, "resource_types": resource_types, + "categories": categories, } serializer = FindingMetadataSerializer(data=result) diff --git a/api/src/backend/api/v1/serializer_utils/lighthouse.py b/api/src/backend/api/v1/serializer_utils/lighthouse.py index e29c679ce6..e0274a7bbb 100644 --- a/api/src/backend/api/v1/serializer_utils/lighthouse.py +++ b/api/src/backend/api/v1/serializer_utils/lighthouse.py @@ -40,11 +40,16 @@ class BedrockCredentialsSerializer(serializers.Serializer): """ Serializer for AWS Bedrock credentials validation. - Validates long-term AWS credentials (AKIA) and region format. + Supports two authentication methods: + 1. AWS access key + secret key + 2. Bedrock API key (bearer token) + + In both cases, region is mandatory. """ - access_key_id = serializers.CharField() - secret_access_key = serializers.CharField() + access_key_id = serializers.CharField(required=False, allow_blank=False) + secret_access_key = serializers.CharField(required=False, allow_blank=False) + api_key = serializers.CharField(required=False, allow_blank=False) region = serializers.CharField() def validate_access_key_id(self, value: str) -> str: @@ -65,6 +70,15 @@ class BedrockCredentialsSerializer(serializers.Serializer): ) return value + def validate_api_key(self, value: str) -> str: + """ + Validate Bedrock API key (bearer token). + """ + pattern = r"^ABSKQmVkcm9ja0FQSUtleS[A-Za-z0-9+/=]{110}$" + if not re.match(pattern, value or ""): + raise serializers.ValidationError("Invalid Bedrock API key format.") + return value + def validate_region(self, value: str) -> str: """Validate AWS region format.""" pattern = r"^[a-z]{2}-[a-z]+-\d+$" @@ -74,6 +88,50 @@ class BedrockCredentialsSerializer(serializers.Serializer): ) return value + def validate(self, attrs): + """ + Enforce either: + - access_key_id + secret_access_key + region + OR + - api_key + region + """ + access_key_id = attrs.get("access_key_id") + secret_access_key = attrs.get("secret_access_key") + api_key = attrs.get("api_key") + region = attrs.get("region") + + errors = {} + + if not region: + errors["region"] = ["Region is required."] + + using_access_keys = bool(access_key_id or secret_access_key) + using_api_key = api_key is not None and api_key != "" + + if using_access_keys and using_api_key: + errors["non_field_errors"] = [ + "Provide either access key + secret key OR api key, not both." + ] + elif not using_access_keys and not using_api_key: + errors["non_field_errors"] = [ + "You must provide either access key + secret key OR api key." + ] + elif using_access_keys: + # Both access_key_id and secret_access_key must be present together + if not access_key_id: + errors.setdefault("access_key_id", []).append( + "AWS access key ID is required when using access key authentication." + ) + if not secret_access_key: + errors.setdefault("secret_access_key", []).append( + "AWS secret access key is required when using access key authentication." + ) + + if errors: + raise serializers.ValidationError(errors) + + return attrs + def to_internal_value(self, data): """Check for unknown fields before DRF filters them out.""" if not isinstance(data, dict): @@ -111,6 +169,15 @@ class BedrockCredentialsUpdateSerializer(BedrockCredentialsSerializer): for field in self.fields.values(): field.required = False + def validate(self, attrs): + """ + For updates, this serializer only checks individual fields. + It does NOT enforce the "either access keys OR api key" rule. + That rule is applied later, after merging with existing stored + credentials, in LighthouseProviderConfigUpdateSerializer. + """ + return attrs + class OpenAICompatibleCredentialsSerializer(serializers.Serializer): """ @@ -168,27 +235,51 @@ class OpenAICompatibleCredentialsSerializer(serializers.Serializer): "required": ["api_key"], }, { - "type": "object", "title": "AWS Bedrock Credentials", - "properties": { - "access_key_id": { - "type": "string", - "description": "AWS access key ID.", - "pattern": "^AKIA[0-9A-Z]{16}$", + "oneOf": [ + { + "title": "IAM Access Key Pair", + "type": "object", + "description": "Authenticate with AWS access key and secret key. Recommended when you manage IAM users or roles.", + "properties": { + "access_key_id": { + "type": "string", + "description": "AWS access key ID.", + "pattern": "^AKIA[0-9A-Z]{16}$", + }, + "secret_access_key": { + "type": "string", + "description": "AWS secret access key.", + "pattern": "^[A-Za-z0-9/+=]{40}$", + }, + "region": { + "type": "string", + "description": "AWS region identifier where Bedrock is available. Examples: us-east-1, " + "us-west-2, eu-west-1, ap-northeast-1.", + "pattern": "^[a-z]{2}-[a-z]+-\\d+$", + }, + }, + "required": ["access_key_id", "secret_access_key", "region"], }, - "secret_access_key": { - "type": "string", - "description": "AWS secret access key.", - "pattern": "^[A-Za-z0-9/+=]{40}$", + { + "title": "Amazon Bedrock API Key", + "type": "object", + "description": "Authenticate with an Amazon Bedrock API key (bearer token). Region is still required.", + "properties": { + "api_key": { + "type": "string", + "description": "Amazon Bedrock API key (bearer token).", + }, + "region": { + "type": "string", + "description": "AWS region identifier where Bedrock is available. Examples: us-east-1, " + "us-west-2, eu-west-1, ap-northeast-1.", + "pattern": "^[a-z]{2}-[a-z]+-\\d+$", + }, + }, + "required": ["api_key", "region"], }, - "region": { - "type": "string", - "description": "AWS region identifier where Bedrock is available. Examples: us-east-1, " - "us-west-2, eu-west-1, ap-northeast-1.", - "pattern": "^[a-z]{2}-[a-z]+-\\d+$", - }, - }, - "required": ["access_key_id", "secret_access_key", "region"], + ], }, { "type": "object", diff --git a/api/src/backend/api/v1/serializer_utils/providers.py b/api/src/backend/api/v1/serializer_utils/providers.py index 4ec772e02f..3ed2a58ef3 100644 --- a/api/src/backend/api/v1/serializer_utils/providers.py +++ b/api/src/backend/api/v1/serializer_utils/providers.py @@ -304,6 +304,48 @@ from rest_framework_json_api import serializers }, "required": ["atlas_public_key", "atlas_private_key"], }, + { + "type": "object", + "title": "Alibaba Cloud Static Credentials", + "properties": { + "access_key_id": { + "type": "string", + "description": "The Alibaba Cloud access key ID for authentication.", + }, + "access_key_secret": { + "type": "string", + "description": "The Alibaba Cloud access key secret for authentication.", + }, + "security_token": { + "type": "string", + "description": "The STS security token for temporary credentials (optional).", + }, + }, + "required": ["access_key_id", "access_key_secret"], + }, + { + "type": "object", + "title": "Alibaba Cloud RAM Role Assumption", + "properties": { + "role_arn": { + "type": "string", + "description": "The ARN of the RAM role to assume (e.g., acs:ram::1234567890123456:role/ProwlerRole).", + }, + "access_key_id": { + "type": "string", + "description": "The Alibaba Cloud access key ID of the RAM user that will assume the role.", + }, + "access_key_secret": { + "type": "string", + "description": "The Alibaba Cloud access key secret of the RAM user that will assume the role.", + }, + "role_session_name": { + "type": "string", + "description": "An identifier for the role session (optional, defaults to 'ProwlerSession').", + }, + }, + "required": ["role_arn", "access_key_id", "access_key_secret"], + }, ] } ) diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index a23aa91519..9156638b87 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -73,6 +73,42 @@ from api.v1.serializer_utils.processors import ProcessorConfigField from api.v1.serializer_utils.providers import ProviderSecretField from prowler.lib.mutelist.mutelist import Mutelist +# Base + + +class BaseModelSerializerV1(serializers.ModelSerializer): + def get_root_meta(self, _resource, _many): + return {"version": "v1"} + + +class BaseSerializerV1(serializers.Serializer): + def get_root_meta(self, _resource, _many): + return {"version": "v1"} + + +class BaseWriteSerializer(BaseModelSerializerV1): + def validate(self, data): + if hasattr(self, "initial_data"): + initial_data = set(self.initial_data.keys()) - {"id", "type"} + unknown_keys = initial_data - set(self.fields.keys()) + if unknown_keys: + raise ValidationError(f"Invalid fields: {unknown_keys}") + return data + + +class RLSSerializer(BaseModelSerializerV1): + def create(self, validated_data): + tenant_id = self.context.get("tenant_id") + validated_data["tenant_id"] = tenant_id + return super().create(validated_data) + + +class StateEnumSerializerField(serializers.ChoiceField): + def __init__(self, **kwargs): + kwargs["choices"] = StateChoices.choices + super().__init__(**kwargs) + + # Tokens @@ -180,7 +216,7 @@ class TokenSocialLoginSerializer(BaseTokenSerializer): # TODO: Check if we can change the parent class to TokenRefreshSerializer from rest_framework_simplejwt.serializers -class TokenRefreshSerializer(serializers.Serializer): +class TokenRefreshSerializer(BaseSerializerV1): refresh = serializers.CharField() # Output token @@ -214,7 +250,7 @@ class TokenRefreshSerializer(serializers.Serializer): raise ValidationError({"refresh": "Invalid or expired token"}) -class TokenSwitchTenantSerializer(serializers.Serializer): +class TokenSwitchTenantSerializer(BaseSerializerV1): tenant_id = serializers.UUIDField( write_only=True, help_text="The tenant ID for which to request a new token." ) @@ -238,41 +274,10 @@ class TokenSwitchTenantSerializer(serializers.Serializer): return generate_tokens(user, tenant_id) -# Base - - -class BaseSerializerV1(serializers.ModelSerializer): - def get_root_meta(self, _resource, _many): - return {"version": "v1"} - - -class BaseWriteSerializer(BaseSerializerV1): - def validate(self, data): - if hasattr(self, "initial_data"): - initial_data = set(self.initial_data.keys()) - {"id", "type"} - unknown_keys = initial_data - set(self.fields.keys()) - if unknown_keys: - raise ValidationError(f"Invalid fields: {unknown_keys}") - return data - - -class RLSSerializer(BaseSerializerV1): - def create(self, validated_data): - tenant_id = self.context.get("tenant_id") - validated_data["tenant_id"] = tenant_id - return super().create(validated_data) - - -class StateEnumSerializerField(serializers.ChoiceField): - def __init__(self, **kwargs): - kwargs["choices"] = StateChoices.choices - super().__init__(**kwargs) - - # Users -class UserSerializer(BaseSerializerV1): +class UserSerializer(BaseModelSerializerV1): """ Serializer for the User model. """ @@ -403,7 +408,7 @@ class UserUpdateSerializer(BaseWriteSerializer): return super().update(instance, validated_data) -class RoleResourceIdentifierSerializer(serializers.Serializer): +class RoleResourceIdentifierSerializer(BaseSerializerV1): resource_type = serializers.CharField(source="type") id = serializers.UUIDField() @@ -586,7 +591,7 @@ class TaskSerializer(RLSSerializer, TaskBase): # Tenants -class TenantSerializer(BaseSerializerV1): +class TenantSerializer(BaseModelSerializerV1): """ Serializer for the Tenant model. """ @@ -598,7 +603,7 @@ class TenantSerializer(BaseSerializerV1): fields = ["id", "name", "memberships"] -class TenantIncludeSerializer(BaseSerializerV1): +class TenantIncludeSerializer(BaseModelSerializerV1): class Meta: model = Tenant fields = ["id", "name"] @@ -774,7 +779,7 @@ class ProviderGroupUpdateSerializer(ProviderGroupSerializer): return super().update(instance, validated_data) -class ProviderResourceIdentifierSerializer(serializers.Serializer): +class ProviderResourceIdentifierSerializer(BaseSerializerV1): resource_type = serializers.CharField(source="type") id = serializers.UUIDField() @@ -1111,7 +1116,7 @@ class ScanTaskSerializer(RLSSerializer): ] -class ScanReportSerializer(serializers.Serializer): +class ScanReportSerializer(BaseSerializerV1): id = serializers.CharField(source="scan") class Meta: @@ -1119,7 +1124,7 @@ class ScanReportSerializer(serializers.Serializer): fields = ["id"] -class ScanComplianceReportSerializer(serializers.Serializer): +class ScanComplianceReportSerializer(BaseSerializerV1): id = serializers.CharField(source="scan") name = serializers.CharField() @@ -1371,7 +1376,7 @@ class ResourceIncludeSerializer(RLSSerializer): return fields -class ResourceMetadataSerializer(serializers.Serializer): +class ResourceMetadataSerializer(BaseSerializerV1): services = serializers.ListField(child=serializers.CharField(), allow_empty=True) regions = serializers.ListField(child=serializers.CharField(), allow_empty=True) types = serializers.ListField(child=serializers.CharField(), allow_empty=True) @@ -1400,6 +1405,7 @@ class FindingSerializer(RLSSerializer): "severity", "check_id", "check_metadata", + "categories", "raw_result", "inserted_at", "updated_at", @@ -1441,7 +1447,7 @@ class FindingIncludeSerializer(RLSSerializer): # To be removed when the related endpoint is removed as well -class FindingDynamicFilterSerializer(serializers.Serializer): +class FindingDynamicFilterSerializer(BaseSerializerV1): services = serializers.ListField(child=serializers.CharField(), allow_empty=True) regions = serializers.ListField(child=serializers.CharField(), allow_empty=True) @@ -1449,12 +1455,13 @@ class FindingDynamicFilterSerializer(serializers.Serializer): resource_name = "finding-dynamic-filters" -class FindingMetadataSerializer(serializers.Serializer): +class FindingMetadataSerializer(BaseSerializerV1): services = serializers.ListField(child=serializers.CharField(), allow_empty=True) regions = serializers.ListField(child=serializers.CharField(), allow_empty=True) resource_types = serializers.ListField( child=serializers.CharField(), allow_empty=True ) + categories = serializers.ListField(child=serializers.CharField(), allow_empty=True) # Temporarily disabled until we implement tag filtering in the UI # tags = serializers.JSONField(help_text="Tags are described as key-value pairs.") @@ -1487,12 +1494,23 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer): serializer = OracleCloudProviderSecret(data=secret) elif provider_type == Provider.ProviderChoices.MONGODBATLAS.value: serializer = MongoDBAtlasProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.ALIBABACLOUD.value: + serializer = AlibabaCloudProviderSecret(data=secret) else: raise serializers.ValidationError( {"provider": f"Provider type not supported {provider_type}"} ) elif secret_type == ProviderSecret.TypeChoices.ROLE: - serializer = AWSRoleAssumptionProviderSecret(data=secret) + if provider_type == Provider.ProviderChoices.AWS.value: + serializer = AWSRoleAssumptionProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.ALIBABACLOUD.value: + serializer = AlibabaCloudRoleAssumptionProviderSecret(data=secret) + else: + raise serializers.ValidationError( + { + "secret_type": f"Role assumption not supported for provider type: {provider_type}" + } + ) elif secret_type == ProviderSecret.TypeChoices.SERVICE_ACCOUNT: serializer = GCPServiceAccountProviderSecret(data=secret) else: @@ -1629,6 +1647,34 @@ class OracleCloudProviderSecret(serializers.Serializer): resource_name = "provider-secrets" +class AlibabaCloudProviderSecret(serializers.Serializer): + access_key_id = serializers.CharField() + access_key_secret = serializers.CharField() + security_token = serializers.CharField(required=False) + + class Meta: + resource_name = "provider-secrets" + + +class AlibabaCloudRoleAssumptionProviderSecret(serializers.Serializer): + role_arn = serializers.CharField( + help_text="Access Key ID of the RAM user that will assume the role" + ) + access_key_id = serializers.CharField( + help_text="Access Key ID of the RAM user that will assume the role" + ) + access_key_secret = serializers.CharField( + help_text="Access Key Secret of the RAM user that will assume the role" + ) + role_session_name = serializers.CharField( + required=False, + help_text="Session name for the assumed role session (optional, defaults to 'ProwlerSession')", + ) + + class Meta: + resource_name = "provider-secrets" + + class AWSRoleAssumptionProviderSecret(serializers.Serializer): role_arn = serializers.CharField() external_id = serializers.CharField() @@ -2143,7 +2189,7 @@ class RoleProviderGroupRelationshipSerializer(RLSSerializer, BaseWriteSerializer # Compliance overview -class ComplianceOverviewSerializer(serializers.Serializer): +class ComplianceOverviewSerializer(BaseSerializerV1): """ Serializer for compliance requirement status aggregated by compliance framework. @@ -2165,7 +2211,7 @@ class ComplianceOverviewSerializer(serializers.Serializer): resource_name = "compliance-overviews" -class ComplianceOverviewDetailSerializer(serializers.Serializer): +class ComplianceOverviewDetailSerializer(BaseSerializerV1): """ Serializer for detailed compliance requirement information. @@ -2194,7 +2240,7 @@ class ComplianceOverviewDetailThreatscoreSerializer(ComplianceOverviewDetailSeri total_findings = serializers.IntegerField() -class ComplianceOverviewAttributesSerializer(serializers.Serializer): +class ComplianceOverviewAttributesSerializer(BaseSerializerV1): id = serializers.CharField() compliance_name = serializers.CharField() framework_description = serializers.CharField() @@ -2208,7 +2254,7 @@ class ComplianceOverviewAttributesSerializer(serializers.Serializer): resource_name = "compliance-requirements-attributes" -class ComplianceOverviewMetadataSerializer(serializers.Serializer): +class ComplianceOverviewMetadataSerializer(BaseSerializerV1): regions = serializers.ListField(child=serializers.CharField(), allow_empty=True) class JSONAPIMeta: @@ -2218,7 +2264,7 @@ class ComplianceOverviewMetadataSerializer(serializers.Serializer): # Overviews -class OverviewProviderSerializer(serializers.Serializer): +class OverviewProviderSerializer(BaseSerializerV1): id = serializers.CharField(source="provider") findings = serializers.SerializerMethodField(read_only=True) resources = serializers.SerializerMethodField(read_only=True) @@ -2226,9 +2272,6 @@ class OverviewProviderSerializer(serializers.Serializer): class JSONAPIMeta: resource_name = "providers-overview" - def get_root_meta(self, _resource, _many): - return {"version": "v1"} - @extend_schema_field( { "type": "object", @@ -2262,18 +2305,15 @@ class OverviewProviderSerializer(serializers.Serializer): } -class OverviewProviderCountSerializer(serializers.Serializer): +class OverviewProviderCountSerializer(BaseSerializerV1): id = serializers.CharField(source="provider") count = serializers.IntegerField() class JSONAPIMeta: resource_name = "providers-count-overview" - def get_root_meta(self, _resource, _many): - return {"version": "v1"} - -class OverviewFindingSerializer(serializers.Serializer): +class OverviewFindingSerializer(BaseSerializerV1): id = serializers.CharField(default="n/a") new = serializers.IntegerField() changed = serializers.IntegerField() @@ -2292,15 +2332,12 @@ class OverviewFindingSerializer(serializers.Serializer): class JSONAPIMeta: resource_name = "findings-overview" - def get_root_meta(self, _resource, _many): - return {"version": "v1"} - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields["pass"] = self.fields.pop("_pass") -class OverviewSeveritySerializer(serializers.Serializer): +class OverviewSeveritySerializer(BaseSerializerV1): id = serializers.CharField(default="n/a") critical = serializers.IntegerField() high = serializers.IntegerField() @@ -2311,11 +2348,24 @@ class OverviewSeveritySerializer(serializers.Serializer): class JSONAPIMeta: resource_name = "findings-severity-overview" - def get_root_meta(self, _resource, _many): - return {"version": "v1"} + +class FindingsSeverityOverTimeSerializer(BaseSerializerV1): + """Serializer for daily findings severity trend data.""" + + id = serializers.DateField(source="date") + critical = serializers.IntegerField() + high = serializers.IntegerField() + medium = serializers.IntegerField() + low = serializers.IntegerField() + informational = serializers.IntegerField() + muted = serializers.IntegerField() + scan_ids = serializers.ListField(child=serializers.UUIDField()) + + class JSONAPIMeta: + resource_name = "findings-severity-over-time" -class OverviewServiceSerializer(serializers.Serializer): +class OverviewServiceSerializer(BaseSerializerV1): id = serializers.CharField(source="service") total = serializers.IntegerField() _pass = serializers.IntegerField() @@ -2329,8 +2379,46 @@ class OverviewServiceSerializer(serializers.Serializer): super().__init__(*args, **kwargs) self.fields["pass"] = self.fields.pop("_pass") - def get_root_meta(self, _resource, _many): - return {"version": "v1"} + +class AttackSurfaceOverviewSerializer(BaseSerializerV1): + """Serializer for attack surface overview aggregations.""" + + id = serializers.CharField(source="attack_surface_type") + total_findings = serializers.IntegerField() + failed_findings = serializers.IntegerField() + muted_failed_findings = serializers.IntegerField() + + class JSONAPIMeta: + resource_name = "attack-surface-overviews" + + +class CategoryOverviewSerializer(BaseSerializerV1): + """Serializer for category overview aggregations.""" + + id = serializers.CharField(source="category") + total_findings = serializers.IntegerField() + failed_findings = serializers.IntegerField() + new_failed_findings = serializers.IntegerField() + severity = serializers.JSONField( + help_text="Severity breakdown: {informational, low, medium, high, critical}" + ) + + class JSONAPIMeta: + resource_name = "category-overviews" + + +class ComplianceWatchlistOverviewSerializer(BaseSerializerV1): + """Serializer for compliance watchlist overview with FAIL-dominant aggregation.""" + + id = serializers.CharField(source="compliance_id") + compliance_id = serializers.CharField() + requirements_passed = serializers.IntegerField() + requirements_failed = serializers.IntegerField() + requirements_manual = serializers.IntegerField() + total_requirements = serializers.IntegerField() + + class JSONAPIMeta: + resource_name = "compliance-watchlist-overviews" class OverviewRegionSerializer(serializers.Serializer): @@ -2360,7 +2448,7 @@ class OverviewRegionSerializer(serializers.Serializer): # Schedules -class ScheduleDailyCreateSerializer(serializers.Serializer): +class ScheduleDailyCreateSerializer(BaseSerializerV1): provider_id = serializers.UUIDField(required=True) class JSONAPIMeta: @@ -2696,7 +2784,7 @@ class IntegrationUpdateSerializer(BaseWriteIntegrationSerializer): return representation -class IntegrationJiraDispatchSerializer(serializers.Serializer): +class IntegrationJiraDispatchSerializer(BaseSerializerV1): """ Serializer for dispatching findings to JIRA integration. """ @@ -2859,14 +2947,14 @@ class ProcessorUpdateSerializer(BaseWriteSerializer): # SSO -class SamlInitiateSerializer(serializers.Serializer): +class SamlInitiateSerializer(BaseSerializerV1): email_domain = serializers.CharField() class JSONAPIMeta: resource_name = "saml-initiate" -class SamlMetadataSerializer(serializers.Serializer): +class SamlMetadataSerializer(BaseSerializerV1): class JSONAPIMeta: resource_name = "saml-meta" @@ -3398,6 +3486,19 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): and provider_type == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK ): + # For updates, enforce that the authentication method (access keys vs API key) + # is immutable. To switch methods, the UI must delete and recreate the provider. + existing_credentials = ( + self.instance.credentials_decoded if self.instance else {} + ) or {} + + existing_uses_api_key = "api_key" in existing_credentials + existing_uses_access_keys = any( + k in existing_credentials + for k in ("access_key_id", "secret_access_key") + ) + + # First run field-level validation on the partial payload try: BedrockCredentialsUpdateSerializer(data=credentials).is_valid( raise_exception=True @@ -3408,6 +3509,31 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): e.detail[f"credentials/{key}"] = value del e.detail[key] raise e + + # Then enforce invariants about not changing the auth method + # If the existing config uses an API key, forbid introducing access keys. + if existing_uses_api_key and any( + k in credentials for k in ("access_key_id", "secret_access_key") + ): + raise ValidationError( + { + "credentials/non_field_errors": [ + "Cannot change Bedrock authentication method from API key " + "to access key via update. Delete and recreate the provider instead." + ] + } + ) + + # If the existing config uses access keys, forbid introducing an API key. + if existing_uses_access_keys and "api_key" in credentials: + raise ValidationError( + { + "credentials/non_field_errors": [ + "Cannot change Bedrock authentication method from access key " + "to API key via update. Delete and recreate the provider instead." + ] + } + ) elif ( credentials is not None and provider_type diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 5b7e484350..9898051bcb 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -90,8 +90,12 @@ from api.db_router import MainRouter from api.db_utils import rls_transaction from api.exceptions import TaskFailedException from api.filters import ( + AttackSurfaceOverviewFilter, + CategoryOverviewFilter, ComplianceOverviewFilter, + ComplianceWatchlistFilter, CustomDjangoFilterBackend, + DailySeveritySummaryFilter, FindingFilter, IntegrationFilter, IntegrationJiraFindingsFilter, @@ -119,7 +123,10 @@ from api.filters import ( UserFilter, ) from api.models import ( + AttackSurfaceOverview, + ComplianceOverviewSummary, ComplianceRequirementOverview, + DailySeveritySummary, Finding, Integration, Invitation, @@ -132,6 +139,7 @@ from api.models import ( MuteRule, Processor, Provider, + ProviderComplianceScore, ProviderGroup, ProviderGroupMembership, ProviderSecret, @@ -145,11 +153,13 @@ from api.models import ( SAMLDomainIndex, SAMLToken, Scan, + ScanCategorySummary, ScanSummary, SeverityChoices, StateChoices, Task, TenantAPIKey, + TenantComplianceSummary, ThreatScoreSnapshot, User, UserRoleRelationship, @@ -169,14 +179,18 @@ from api.v1.serializers import ( AttackPathsQuerySerializer, AttackPathsQueryResultSerializer, AttackPathsScanSerializer, + AttackSurfaceOverviewSerializer, + CategoryOverviewSerializer, ComplianceOverviewAttributesSerializer, ComplianceOverviewDetailSerializer, ComplianceOverviewDetailThreatscoreSerializer, ComplianceOverviewMetadataSerializer, ComplianceOverviewSerializer, + ComplianceWatchlistOverviewSerializer, FindingDynamicFilterSerializer, FindingMetadataSerializer, FindingSerializer, + FindingsSeverityOverTimeSerializer, IntegrationCreateSerializer, IntegrationJiraDispatchSerializer, IntegrationSerializer, @@ -250,6 +264,7 @@ from tasks.beat import schedule_provider_scan from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils from tasks.jobs.export import get_s3_client from tasks.tasks import ( + backfill_compliance_summaries_task, backfill_scan_resource_summaries_task, check_integration_connection_task, check_lighthouse_connection_task, @@ -363,7 +378,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.15.0" + spectacular_settings.VERSION = "1.18.0" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -2957,12 +2972,15 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): queryset = ResourceScanSummary.objects.filter(tenant_id=tenant_id) scan_based_filters = {} + category_scan_filters = {} # Filters for ScanCategorySummary if scans := query_params.get("filter[scan__in]") or query_params.get( "filter[scan]" ): - queryset = queryset.filter(scan_id__in=scans.split(",")) - scan_based_filters = {"id__in": scans.split(",")} + scan_ids_list = scans.split(",") + queryset = queryset.filter(scan_id__in=scan_ids_list) + scan_based_filters = {"id__in": scan_ids_list} + category_scan_filters = {"scan_id__in": scan_ids_list} else: exact = query_params.get("filter[inserted_at]") gte = query_params.get("filter[inserted_at__gte]") @@ -3006,6 +3024,7 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): scan_based_filters = { key.lstrip("scan_"): value for key, value in date_filters.items() } + category_scan_filters = date_filters # ToRemove: Temporary fallback mechanism if not queryset.exists(): @@ -3052,10 +3071,31 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): .order_by("resource_type") ) + # Get categories from ScanCategorySummary using same scan filters + categories = list( + ScanCategorySummary.objects.filter( + tenant_id=tenant_id, **category_scan_filters + ) + .values_list("category", flat=True) + .distinct() + .order_by("category") + ) + + # Fallback to finding aggregation if no ScanCategorySummary exists + if not categories: + categories_set = set() + for categories_list in filtered_queryset.values_list( + "categories", flat=True + ): + if categories_list: + categories_set.update(categories_list) + categories = sorted(categories_set) + result = { "services": services, "regions": regions, "resource_types": resource_types, + "categories": categories, } serializer = self.get_serializer(data=result) @@ -3160,10 +3200,36 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): .order_by("resource_type") ) + # Get categories from ScanCategorySummary for latest scans + categories = list( + ScanCategorySummary.objects.filter( + tenant_id=tenant_id, + scan_id__in=latest_scans_queryset.values_list("id", flat=True), + ) + .values_list("category", flat=True) + .distinct() + .order_by("category") + ) + + # Fallback to finding aggregation if no ScanCategorySummary exists + if not categories: + filtered_queryset = self.filter_queryset(self.get_queryset()).filter( + tenant_id=tenant_id, + scan_id__in=latest_scans_queryset.values_list("id", flat=True), + ) + categories_set = set() + for categories_list in filtered_queryset.values_list( + "categories", flat=True + ): + if categories_list: + categories_set.update(categories_list) + categories = sorted(categories_set) + result = { "services": services, "regions": regions, "resource_types": resource_types, + "categories": categories, } serializer = self.get_serializer(data=result) @@ -3728,6 +3794,126 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): def retrieve(self, request, *args, **kwargs): raise MethodNotAllowed(method="GET") + def _compliance_summaries_queryset(self, scan_id): + """Return pre-aggregated summaries constrained by RBAC visibility.""" + role = get_role(self.request.user) + unlimited_visibility = getattr( + role, Permissions.UNLIMITED_VISIBILITY.value, False + ) + summaries = ComplianceOverviewSummary.objects.filter( + tenant_id=self.request.tenant_id, + scan_id=scan_id, + ) + + if not unlimited_visibility: + providers = Provider.all_objects.filter( + provider_groups__in=role.provider_groups.all() + ).distinct() + summaries = summaries.filter(scan__provider__in=providers) + + return summaries + + def _get_compliance_template(self, *, provider=None, scan_id=None): + """Return the compliance template for the given provider or scan.""" + if provider is None and scan_id is not None: + try: + scan = Scan.all_objects.select_related("provider").get(pk=scan_id) + except Scan.DoesNotExist: + raise ValidationError( + [ + { + "detail": "Scan not found", + "status": 404, + "source": {"pointer": "filter[scan_id]"}, + "code": "not_found", + } + ] + ) + provider = scan.provider + + if not provider: + return {} + + return PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE.get(provider.provider, {}) + + def _aggregate_compliance_overview(self, queryset, template_metadata=None): + """ + Aggregate requirement rows into compliance overview dictionaries. + + Args: + queryset: ComplianceRequirementOverview queryset already filtered. + template_metadata: Optional dict mapping compliance_id -> metadata. + """ + template_metadata = template_metadata or {} + requirement_status_subquery = queryset.values( + "compliance_id", "requirement_id" + ).annotate( + fail_count=Count("id", filter=Q(requirement_status="FAIL")), + pass_count=Count("id", filter=Q(requirement_status="PASS")), + total_count=Count("id"), + ) + + compliance_data = {} + fallback_metadata = { + item["compliance_id"]: { + "framework": item["framework"], + "version": item["version"], + } + for item in queryset.values( + "compliance_id", "framework", "version" + ).distinct() + } + + for item in requirement_status_subquery: + compliance_id = item["compliance_id"] + + if item["fail_count"] > 0: + req_status = "FAIL" + elif item["pass_count"] == item["total_count"]: + req_status = "PASS" + else: + req_status = "MANUAL" + + compliance_status = compliance_data.setdefault( + compliance_id, + { + "total_requirements": 0, + "requirements_passed": 0, + "requirements_failed": 0, + "requirements_manual": 0, + }, + ) + + compliance_status["total_requirements"] += 1 + if req_status == "PASS": + compliance_status["requirements_passed"] += 1 + elif req_status == "FAIL": + compliance_status["requirements_failed"] += 1 + else: + compliance_status["requirements_manual"] += 1 + + response_data = [] + for compliance_id, data in compliance_data.items(): + template = template_metadata.get(compliance_id, {}) + fallback = fallback_metadata.get(compliance_id, {}) + + response_data.append( + { + "id": compliance_id, + "compliance_id": compliance_id, + "framework": template.get("framework") + or fallback.get("framework", ""), + "version": template.get("version") or fallback.get("version", ""), + "requirements_passed": data["requirements_passed"], + "requirements_failed": data["requirements_failed"], + "requirements_manual": data["requirements_manual"], + "total_requirements": data["total_requirements"], + } + ) + + serializer = self.get_serializer(response_data, many=True) + return serializer.data + def _task_response_if_running(self, scan_id): """Check for an in-progress task only when no compliance data exists.""" try: @@ -3742,90 +3928,84 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): status=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - def list(self, request, *args, **kwargs): - scan_id = request.query_params.get("filter[scan_id]") - if not scan_id: - raise ValidationError( - [ - { - "detail": "This query parameter is required.", - "status": 400, - "source": {"pointer": "filter[scan_id]"}, - "code": "required", - } - ] - ) - try: - if task := self.get_task_response_if_running( - task_name="scan-compliance-overviews", - task_kwargs={"tenant_id": self.request.tenant_id, "scan_id": scan_id}, - raise_on_not_found=False, - ): - return task - except TaskFailedException: - return Response( - {"detail": "Task failed to generate compliance overview data."}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) - queryset = self.filter_queryset(self.filter_queryset(self.get_queryset())) - - requirement_status_subquery = queryset.values( - "compliance_id", "requirement_id" - ).annotate( - fail_count=Count("id", filter=Q(requirement_status="FAIL")), - pass_count=Count("id", filter=Q(requirement_status="PASS")), - total_count=Count("id"), + def _list_with_region_filter(self, scan_id, region_filter): + """ + Fall back to detailed ComplianceRequirementOverview query when region filter is applied. + This uses the original aggregation logic across filtered regions. + """ + regions = region_filter.split(",") if "," in region_filter else [region_filter] + queryset = self.filter_queryset(self.get_queryset()).filter( + scan_id=scan_id, + region__in=regions, ) - compliance_data = {} - framework_info = {} + data = self._aggregate_compliance_overview(queryset) + if data: + return Response(data) - for item in queryset.values("compliance_id", "framework", "version").distinct(): - framework_info[item["compliance_id"]] = { - "framework": item["framework"], - "version": item["version"], - } + task_response = self._task_response_if_running(scan_id) + if task_response: + return task_response - for item in requirement_status_subquery: - compliance_id = item["compliance_id"] + return Response(data) - if item["fail_count"] > 0: - req_status = "FAIL" - elif item["pass_count"] == item["total_count"]: - req_status = "PASS" - else: - req_status = "MANUAL" + def _list_without_region_aggregation(self, scan_id): + """ + Fall back aggregation when compliance summaries don't exist yet. + Aggregates ComplianceRequirementOverview data across ALL regions. + """ + queryset = self.filter_queryset(self.get_queryset()).filter(scan_id=scan_id) + compliance_template = self._get_compliance_template(scan_id=scan_id) + data = self._aggregate_compliance_overview( + queryset, template_metadata=compliance_template + ) + if data: + return Response(data) - if compliance_id not in compliance_data: - compliance_data[compliance_id] = { - "total_requirements": 0, - "requirements_passed": 0, - "requirements_failed": 0, - "requirements_manual": 0, - } + task_response = self._task_response_if_running(scan_id) + if task_response: + return task_response - compliance_data[compliance_id]["total_requirements"] += 1 - if req_status == "PASS": - compliance_data[compliance_id]["requirements_passed"] += 1 - elif req_status == "FAIL": - compliance_data[compliance_id]["requirements_failed"] += 1 - else: - compliance_data[compliance_id]["requirements_manual"] += 1 + return Response(data) + def list(self, request, *args, **kwargs): + scan_id = request.query_params.get("filter[scan_id]") + + # Specific scan requested - use optimized summaries with region support + region_filter = request.query_params.get( + "filter[region]" + ) or request.query_params.get("filter[region__in]") + + if region_filter: + # Fall back to detailed query with region filtering + return self._list_with_region_filter(scan_id, region_filter) + + summaries = list(self._compliance_summaries_queryset(scan_id)) + if not summaries: + # Trigger async backfill for next time + backfill_compliance_summaries_task.delay( + tenant_id=self.request.tenant_id, scan_id=scan_id + ) + # Use fallback aggregation for this request + return self._list_without_region_aggregation(scan_id) + + # Get compliance template for provider to enrich with framework/version + compliance_template = self._get_compliance_template(scan_id=scan_id) + + # Convert to response format with framework/version enrichment response_data = [] - for compliance_id, data in compliance_data.items(): - framework = framework_info.get(compliance_id, {}) - + for summary in summaries: + compliance_metadata = compliance_template.get(summary.compliance_id, {}) response_data.append( { - "id": compliance_id, - "compliance_id": compliance_id, - "framework": framework.get("framework", ""), - "version": framework.get("version", ""), - "requirements_passed": data["requirements_passed"], - "requirements_failed": data["requirements_failed"], - "requirements_manual": data["requirements_manual"], - "total_requirements": data["total_requirements"], + "id": summary.compliance_id, + "compliance_id": summary.compliance_id, + "framework": compliance_metadata.get("framework", ""), + "version": compliance_metadata.get("version", ""), + "requirements_passed": summary.requirements_passed, + "requirements_failed": summary.requirements_failed, + "requirements_manual": summary.requirements_manual, + "total_requirements": summary.total_requirements, } ) @@ -4095,6 +4275,34 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): ), filters=True, ), + findings_severity_timeseries=extend_schema( + summary="Get findings severity data over time", + description=( + "Retrieve daily aggregated findings data grouped by severity levels over a date range. " + "Returns one data point per day with counts of failed findings by severity (critical, high, " + "medium, low, informational) and muted findings. Days without scans are filled forward with " + "the most recent known values. Use date_from (required) and date_to filters to specify the range." + ), + filters=True, + ), + attack_surface=extend_schema( + summary="Get attack surface overview", + description="Retrieve aggregated attack surface metrics from latest completed scans per provider.", + tags=["Overview"], + filters=True, + responses={200: AttackSurfaceOverviewSerializer(many=True)}, + ), + categories=extend_schema( + summary="Get category overview", + description=( + "Retrieve aggregated category metrics from latest completed scans per provider. " + "Returns one row per category with total, failed, and new failed findings counts, " + "plus a severity breakdown showing failed findings per severity level. " + ), + tags=["Overview"], + filters=True, + responses={200: CategoryOverviewSerializer(many=True)}, + ), ) @method_decorator(CACHE_DECORATOR, name="list") class OverviewViewSet(BaseRLSViewSet): @@ -4112,7 +4320,16 @@ class OverviewViewSet(BaseRLSViewSet): if not role.unlimited_visibility: self.allowed_providers = providers - return ScanSummary.all_objects.filter(tenant_id=self.request.tenant_id) + tenant_id = self.request.tenant_id + + # Return appropriate queryset per action + if self.action == "findings_severity_timeseries": + qs = DailySeveritySummary.objects.filter(tenant_id=tenant_id) + if hasattr(self, "allowed_providers"): + qs = qs.filter(provider_id__in=self.allowed_providers) + return qs + + return ScanSummary.all_objects.filter(tenant_id=tenant_id) def get_serializer_class(self): if self.action == "providers": @@ -4123,12 +4340,20 @@ class OverviewViewSet(BaseRLSViewSet): return OverviewFindingSerializer elif self.action == "findings_severity": return OverviewSeveritySerializer + elif self.action == "findings_severity_timeseries": + return FindingsSeverityOverTimeSerializer elif self.action == "services": return OverviewServiceSerializer elif self.action == "regions": return OverviewRegionSerializer elif self.action == "threatscore": return ThreatScoreSnapshotSerializer + elif self.action == "attack_surface": + return AttackSurfaceOverviewSerializer + elif self.action == "categories": + return CategoryOverviewSerializer + elif self.action == "compliance_watchlist": + return ComplianceWatchlistOverviewSerializer return super().get_serializer_class() def get_filterset_class(self): @@ -4138,8 +4363,24 @@ class OverviewViewSet(BaseRLSViewSet): return ScanSummaryFilter elif self.action == "findings_severity": return ScanSummarySeverityFilter + elif self.action == "findings_severity_timeseries": + return DailySeveritySummaryFilter + elif self.action == "categories": + return CategoryOverviewFilter + elif self.action == "attack_surface": + return AttackSurfaceOverviewFilter + elif self.action == "compliance_watchlist": + return ComplianceWatchlistFilter return None + def filter_queryset(self, queryset): + # Skip OrderingFilter for findings_severity_timeseries (no inserted_at field) + if self.action == "findings_severity_timeseries": + return CustomDjangoFilterBackend().filter_queryset( + self.request, queryset, self + ) + return super().filter_queryset(queryset) + @extend_schema(exclude=True) def list(self, request, *args, **kwargs): raise MethodNotAllowed(method="GET") @@ -4177,6 +4418,93 @@ class OverviewViewSet(BaseRLSViewSet): tenant_id=tenant_id, scan_id__in=latest_scan_ids ) + def _normalize_jsonapi_params(self, query_params, exclude_keys=None): + """Convert JSON:API filter params (filter[X]) to flat params (X).""" + exclude_keys = exclude_keys or set() + normalized = QueryDict(mutable=True) + for key, values in query_params.lists(): + normalized_key = ( + key[7:-1] if key.startswith("filter[") and key.endswith("]") else key + ) + if normalized_key not in exclude_keys: + normalized.setlist(normalized_key, values) + return normalized + + def _ensure_allowed_providers(self): + """Populate allowed providers for RBAC-aware queries once per request.""" + if getattr(self, "_providers_initialized", False): + return + self.get_queryset() + self._providers_initialized = True + + def _get_provider_filter(self, provider_field="provider"): + self._ensure_allowed_providers() + if hasattr(self, "allowed_providers"): + return {f"{provider_field}__in": self.allowed_providers} + return {} + + def _apply_provider_filter(self, queryset, provider_field="provider"): + provider_filter = self._get_provider_filter(provider_field) + if provider_filter: + return queryset.filter(**provider_filter) + return queryset + + def _apply_filterset(self, queryset, filterset_class, exclude_keys=None): + normalized_params = self._normalize_jsonapi_params( + self.request.query_params, exclude_keys=set(exclude_keys or []) + ) + filterset = filterset_class(normalized_params, queryset=queryset) + if not filterset.is_valid(): + raise ValidationError(filterset.errors) + return filterset.qs + + def _latest_scan_ids_for_allowed_providers(self, tenant_id, provider_filters=None): + provider_filter = self._get_provider_filter() + queryset = Scan.all_objects.filter( + tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter + ) + if provider_filters: + queryset = queryset.filter(**provider_filters) + return ( + queryset.order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + + def _extract_provider_filters_from_params(self): + """Extract and validate provider filters from query params.""" + params = self.request.query_params + filters = {} + valid_provider_types = {c[0] for c in Provider.ProviderChoices.choices} + + provider_id = params.get("filter[provider_id]") + if provider_id: + filters["provider_id"] = provider_id + + provider_id_in = params.get("filter[provider_id__in]") + if provider_id_in: + filters["provider_id__in"] = provider_id_in.split(",") + + provider_type = params.get("filter[provider_type]") + if provider_type: + if provider_type not in valid_provider_types: + raise ValidationError( + {"provider_type": f"Invalid choice: {provider_type}"} + ) + filters["provider__provider"] = provider_type + + provider_type_in = params.get("filter[provider_type__in]") + if provider_type_in: + types = provider_type_in.split(",") + invalid = [t for t in types if t not in valid_provider_types] + if invalid: + raise ValidationError( + {"provider_type__in": f"Invalid choices: {', '.join(invalid)}"} + ) + filters["provider__provider__in"] = types + + return filters + @action(detail=False, methods=["get"], url_name="providers") def providers(self, request): tenant_id = self.request.tenant_id @@ -4354,6 +4682,108 @@ class OverviewViewSet(BaseRLSViewSet): return Response(serializer.data, status=status.HTTP_200_OK) + @action( + detail=False, + methods=["get"], + url_path="findings_severity/timeseries", + url_name="findings_severity_timeseries", + ) + def findings_severity_timeseries(self, request): + """ + Daily severity trends for charts. Uses DailySeveritySummary pre-aggregation. + Requires date_from filter. + """ + # Get queryset with RBAC, provider, and date filters applied + # Date validation is handled by DailySeveritySummaryFilter + daily_qs = self.filter_queryset(self.get_queryset()) + + date_from = request._date_from + date_to = request._date_to + + if not daily_qs.exists(): + # No data matches filters - return zeros + result = self._generate_zero_result(date_from, date_to) + serializer = self.get_serializer(result, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + # Fetch all data for fill-forward logic + daily_summaries = list( + daily_qs.order_by("provider_id", "-date").values( + "provider_id", + "scan_id", + "date", + "critical", + "high", + "medium", + "low", + "informational", + "muted", + ) + ) + + if not daily_summaries: + result = self._generate_zero_result(date_from, date_to) + serializer = self.get_serializer(result, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + # Build provider_data: {provider_id: [(date, data), ...]} sorted by date desc + provider_data = defaultdict(list) + for summary in daily_summaries: + provider_data[summary["provider_id"]].append(summary) + + # For each day, find the latest data per provider and sum values + result = [] + current_date = date_from + while current_date <= date_to: + day_totals = { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "informational": 0, + "muted": 0, + } + day_scan_ids = [] + + for provider_id, summaries in provider_data.items(): + # Find the latest data for this provider <= current_date + for summary in summaries: # Already sorted by date desc + if summary["date"] <= current_date: + day_totals["critical"] += summary["critical"] or 0 + day_totals["high"] += summary["high"] or 0 + day_totals["medium"] += summary["medium"] or 0 + day_totals["low"] += summary["low"] or 0 + day_totals["informational"] += summary["informational"] or 0 + day_totals["muted"] += summary["muted"] or 0 + day_scan_ids.append(summary["scan_id"]) + break # Found the latest data for this provider + + result.append( + {"date": current_date, "scan_ids": day_scan_ids, **day_totals} + ) + current_date += timedelta(days=1) + + serializer = self.get_serializer(result, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + def _generate_zero_result(self, date_from, date_to): + """Generate a list of zero-filled results for each date in range.""" + result = [] + current_date = date_from + zero_values = { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "informational": 0, + "muted": 0, + "scan_ids": [], + } + while current_date <= date_to: + result.append({"date": current_date, **zero_values}) + current_date += timedelta(days=1) + return result + @extend_schema( summary="Get ThreatScore snapshots", description=( @@ -4406,11 +4836,9 @@ class OverviewViewSet(BaseRLSViewSet): snapshot_id = request.query_params.get("snapshot_id") # Base queryset with RLS - base_queryset = ThreatScoreSnapshot.objects.filter(tenant_id=tenant_id) - - # Apply RBAC filtering - if hasattr(self, "allowed_providers"): - base_queryset = base_queryset.filter(provider__in=self.allowed_providers) + base_queryset = self._apply_provider_filter( + ThreatScoreSnapshot.objects.filter(tenant_id=tenant_id) + ) # Case 1: Specific snapshot requested if snapshot_id: @@ -4426,17 +4854,9 @@ class OverviewViewSet(BaseRLSViewSet): # Case 2: Latest snapshot per provider (default) # Apply filters manually: this @action is outside the standard list endpoint flow, # so DRF's filter backends don't execute and we must flatten JSON:API params ourselves. - normalized_params = QueryDict(mutable=True) - for param_key, values in request.query_params.lists(): - normalized_key = param_key - if param_key.startswith("filter[") and param_key.endswith("]"): - normalized_key = param_key[7:-1] - if normalized_key == "snapshot_id": - continue - normalized_params.setlist(normalized_key, values) - - filterset = ThreatScoreSnapshotFilter(normalized_params, queryset=base_queryset) - filtered_queryset = filterset.qs + filtered_queryset = self._apply_filterset( + base_queryset, ThreatScoreSnapshotFilter, exclude_keys={"snapshot_id"} + ) # Get distinct provider IDs from filtered queryset # Pick the latest snapshot per provider using Postgres DISTINCT ON pattern. @@ -4680,6 +5100,203 @@ class OverviewViewSet(BaseRLSViewSet): return aggregated_snapshot + @action( + detail=False, + methods=["get"], + url_name="attack-surface", + url_path="attack-surfaces", + ) + def attack_surface(self, request): + tenant_id = request.tenant_id + latest_scan_ids = self._latest_scan_ids_for_allowed_providers(tenant_id) + + base_queryset = AttackSurfaceOverview.objects.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids + ) + filtered_queryset = self._apply_filterset( + base_queryset, AttackSurfaceOverviewFilter + ) + + aggregation = filtered_queryset.values("attack_surface_type").annotate( + total_findings=Coalesce(Sum("total_findings"), 0), + failed_findings=Coalesce(Sum("failed_findings"), 0), + muted_failed_findings=Coalesce(Sum("muted_failed_findings"), 0), + ) + + results = { + attack_surface_type: { + "total_findings": 0, + "failed_findings": 0, + "muted_failed_findings": 0, + } + for attack_surface_type in AttackSurfaceOverview.AttackSurfaceTypeChoices.values + } + for item in aggregation: + results[item["attack_surface_type"]] = { + "total_findings": item["total_findings"], + "failed_findings": item["failed_findings"], + "muted_failed_findings": item["muted_failed_findings"], + } + + response_data = [ + {"attack_surface_type": key, **value} for key, value in results.items() + ] + + return Response( + self.get_serializer(response_data, many=True).data, + status=status.HTTP_200_OK, + ) + + @action(detail=False, methods=["get"], url_name="categories") + def categories(self, request): + tenant_id = request.tenant_id + provider_filters = self._extract_provider_filters_from_params() + latest_scan_ids = self._latest_scan_ids_for_allowed_providers( + tenant_id, provider_filters + ) + + base_queryset = ScanCategorySummary.objects.filter( + tenant_id=tenant_id, scan_id__in=latest_scan_ids + ) + provider_filter_keys = { + "provider_id", + "provider_id__in", + "provider_type", + "provider_type__in", + } + filtered_queryset = self._apply_filterset( + base_queryset, CategoryOverviewFilter, exclude_keys=provider_filter_keys + ) + + aggregation = ( + filtered_queryset.values("category", "severity") + .annotate( + total=Coalesce(Sum("total_findings"), 0), + failed=Coalesce(Sum("failed_findings"), 0), + new_failed=Coalesce(Sum("new_failed_findings"), 0), + ) + .order_by("category", "severity") + ) + + category_data = defaultdict( + lambda: { + "total_findings": 0, + "failed_findings": 0, + "new_failed_findings": 0, + "severity": { + "informational": 0, + "low": 0, + "medium": 0, + "high": 0, + "critical": 0, + }, + } + ) + + for row in aggregation: + cat = row["category"] + sev = row["severity"] + category_data[cat]["total_findings"] += row["total"] + category_data[cat]["failed_findings"] += row["failed"] + category_data[cat]["new_failed_findings"] += row["new_failed"] + if sev in category_data[cat]["severity"]: + category_data[cat]["severity"][sev] = row["failed"] + + response_data = [ + {"category": cat, **data} for cat, data in sorted(category_data.items()) + ] + + return Response( + self.get_serializer(response_data, many=True).data, + status=status.HTTP_200_OK, + ) + + @action( + detail=False, + methods=["get"], + url_name="compliance-watchlist", + url_path="compliance-watchlist", + ) + def compliance_watchlist(self, request): + """ + Get compliance watchlist overview with FAIL-dominant aggregation. + + Without filters: uses pre-aggregated TenantComplianceSummary (~70 rows). + With provider filters: queries ProviderComplianceScore with FAIL-dominant logic. + """ + tenant_id = request.tenant_id + rbac_filter = self._get_provider_filter() + query_params = request.query_params + + has_provider_filter = any( + key.startswith("filter[provider") for key in query_params.keys() + ) + has_rbac_restriction = bool(rbac_filter) + + if not has_provider_filter and not has_rbac_restriction: + response_data = list( + TenantComplianceSummary.objects.filter(tenant_id=tenant_id) + .values( + "compliance_id", + "requirements_passed", + "requirements_failed", + "requirements_manual", + "total_requirements", + ) + .order_by("compliance_id") + ) + else: + base_queryset = ProviderComplianceScore.objects.filter( + tenant_id=tenant_id, **rbac_filter + ) + + filtered_queryset = self._apply_filterset( + base_queryset, ComplianceWatchlistFilter + ) + + aggregation = ( + filtered_queryset.values("compliance_id", "requirement_id") + .annotate( + has_fail=Sum( + Case(When(requirement_status="FAIL", then=1), default=0) + ), + has_manual=Sum( + Case(When(requirement_status="MANUAL", then=1), default=0) + ), + ) + .values("compliance_id", "requirement_id", "has_fail", "has_manual") + ) + + compliance_data = defaultdict( + lambda: { + "requirements_passed": 0, + "requirements_failed": 0, + "requirements_manual": 0, + "total_requirements": 0, + } + ) + + for row in aggregation: + cid = row["compliance_id"] + compliance_data[cid]["total_requirements"] += 1 + + if row["has_fail"] and row["has_fail"] > 0: + compliance_data[cid]["requirements_failed"] += 1 + elif row["has_manual"] and row["has_manual"] > 0: + compliance_data[cid]["requirements_manual"] += 1 + else: + compliance_data[cid]["requirements_passed"] += 1 + + response_data = [ + {"compliance_id": cid, **data} + for cid, data in sorted(compliance_data.items()) + ] + + return Response( + self.get_serializer(response_data, many=True).data, + status=status.HTTP_200_OK, + ) + @extend_schema(tags=["Schedule"]) @extend_schema_view( diff --git a/api/src/backend/config/django/devel.py b/api/src/backend/config/django/devel.py index 73662980f6..9c83557b77 100644 --- a/api/src/backend/config/django/devel.py +++ b/api/src/backend/config/django/devel.py @@ -36,6 +36,14 @@ DATABASES = { "HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host), "PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port), }, + "admin_replica": { + "ENGINE": "psqlextra.backend", + "NAME": env("POSTGRES_REPLICA_DB", default=default_db_name), + "USER": env("POSTGRES_ADMIN_USER", default="prowler"), + "PASSWORD": env("POSTGRES_ADMIN_PASSWORD", default="S3cret"), + "HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host), + "PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port), + }, "neo4j": { "HOST": env.str("NEO4J_HOST", "neo4j"), "PORT": env.str("NEO4J_PORT", "7687"), diff --git a/api/src/backend/config/django/production.py b/api/src/backend/config/django/production.py index 1b0fc90962..b2769237fc 100644 --- a/api/src/backend/config/django/production.py +++ b/api/src/backend/config/django/production.py @@ -37,6 +37,14 @@ DATABASES = { "HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host), "PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port), }, + "admin_replica": { + "ENGINE": "psqlextra.backend", + "NAME": env("POSTGRES_REPLICA_DB", default=default_db_name), + "USER": env("POSTGRES_ADMIN_USER"), + "PASSWORD": env("POSTGRES_ADMIN_PASSWORD"), + "HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host), + "PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port), + }, "neo4j": { "HOST": env.str("NEO4J_HOST"), "PORT": env.str("NEO4J_PORT"), diff --git a/api/src/backend/config/settings/sentry.py b/api/src/backend/config/settings/sentry.py index 98666c08de..6b8c554258 100644 --- a/api/src/backend/config/settings/sentry.py +++ b/api/src/backend/config/settings/sentry.py @@ -5,6 +5,9 @@ IGNORED_EXCEPTIONS = [ # Provider is not connected due to credentials errors "is not connected", "ProviderConnectionError", + # Provider was deleted during a scan + "ProviderDeletedException", + "violates foreign key constraint", # Authentication Errors from AWS "InvalidToken", "AccessDeniedException", diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index 20221aca2a..54ea2d8039 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -22,6 +22,7 @@ from api.attack_paths import ( from api.db_utils import rls_transaction from api.models import ( AttackPathsScan, + AttackSurfaceOverview, ComplianceOverview, ComplianceRequirementOverview, Finding, @@ -33,6 +34,7 @@ from api.models import ( MuteRule, Processor, Provider, + ProviderComplianceScore, ProviderGroup, ProviderSecret, Resource, @@ -42,11 +44,13 @@ from api.models import ( SAMLConfiguration, SAMLDomainIndex, Scan, + ScanCategorySummary, ScanSummary, StateChoices, StatusChoices, Task, TenantAPIKey, + TenantComplianceSummary, User, UserRoleRelationship, ) @@ -54,7 +58,7 @@ from api.rls import Tenant from api.v1.serializers import TokenSerializer from prowler.lib.check.models import Severity from prowler.lib.outputs.finding import Status -from tasks.jobs.backfill import backfill_resource_scan_summaries +from tasks.jobs.backfill import backfill_resource_scan_summaries, backfill_scan_category_summaries TODAY = str(datetime.today().date()) API_JSON_CONTENT_TYPE = "application/vnd.api+json" @@ -518,6 +522,12 @@ def providers_fixture(tenants_fixture): alias="mongodbatlas_testing", tenant_id=tenant.id, ) + provider9 = Provider.objects.create( + provider="alibabacloud", + uid="1234567890123456", + alias="alibabacloud_testing", + tenant_id=tenant.id, + ) return ( provider1, @@ -528,6 +538,7 @@ def providers_fixture(tenants_fixture): provider6, provider7, provider8, + provider9, ) @@ -1276,6 +1287,113 @@ def latest_scan_finding(authenticated_client, providers_fixture, resources_fixtu return finding +@pytest.fixture(scope="function") +def findings_with_categories(scans_fixture, resources_fixture): + scan = scans_fixture[0] + resource = resources_fixture[0] + + finding = Finding.objects.create( + tenant_id=scan.tenant_id, + uid="finding_with_categories_1", + scan=scan, + delta=None, + status=Status.FAIL, + status_extended="test status", + impact=Severity.critical, + impact_extended="test impact", + severity=Severity.critical, + raw_result={"status": Status.FAIL}, + check_id="genai_check", + check_metadata={"CheckId": "genai_check"}, + categories=["gen-ai", "security"], + first_seen_at="2024-01-02T00:00:00Z", + ) + finding.add_resources([resource]) + backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id)) + return finding + + +@pytest.fixture(scope="function") +def findings_with_multiple_categories(scans_fixture, resources_fixture): + scan = scans_fixture[0] + resource1, resource2 = resources_fixture[:2] + + finding1 = Finding.objects.create( + tenant_id=scan.tenant_id, + uid="finding_multi_cat_1", + scan=scan, + delta=None, + status=Status.FAIL, + status_extended="test status", + impact=Severity.critical, + impact_extended="test impact", + severity=Severity.critical, + raw_result={"status": Status.FAIL}, + check_id="genai_check", + check_metadata={"CheckId": "genai_check"}, + categories=["gen-ai", "security"], + first_seen_at="2024-01-02T00:00:00Z", + ) + finding1.add_resources([resource1]) + + finding2 = Finding.objects.create( + tenant_id=scan.tenant_id, + uid="finding_multi_cat_2", + scan=scan, + delta=None, + status=Status.FAIL, + status_extended="test status 2", + impact=Severity.high, + impact_extended="test impact 2", + severity=Severity.high, + raw_result={"status": Status.FAIL}, + check_id="iam_check", + check_metadata={"CheckId": "iam_check"}, + categories=["iam", "security"], + first_seen_at="2024-01-02T00:00:00Z", + ) + finding2.add_resources([resource2]) + + backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id)) + return finding1, finding2 + + +@pytest.fixture(scope="function") +def latest_scan_finding_with_categories( + authenticated_client, providers_fixture, resources_fixture +): + provider = providers_fixture[0] + tenant_id = str(providers_fixture[0].tenant_id) + resource = resources_fixture[0] + scan = Scan.objects.create( + name="latest completed scan with categories", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant_id=tenant_id, + ) + finding = Finding.objects.create( + tenant_id=tenant_id, + uid="latest_finding_with_categories", + scan=scan, + delta="new", + status=Status.FAIL, + status_extended="test status", + impact=Severity.critical, + impact_extended="test impact", + severity=Severity.critical, + raw_result={"status": Status.FAIL}, + check_id="genai_iam_check", + check_metadata={"CheckId": "genai_iam_check"}, + categories=["gen-ai", "iam"], + first_seen_at="2024-01-02T00:00:00Z", + ) + finding.add_resources([resource]) + backfill_resource_scan_summaries(tenant_id, str(scan.id)) + backfill_scan_category_summaries(tenant_id, str(scan.id)) + return finding + + @pytest.fixture(scope="function") def latest_scan_resource(authenticated_client, providers_fixture): provider = providers_fixture[0] @@ -1573,10 +1691,152 @@ def attack_paths_graph_stub_classes(): ) +@pytest.fixture +def create_attack_surface_overview(): + def _create(tenant, scan, attack_surface_type, total=10, failed=5, muted_failed=2): + return AttackSurfaceOverview.objects.create( + tenant=tenant, + scan=scan, + attack_surface_type=attack_surface_type, + total_findings=total, + failed_findings=failed, + muted_failed_findings=muted_failed, + ) + + return _create + + +@pytest.fixture +def create_scan_category_summary(): + def _create( + tenant, + scan, + category, + severity, + total_findings=10, + failed_findings=5, + new_failed_findings=2, + ): + return ScanCategorySummary.objects.create( + tenant=tenant, + scan=scan, + category=category, + severity=severity, + total_findings=total_findings, + failed_findings=failed_findings, + new_failed_findings=new_failed_findings, + ) + + return _create + + +@pytest.fixture def get_authorization_header(access_token: str) -> dict: return {"Authorization": f"Bearer {access_token}"} +@pytest.fixture +def provider_compliance_scores_fixture( + tenants_fixture, providers_fixture, scans_fixture +): + """Create ProviderComplianceScore entries for compliance watchlist tests.""" + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + scan1, _, scan3 = scans_fixture + + scan1.completed_at = datetime.now(timezone.utc) - timedelta(hours=1) + scan1.save() + scan3.state = StateChoices.COMPLETED + scan3.completed_at = datetime.now(timezone.utc) + scan3.save() + + scores = [ + ProviderComplianceScore.objects.create( + tenant_id=tenant.id, + provider=provider1, + scan=scan1, + compliance_id="aws_cis_2.0", + requirement_id="req_1", + requirement_status=StatusChoices.PASS, + scan_completed_at=scan1.completed_at, + ), + ProviderComplianceScore.objects.create( + tenant_id=tenant.id, + provider=provider1, + scan=scan1, + compliance_id="aws_cis_2.0", + requirement_id="req_2", + requirement_status=StatusChoices.FAIL, + scan_completed_at=scan1.completed_at, + ), + ProviderComplianceScore.objects.create( + tenant_id=tenant.id, + provider=provider1, + scan=scan1, + compliance_id="aws_cis_2.0", + requirement_id="req_3", + requirement_status=StatusChoices.MANUAL, + scan_completed_at=scan1.completed_at, + ), + ProviderComplianceScore.objects.create( + tenant_id=tenant.id, + provider=provider2, + scan=scan3, + compliance_id="aws_cis_2.0", + requirement_id="req_1", + requirement_status=StatusChoices.FAIL, + scan_completed_at=scan3.completed_at, + ), + ProviderComplianceScore.objects.create( + tenant_id=tenant.id, + provider=provider2, + scan=scan3, + compliance_id="aws_cis_2.0", + requirement_id="req_2", + requirement_status=StatusChoices.PASS, + scan_completed_at=scan3.completed_at, + ), + ProviderComplianceScore.objects.create( + tenant_id=tenant.id, + provider=provider1, + scan=scan1, + compliance_id="gdpr_aws", + requirement_id="gdpr_req_1", + requirement_status=StatusChoices.PASS, + scan_completed_at=scan1.completed_at, + ), + ] + + return scores + + +@pytest.fixture +def tenant_compliance_summary_fixture(tenants_fixture): + """Create TenantComplianceSummary entries for compliance watchlist tests.""" + tenant = tenants_fixture[0] + + summaries = [ + TenantComplianceSummary.objects.create( + tenant_id=tenant.id, + compliance_id="aws_cis_2.0", + requirements_passed=1, + requirements_failed=2, + requirements_manual=1, + total_requirements=4, + ), + TenantComplianceSummary.objects.create( + tenant_id=tenant.id, + compliance_id="gdpr_aws", + requirements_passed=5, + requirements_failed=0, + requirements_manual=2, + total_requirements=7, + ), + ] + + return summaries + + def pytest_collection_modifyitems(items): """Ensure test_rbac.py is executed first.""" items.sort(key=lambda item: 0 if "test_rbac.py" in item.nodeid else 1) diff --git a/api/src/backend/tasks/beat.py b/api/src/backend/tasks/beat.py index e42f9918ab..e9eb9c9309 100644 --- a/api/src/backend/tasks/beat.py +++ b/api/src/backend/tasks/beat.py @@ -68,4 +68,5 @@ def schedule_provider_scan(provider_instance: Provider): "tenant_id": str(provider_instance.tenant_id), "provider_id": provider_id, }, + countdown=5, # Avoid race conditions between the worker and the database ) diff --git a/api/src/backend/tasks/jobs/backfill.py b/api/src/backend/tasks/jobs/backfill.py index 39fcd41fe1..851efe4e4a 100644 --- a/api/src/backend/tasks/jobs/backfill.py +++ b/api/src/backend/tasks/jobs/backfill.py @@ -1,17 +1,39 @@ from collections import defaultdict +from datetime import timedelta -from api.db_router import READ_REPLICA_ALIAS -from api.db_utils import rls_transaction +from celery.utils.log import get_task_logger +from django.db.models import Sum +from django.utils import timezone +from tasks.jobs.queries import ( + COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL, + COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL, +) +from tasks.jobs.scan import aggregate_category_counts + +from api.db_router import READ_REPLICA_ALIAS, MainRouter +from api.db_utils import ( + POSTGRES_TENANT_VAR, + SET_CONFIG_QUERY, + psycopg_connection, + rls_transaction, +) from api.models import ( ComplianceOverviewSummary, ComplianceRequirementOverview, + DailySeveritySummary, + Finding, + ProviderComplianceScore, Resource, ResourceFindingMapping, ResourceScanSummary, Scan, + ScanCategorySummary, + ScanSummary, StateChoices, ) +logger = get_task_logger(__name__) + def backfill_resource_scan_summaries(tenant_id: str, scan_id: str): with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): @@ -175,3 +197,271 @@ def backfill_compliance_summaries(tenant_id: str, scan_id: str): ) return {"status": "backfilled", "inserted": len(summary_objects)} + + +def backfill_daily_severity_summaries(tenant_id: str, days: int = None): + """ + Backfill DailySeveritySummary from completed scans. + Groups by provider+date, keeps latest scan per day. + """ + created_count = 0 + updated_count = 0 + + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + scan_filter = { + "tenant_id": tenant_id, + "state": StateChoices.COMPLETED, + "completed_at__isnull": False, + } + + if days is not None: + cutoff_date = timezone.now() - timedelta(days=days) + scan_filter["completed_at__gte"] = cutoff_date + + completed_scans = ( + Scan.objects.filter(**scan_filter) + .order_by("provider_id", "-completed_at") + .values("id", "provider_id", "completed_at") + ) + + if not completed_scans: + return {"status": "no scans to backfill"} + + # Keep only latest scan per provider/day + latest_scans_by_day = {} + for scan in completed_scans: + key = (scan["provider_id"], scan["completed_at"].date()) + if key not in latest_scans_by_day: + latest_scans_by_day[key] = scan + + # Process each provider/day + for (provider_id, scan_date), scan in latest_scans_by_day.items(): + scan_id = scan["id"] + + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + severity_totals = ( + ScanSummary.objects.filter( + tenant_id=tenant_id, + scan_id=scan_id, + ) + .values("severity") + .annotate(total_fail=Sum("fail"), total_muted=Sum("muted")) + ) + + severity_data = { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "informational": 0, + "muted": 0, + } + + for row in severity_totals: + severity = row["severity"] + if severity in severity_data: + severity_data[severity] = row["total_fail"] or 0 + severity_data["muted"] += row["total_muted"] or 0 + + with rls_transaction(tenant_id): + _, created = DailySeveritySummary.objects.update_or_create( + tenant_id=tenant_id, + provider_id=provider_id, + date=scan_date, + defaults={ + "scan_id": scan_id, + "critical": severity_data["critical"], + "high": severity_data["high"], + "medium": severity_data["medium"], + "low": severity_data["low"], + "informational": severity_data["informational"], + "muted": severity_data["muted"], + }, + ) + + if created: + created_count += 1 + else: + updated_count += 1 + + return { + "status": "backfilled", + "created": created_count, + "updated": updated_count, + "total_days": len(latest_scans_by_day), + } + + +def backfill_scan_category_summaries(tenant_id: str, scan_id: str): + """ + Backfill ScanCategorySummary for a completed scan. + + Aggregates category counts from all findings in the scan and creates + one ScanCategorySummary row per (category, severity) combination. + + Args: + tenant_id: Target tenant UUID + scan_id: Scan UUID to backfill + + Returns: + dict: Status indicating whether backfill was performed + """ + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + if ScanCategorySummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).exists(): + return {"status": "already backfilled"} + + if not Scan.objects.filter( + tenant_id=tenant_id, + id=scan_id, + state__in=(StateChoices.COMPLETED, StateChoices.FAILED), + ).exists(): + return {"status": "scan is not completed"} + + category_counts: dict[tuple[str, str], dict[str, int]] = {} + for finding in Finding.all_objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).values("categories", "severity", "status", "delta", "muted"): + aggregate_category_counts( + categories=finding.get("categories") or [], + severity=finding.get("severity"), + status=finding.get("status"), + delta=finding.get("delta"), + muted=finding.get("muted", False), + cache=category_counts, + ) + + if not category_counts: + return {"status": "no categories to backfill"} + + category_summaries = [ + ScanCategorySummary( + tenant_id=tenant_id, + scan_id=scan_id, + category=category, + severity=severity, + total_findings=counts["total"], + failed_findings=counts["failed"], + new_failed_findings=counts["new_failed"], + ) + for (category, severity), counts in category_counts.items() + ] + + with rls_transaction(tenant_id): + ScanCategorySummary.objects.bulk_create( + category_summaries, batch_size=500, ignore_conflicts=True + ) + + return {"status": "backfilled", "categories_count": len(category_counts)} + + +def backfill_provider_compliance_scores(tenant_id: str) -> dict: + """ + Backfill ProviderComplianceScore from latest completed scan per provider. + + For each provider with completed scans, finds the most recent scan and + upserts compliance requirement statuses with FAIL-dominant aggregation. + + Args: + tenant_id: Target tenant UUID + + Returns: + dict: Statistics about the backfill operation + """ + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + completed_scans = Scan.all_objects.filter( + tenant_id=tenant_id, + state=StateChoices.COMPLETED, + completed_at__isnull=False, + ) + if not completed_scans.exists(): + return {"status": "no completed scans"} + + existing_providers = set( + ProviderComplianceScore.objects.filter(tenant_id=tenant_id) + .values_list("provider_id", flat=True) + .distinct() + ) + + if existing_providers: + completed_scans = completed_scans.exclude( + provider_id__in=existing_providers + ) + + scan_info = list( + completed_scans.order_by("provider_id", "-completed_at") + .distinct("provider_id") + .values("id", "provider_id", "completed_at") + ) + + if not scan_info: + return {"status": "no scans to process"} + + total_upserted = 0 + providers_processed = 0 + providers_skipped = 0 + + for scan in scan_info: + provider_id = scan["provider_id"] + + scan_id = scan["id"] + + try: + with psycopg_connection(MainRouter.default_db) as connection: + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.execute( + SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id] + ) + cursor.execute( + COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL, + [tenant_id, str(scan_id)], + ) + upserted = cursor.rowcount + connection.commit() + total_upserted += upserted + providers_processed += 1 + except Exception: + connection.rollback() + raise + except Exception as e: + providers_skipped += 1 + logger.exception( + "Error backfilling provider %s for tenant %s: %s", + provider_id, + tenant_id, + e, + ) + + # Recalculate tenant summary after all providers are backfilled + if providers_processed > 0: + with psycopg_connection(MainRouter.default_db) as connection: + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id]) + # Advisory lock to prevent race conditions + cursor.execute( + "SELECT pg_advisory_xact_lock(hashtext(%s))", [tenant_id] + ) + cursor.execute( + COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL, + [tenant_id, tenant_id], + ) + tenant_summary_count = cursor.rowcount + connection.commit() + except Exception: + connection.rollback() + raise + else: + tenant_summary_count = 0 + + return { + "status": "backfilled", + "providers_processed": providers_processed, + "providers_skipped": providers_skipped, + "total_upserted": total_upserted, + "tenant_summary_count": tenant_summary_count, + } diff --git a/api/src/backend/tasks/jobs/export.py b/api/src/backend/tasks/jobs/export.py index f7d286007c..883d7d0dbe 100644 --- a/api/src/backend/tasks/jobs/export.py +++ b/api/src/backend/tasks/jobs/export.py @@ -27,6 +27,7 @@ from prowler.lib.outputs.compliance.c5.c5_gcp import GCPC5 from prowler.lib.outputs.compliance.ccc.ccc_aws import CCC_AWS from prowler.lib.outputs.compliance.ccc.ccc_azure import CCC_Azure from prowler.lib.outputs.compliance.ccc.ccc_gcp import CCC_GCP +from prowler.lib.outputs.compliance.cis.cis_alibabacloud import AlibabaCloudCIS from prowler.lib.outputs.compliance.cis.cis_aws import AWSCIS from prowler.lib.outputs.compliance.cis.cis_azure import AzureCIS from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS @@ -50,6 +51,9 @@ from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_azure import ( AzureMitreAttack, ) from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_gcp import GCPMitreAttack +from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_alibaba import ( + ProwlerThreatScoreAlibaba, +) from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_aws import ( ProwlerThreatScoreAWS, ) @@ -128,6 +132,13 @@ COMPLIANCE_CLASS_MAP = { "oraclecloud": [ (lambda name: name.startswith("cis_"), OracleCloudCIS), ], + "alibabacloud": [ + (lambda name: name.startswith("cis_"), AlibabaCloudCIS), + ( + lambda name: name == "prowler_threatscore_alibabacloud", + ProwlerThreatScoreAlibaba, + ), + ], } diff --git a/api/src/backend/tasks/jobs/integrations.py b/api/src/backend/tasks/jobs/integrations.py index 71d2185deb..cd76762a40 100644 --- a/api/src/backend/tasks/jobs/integrations.py +++ b/api/src/backend/tasks/jobs/integrations.py @@ -19,6 +19,9 @@ from prowler.providers.aws.aws_provider import AwsProvider from prowler.providers.aws.lib.s3.s3 import S3 from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub from prowler.providers.common.models import Connection +from prowler.providers.aws.lib.security_hub.exceptions.exceptions import ( + SecurityHubNoEnabledRegionsError, +) logger = get_task_logger(__name__) @@ -222,8 +225,9 @@ def get_security_hub_client_from_integration( ) return True, security_hub else: - # Reset regions information if connection fails + # Reset regions information if connection fails and integration is not connected with rls_transaction(tenant_id, using=MainRouter.default_db): + integration.connected = False integration.configuration["regions"] = {} integration.save() @@ -330,15 +334,18 @@ def upload_security_hub_integration( ) if not connected: - logger.error( - f"Security Hub connection failed for integration {integration.id}: " - f"{security_hub.error}" - ) - with rls_transaction( - tenant_id, using=MainRouter.default_db + if isinstance( + security_hub.error, + SecurityHubNoEnabledRegionsError, ): - integration.connected = False - integration.save() + logger.warning( + f"Security Hub integration {integration.id} has no enabled regions" + ) + else: + logger.error( + f"Security Hub connection failed for integration {integration.id}: " + f"{security_hub.error}" + ) break # Skip this integration security_hub_client = security_hub @@ -409,22 +416,16 @@ def upload_security_hub_integration( logger.warning( f"Failed to archive previous findings: {str(archive_error)}" ) - except Exception as e: logger.error( f"Security Hub integration {integration.id} failed: {str(e)}" ) - continue result = integration_executions == len(integrations) if result: logger.info( f"All Security Hub integrations completed successfully for provider {provider_id}" ) - else: - logger.error( - f"Some Security Hub integrations failed for provider {provider_id}" - ) return result diff --git a/api/src/backend/tasks/jobs/lighthouse_providers.py b/api/src/backend/tasks/jobs/lighthouse_providers.py index 53f9e4dd97..29e36e5e57 100644 --- a/api/src/backend/tasks/jobs/lighthouse_providers.py +++ b/api/src/backend/tasks/jobs/lighthouse_providers.py @@ -2,6 +2,8 @@ from typing import Dict import boto3 import openai +from botocore import UNSIGNED +from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError from celery.utils.log import get_task_logger @@ -9,6 +11,74 @@ from api.models import LighthouseProviderConfiguration, LighthouseProviderModels logger = get_task_logger(__name__) +# OpenAI model prefixes to exclude from Lighthouse model selection. +# These models don't support text chat completions and tool calling. +EXCLUDED_OPENAI_MODEL_PREFIXES = ( + "dall-e", # Image generation + "whisper", # Audio transcription + "tts-", # Text-to-speech (tts-1, tts-1-hd, etc.) + "sora", # Text-to-video (sora-2, sora-2-pro, etc.) + "text-embedding", # Embeddings + "embedding", # Embeddings (alternative naming) + "text-moderation", # Content moderation + "omni-moderation", # Content moderation + "text-davinci", # Legacy completion models + "text-curie", # Legacy completion models + "text-babbage", # Legacy completion models + "text-ada", # Legacy completion models + "davinci", # Legacy completion models + "curie", # Legacy completion models + "babbage", # Legacy completion models + "ada", # Legacy completion models + "computer-use", # Computer control agent + "gpt-image", # Image generation + "gpt-audio", # Audio models + "gpt-realtime", # Realtime voice API +) + +# OpenAI model substrings to exclude (patterns that can appear anywhere in model ID). +# These patterns identify non-chat model variants. +EXCLUDED_OPENAI_MODEL_SUBSTRINGS = ( + "-audio-", # Audio preview models (gpt-4o-audio-preview, etc.) + "-realtime-", # Realtime preview models (gpt-4o-realtime-preview, etc.) + "-transcribe", # Transcription models (gpt-4o-transcribe, etc.) + "-tts", # TTS models (gpt-4o-mini-tts) + "-instruct", # Legacy instruct models (gpt-3.5-turbo-instruct, etc.) +) + + +def _extract_error_message(e: Exception) -> str: + """ + Extract a user-friendly error message from various exception types. + + This function handles exceptions from different providers (OpenAI, AWS Bedrock) + and extracts the most relevant error message for display to users. + + Args: + e: The exception to extract a message from. + + Returns: + str: A user-friendly error message. + """ + # For OpenAI SDK errors (>= v1.0) + # OpenAI exceptions have a 'body' attribute with error details + if hasattr(e, "body") and isinstance(e.body, dict): + if "message" in e.body: + return e.body["message"] + # Sometimes nested under 'error' key + if "error" in e.body and isinstance(e.body["error"], dict): + return e.body["error"].get("message", str(e)) + + # For boto3 ClientError + # Boto3 exceptions have a 'response' attribute with error details + if hasattr(e, "response") and isinstance(e.response, dict): + error_info = e.response.get("Error", {}) + if error_info.get("Message"): + return error_info["Message"] + + # Fallback to string representation for unknown error types + return str(e) + def _extract_openai_api_key( provider_cfg: LighthouseProviderConfiguration, @@ -56,21 +126,39 @@ def _extract_bedrock_credentials( """ Safely extract AWS Bedrock credentials from a provider configuration. + Supports two authentication methods: + 1. AWS access key + secret key + region + 2. Bedrock API key (bearer token) + region + Args: provider_cfg (LighthouseProviderConfiguration): The provider configuration instance containing the credentials. Returns: - Dict[str, str] | None: Dictionary with 'access_key_id', 'secret_access_key', and - 'region' if present and valid, otherwise None. + Dict[str, str] | None: Dictionary with either: + - 'access_key_id', 'secret_access_key', and 'region' for access key auth + - 'api_key' and 'region' for API key (bearer token) auth + Returns None if credentials are invalid or missing. """ creds = provider_cfg.credentials_decoded if not isinstance(creds, dict): return None + region = creds.get("region") + if not isinstance(region, str) or not region: + return None + + # Check for API key authentication first + api_key = creds.get("api_key") + if isinstance(api_key, str) and api_key: + return { + "api_key": api_key, + "region": region, + } + + # Fall back to access key authentication access_key_id = creds.get("access_key_id") secret_access_key = creds.get("secret_access_key") - region = creds.get("region") # Validate all required fields are present and are strings if ( @@ -78,8 +166,6 @@ def _extract_bedrock_credentials( or not access_key_id or not isinstance(secret_access_key, str) or not secret_access_key - or not isinstance(region, str) - or not region ): return None @@ -90,6 +176,51 @@ def _extract_bedrock_credentials( } +def _create_bedrock_client( + bedrock_creds: Dict[str, str], service_name: str = "bedrock" +): + """ + Create a boto3 Bedrock client with the appropriate authentication method. + + Supports two authentication methods: + 1. API key (bearer token) - uses unsigned requests with Authorization header + 2. AWS access key + secret key - uses standard SigV4 signing + + Args: + bedrock_creds: Dictionary with either: + - 'api_key' and 'region' for API key (bearer token) auth + - 'access_key_id', 'secret_access_key', and 'region' for access key auth + service_name: The Bedrock service name. Use 'bedrock' for control plane + operations (list_foundation_models, etc.) or 'bedrock-runtime' for + inference operations. + + Returns: + boto3 client configured for the specified Bedrock service. + """ + region = bedrock_creds["region"] + + if "api_key" in bedrock_creds: + bearer_token = bedrock_creds["api_key"] + client = boto3.client( + service_name=service_name, + region_name=region, + config=Config(signature_version=UNSIGNED), + ) + + def inject_bearer_token(request, **kwargs): + request.headers["Authorization"] = f"Bearer {bearer_token}" + + client.meta.events.register("before-send.*.*", inject_bearer_token) + return client + + return boto3.client( + service_name=service_name, + region_name=region, + aws_access_key_id=bedrock_creds["access_key_id"], + aws_secret_access_key=bedrock_creds["secret_access_key"], + ) + + def check_lighthouse_provider_connection(provider_config_id: str) -> Dict: """ Validate a Lighthouse provider configuration by calling the provider API and @@ -141,12 +272,7 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> Dict: } # Test connection by listing foundation models - bedrock_client = boto3.client( - "bedrock", - aws_access_key_id=bedrock_creds["access_key_id"], - aws_secret_access_key=bedrock_creds["secret_access_key"], - region_name=bedrock_creds["region"], - ) + bedrock_client = _create_bedrock_client(bedrock_creds) _ = bedrock_client.list_foundation_models() elif ( @@ -179,32 +305,54 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> Dict: return {"connected": True, "error": None} except Exception as e: + error_message = _extract_error_message(e) logger.warning( - "%s connection check failed: %s", provider_cfg.provider_type, str(e) + "%s connection check failed: %s", provider_cfg.provider_type, error_message ) provider_cfg.is_active = False provider_cfg.save() - return {"connected": False, "error": str(e)} + return {"connected": False, "error": error_message} def _fetch_openai_models(api_key: str) -> Dict[str, str]: """ Fetch available models from OpenAI API. + Filters out models that don't support text input/output and tool calling, + such as image generation (DALL-E), audio transcription (Whisper), + text-to-speech (TTS), embeddings, and moderation models. + Args: api_key: OpenAI API key for authentication. Returns: Dict mapping model_id to model_name. For OpenAI, both are the same - as the API doesn't provide separate display names. + as the API doesn't provide separate display names. Only includes + models that support text input, text output or tool calling. Raises: Exception: If the API call fails. """ client = openai.OpenAI(api_key=api_key) models = client.models.list() - # OpenAI uses model.id for both ID and display name - return {m.id: m.id for m in getattr(models, "data", [])} + + # Filter models to only include those supporting chat completions + tool calling + filtered_models = {} + for model in getattr(models, "data", []): + model_id = model.id + + # Skip if model ID starts with excluded prefixes + if model_id.startswith(EXCLUDED_OPENAI_MODEL_PREFIXES): + continue + + # Skip if model ID contains excluded substrings + if any(substring in model_id for substring in EXCLUDED_OPENAI_MODEL_SUBSTRINGS): + continue + + # Include model (supports chat completions + tool calling) + filtered_models[model_id] = model_id + + return filtered_models def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, str]: @@ -232,105 +380,219 @@ def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, st return available_models -def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]: +def _get_region_prefix(region: str) -> str: """ - Fetch available models from AWS Bedrock with entitlement verification. + Determine geographic prefix for AWS region. - This function: - 1. Lists foundation models with TEXT modality support - 2. Lists inference profiles with TEXT modality support - 3. Verifies user has entitlement access to each model + Examples: ap-south-1 -> apac, us-east-1 -> us, eu-west-1 -> eu + """ + if region.startswith(("us-", "ca-", "sa-")): + return "us" + elif region.startswith("eu-"): + return "eu" + elif region.startswith("ap-"): + return "apac" + return "global" - Args: - bedrock_creds: Dictionary with 'access_key_id', 'secret_access_key', and 'region'. + +def _clean_inference_profile_name(profile_name: str) -> str: + """ + Remove geographic prefix from inference profile name. + + AWS includes geographic prefixes in profile names which are redundant + since the profile ID already contains this information. + + Examples: + "APAC Anthropic Claude 3.5 Sonnet" -> "Anthropic Claude 3.5 Sonnet" + "GLOBAL Claude Sonnet 4.5" -> "Claude Sonnet 4.5" + "US Anthropic Claude 3 Haiku" -> "Anthropic Claude 3 Haiku" + """ + prefixes = ["APAC ", "GLOBAL ", "US ", "EU ", "APAC-", "GLOBAL-", "US-", "EU-"] + + for prefix in prefixes: + if profile_name.upper().startswith(prefix.upper()): + return profile_name[len(prefix) :].strip() + + return profile_name + + +def _supports_text_modality(input_modalities: list, output_modalities: list) -> bool: + """Check if model supports TEXT for both input and output.""" + return "TEXT" in input_modalities and "TEXT" in output_modalities + + +def _get_foundation_model_modalities( + bedrock_client, model_id: str +) -> tuple[list, list] | None: + """ + Fetch input and output modalities for a foundation model. Returns: - Dict mapping model_id to model_name for all accessible models. - - Raises: - BotoCoreError, ClientError: If AWS API calls fail. + (input_modalities, output_modalities) or None if fetch fails """ - bedrock_client = boto3.client( - "bedrock", - aws_access_key_id=bedrock_creds["access_key_id"], - aws_secret_access_key=bedrock_creds["secret_access_key"], - region_name=bedrock_creds["region"], - ) + try: + model_info = bedrock_client.get_foundation_model(modelIdentifier=model_id) + model_details = model_info.get("modelDetails", {}) + input_mods = model_details.get("inputModalities", []) + output_mods = model_details.get("outputModalities", []) + return (input_mods, output_mods) + except (BotoCoreError, ClientError) as e: + logger.debug("Could not fetch model details for %s: %s", model_id, str(e)) + return None - models_to_check: Dict[str, str] = {} - # Step 1: Get foundation models with TEXT modality +def _extract_foundation_model_ids(profile_models: list) -> list[str]: + """ + Extract foundation model IDs from inference profile model ARNs. + + Args: + profile_models: List of model references from inference profile + + Returns: + List of foundation model IDs extracted from ARNs + """ + model_ids = [] + for model_ref in profile_models: + model_arn = model_ref.get("modelArn", "") + if "foundation-model/" in model_arn: + model_id = model_arn.split("foundation-model/")[1] + model_ids.append(model_id) + return model_ids + + +def _build_inference_profile_map( + bedrock_client, region: str +) -> Dict[str, tuple[str, str]]: + """ + Build map of foundation_model_id -> best inference profile. + + Returns: + Dict mapping foundation_model_id to (profile_id, profile_name) + Only includes profiles with TEXT modality support + Prefers region-matched profiles over others + """ + region_prefix = _get_region_prefix(region) + model_to_profile: Dict[str, tuple[str, str]] = {} + + try: + response = bedrock_client.list_inference_profiles() + profiles = response.get("inferenceProfileSummaries", []) + + for profile in profiles: + profile_id = profile.get("inferenceProfileId") + profile_name = profile.get("inferenceProfileName") + + if not profile_id or not profile_name: + continue + + profile_models = profile.get("models", []) + if not profile_models: + continue + + foundation_model_ids = _extract_foundation_model_ids(profile_models) + if not foundation_model_ids: + continue + + modalities = _get_foundation_model_modalities( + bedrock_client, foundation_model_ids[0] + ) + if not modalities: + continue + + input_mods, output_mods = modalities + if not _supports_text_modality(input_mods, output_mods): + continue + + is_preferred = profile_id.startswith(f"{region_prefix}.") + clean_name = _clean_inference_profile_name(profile_name) + + for foundation_model_id in foundation_model_ids: + if foundation_model_id not in model_to_profile: + model_to_profile[foundation_model_id] = (profile_id, clean_name) + elif is_preferred and not model_to_profile[foundation_model_id][ + 0 + ].startswith(f"{region_prefix}."): + model_to_profile[foundation_model_id] = (profile_id, clean_name) + + except (BotoCoreError, ClientError) as e: + logger.info("Could not fetch inference profiles in %s: %s", region, str(e)) + + return model_to_profile + + +def _check_on_demand_availability(bedrock_client, model_id: str) -> bool: + """Check if an ON_DEMAND foundation model is entitled and available.""" + try: + availability = bedrock_client.get_foundation_model_availability( + modelId=model_id + ) + entitlement = availability.get("entitlementAvailability") + return entitlement == "AVAILABLE" + except (BotoCoreError, ClientError) as e: + logger.debug("Could not check availability for %s: %s", model_id, str(e)) + return False + + +def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]: + """ + Fetch available models from AWS Bedrock, preferring inference profiles over ON_DEMAND. + + Strategy: + 1. Build map of foundation_model -> best_inference_profile (with TEXT validation) + 2. For each TEXT-capable foundation model: + - Use inference profile ID if available (preferred - better throughput) + - Fallback to foundation model ID if only ON_DEMAND available + 3. Verify entitlement for ON_DEMAND models + + Args: + bedrock_creds: Dict with 'region' and auth credentials + + Returns: + Dict mapping model_id to model_name. IDs can be: + - Inference profile IDs (e.g., "apac.anthropic.claude-3-5-sonnet-20240620-v1:0") + - Foundation model IDs (e.g., "anthropic.claude-3-5-sonnet-20240620-v1:0") + """ + bedrock_client = _create_bedrock_client(bedrock_creds) + region = bedrock_creds["region"] + + model_to_profile = _build_inference_profile_map(bedrock_client, region) + foundation_response = bedrock_client.list_foundation_models() model_summaries = foundation_response.get("modelSummaries", []) - for model in model_summaries: - # Check if model supports TEXT input and output modality - input_modalities = model.get("inputModalities", []) - output_modalities = model.get("outputModalities", []) + models_to_return: Dict[str, str] = {} + on_demand_models: set[str] = set() - if "TEXT" not in input_modalities or "TEXT" not in output_modalities: + for model in model_summaries: + input_mods = model.get("inputModalities", []) + output_mods = model.get("outputModalities", []) + + if not _supports_text_modality(input_mods, output_mods): continue model_id = model.get("modelId") - if not model_id: + model_name = model.get("modelName") + + if not model_id or not model_name: continue - inference_types = model.get("inferenceTypesSupported", []) + if model_id in model_to_profile: + profile_id, profile_name = model_to_profile[model_id] + models_to_return[profile_id] = profile_name + else: + inference_types = model.get("inferenceTypesSupported", []) + if "ON_DEMAND" in inference_types: + models_to_return[model_id] = model_name + on_demand_models.add(model_id) - # Only include models with ON_DEMAND inference support - if "ON_DEMAND" in inference_types: - models_to_check[model_id] = model["modelName"] - - # Step 2: Get inference profiles - try: - inference_profiles_response = bedrock_client.list_inference_profiles() - inference_profiles = inference_profiles_response.get( - "inferenceProfileSummaries", [] - ) - - for profile in inference_profiles: - # Check if profile supports TEXT modality - input_modalities = profile.get("inputModalities", []) - output_modalities = profile.get("outputModalities", []) - - if "TEXT" not in input_modalities or "TEXT" not in output_modalities: - continue - - profile_id = profile.get("inferenceProfileId") - if profile_id: - models_to_check[profile_id] = profile["inferenceProfileName"] - - except (BotoCoreError, ClientError) as e: - logger.info( - "Could not fetch inference profiles in %s: %s", - bedrock_creds["region"], - str(e), - ) - - # Step 3: Verify entitlement availability for each model available_models: Dict[str, str] = {} - for model_id, model_name in models_to_check.items(): - try: - availability = bedrock_client.get_foundation_model_availability( - modelId=model_id - ) - - entitlement = availability.get("entitlementAvailability") - - # Only include models user has access to - if entitlement == "AVAILABLE": + for model_id, model_name in models_to_return.items(): + if model_id in on_demand_models: + if _check_on_demand_availability(bedrock_client, model_id): available_models[model_id] = model_name - else: - logger.debug( - "Skipping model %s - entitlement status: %s", model_id, entitlement - ) - - except (BotoCoreError, ClientError) as e: - logger.debug( - "Could not check availability for model %s: %s", model_id, str(e) - ) - continue + else: + available_models[model_id] = model_name return available_models @@ -359,7 +621,6 @@ def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict: provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id) fetched_models: Dict[str, str] = {} - # Fetch models from the appropriate provider try: if ( provider_cfg.provider_type @@ -414,12 +675,13 @@ def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict: } except Exception as e: + error_message = _extract_error_message(e) logger.warning( "Unexpected error refreshing %s models: %s", provider_cfg.provider_type, - str(e), + error_message, ) - return {"created": 0, "updated": 0, "deleted": 0, "error": str(e)} + return {"created": 0, "updated": 0, "deleted": 0, "error": error_message} # Upsert models into the catalog created = 0 diff --git a/api/src/backend/tasks/jobs/queries.py b/api/src/backend/tasks/jobs/queries.py new file mode 100644 index 0000000000..9fbe4901b2 --- /dev/null +++ b/api/src/backend/tasks/jobs/queries.py @@ -0,0 +1,134 @@ +""" +Shared SQL queries for tasks. + +This module centralizes raw SQL queries used across multiple task modules +to ensure consistency and maintainability. +""" + +# ============================================================================= +# COMPLIANCE SCORE QUERIES +# ============================================================================= + +# Upsert provider compliance scores from a scan's compliance requirements. +# Uses FAIL-dominant aggregation: FAIL > MANUAL > PASS +# Parameters: [tenant_id, scan_id] +COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL = """ + INSERT INTO provider_compliance_scores + (id, tenant_id, provider_id, scan_id, compliance_id, requirement_id, + requirement_status, scan_completed_at) + SELECT + gen_random_uuid(), + agg.tenant_id, + agg.provider_id, + agg.scan_id, + agg.compliance_id, + agg.requirement_id, + agg.requirement_status, + agg.completed_at + FROM ( + SELECT DISTINCT ON (cro.compliance_id, cro.requirement_id) + cro.tenant_id, + s.provider_id, + cro.scan_id, + cro.compliance_id, + cro.requirement_id, + (CASE + WHEN bool_or(cro.requirement_status = 'FAIL') + OVER (PARTITION BY cro.compliance_id, cro.requirement_id) THEN 'FAIL' + WHEN bool_or(cro.requirement_status = 'MANUAL') + OVER (PARTITION BY cro.compliance_id, cro.requirement_id) THEN 'MANUAL' + ELSE 'PASS' + END)::status as requirement_status, + s.completed_at + FROM compliance_requirements_overviews cro + JOIN scans s ON s.id = cro.scan_id + WHERE cro.tenant_id = %s AND cro.scan_id = %s + ORDER BY cro.compliance_id, cro.requirement_id + ) agg + ON CONFLICT (tenant_id, provider_id, compliance_id, requirement_id) + DO UPDATE SET + requirement_status = EXCLUDED.requirement_status, + scan_id = EXCLUDED.scan_id, + scan_completed_at = EXCLUDED.scan_completed_at + WHERE EXCLUDED.scan_completed_at > provider_compliance_scores.scan_completed_at +""" + +# Upsert tenant compliance summary for specific compliance IDs. +# Aggregates across all providers with FAIL-dominant logic at requirement level. +# Parameters: [tenant_id, tenant_id, compliance_ids_array] +COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL = """ + INSERT INTO tenant_compliance_summaries + (id, tenant_id, compliance_id, + requirements_passed, requirements_failed, requirements_manual, + total_requirements, updated_at) + SELECT + gen_random_uuid(), + %s as tenant_id, + compliance_id, + COUNT(*) FILTER (WHERE req_status = 'PASS') as requirements_passed, + COUNT(*) FILTER (WHERE req_status = 'FAIL') as requirements_failed, + COUNT(*) FILTER (WHERE req_status = 'MANUAL') as requirements_manual, + COUNT(*) as total_requirements, + NOW() as updated_at + FROM ( + SELECT + compliance_id, + requirement_id, + CASE + WHEN bool_or(requirement_status = 'FAIL') THEN 'FAIL' + WHEN bool_or(requirement_status = 'MANUAL') THEN 'MANUAL' + ELSE 'PASS' + END as req_status + FROM provider_compliance_scores + WHERE tenant_id = %s AND compliance_id = ANY(%s) + GROUP BY compliance_id, requirement_id + ) req_agg + GROUP BY compliance_id + ON CONFLICT (tenant_id, compliance_id) + DO UPDATE SET + requirements_passed = EXCLUDED.requirements_passed, + requirements_failed = EXCLUDED.requirements_failed, + requirements_manual = EXCLUDED.requirements_manual, + total_requirements = EXCLUDED.total_requirements, + updated_at = NOW() +""" + +# Upsert tenant compliance summary for ALL compliance IDs in tenant. +# Used by backfill when recalculating entire tenant summary. +# Parameters: [tenant_id, tenant_id] +COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL = """ + INSERT INTO tenant_compliance_summaries + (id, tenant_id, compliance_id, + requirements_passed, requirements_failed, requirements_manual, + total_requirements, updated_at) + SELECT + gen_random_uuid(), + %s as tenant_id, + compliance_id, + COUNT(*) FILTER (WHERE req_status = 'PASS') as requirements_passed, + COUNT(*) FILTER (WHERE req_status = 'FAIL') as requirements_failed, + COUNT(*) FILTER (WHERE req_status = 'MANUAL') as requirements_manual, + COUNT(*) as total_requirements, + NOW() as updated_at + FROM ( + SELECT + compliance_id, + requirement_id, + CASE + WHEN bool_or(requirement_status = 'FAIL') THEN 'FAIL' + WHEN bool_or(requirement_status = 'MANUAL') THEN 'MANUAL' + ELSE 'PASS' + END as req_status + FROM provider_compliance_scores + WHERE tenant_id = %s + GROUP BY compliance_id, requirement_id + ) req_agg + GROUP BY compliance_id + ON CONFLICT (tenant_id, compliance_id) + DO UPDATE SET + requirements_passed = EXCLUDED.requirements_passed, + requirements_failed = EXCLUDED.requirements_failed, + requirements_manual = EXCLUDED.requirements_manual, + total_requirements = EXCLUDED.total_requirements, + updated_at = NOW() +""" diff --git a/api/src/backend/tasks/jobs/report.py b/api/src/backend/tasks/jobs/report.py index 76a6527560..1fbf9161d6 100644 --- a/api/src/backend/tasks/jobs/report.py +++ b/api/src/backend/tasks/jobs/report.py @@ -243,15 +243,28 @@ def _safe_getattr(obj, attr: str, default: str = "N/A") -> str: def _create_info_table_style() -> TableStyle: - """Create a reusable table style for information/metadata tables.""" + """Create a reusable table style for information/metadata tables. + + ReportLab TableStyle coordinate system: + - Format: (COMMAND, (start_col, start_row), (end_col, end_row), value) + - Coordinates use (column, row) format, starting at (0, 0) for top-left cell + - Negative indices work like Python slicing: -1 means "last row/column" + - (0, 0) to (0, -1) = entire first column (all rows) + - (0, 0) to (-1, 0) = entire first row (all columns) + - (0, 0) to (-1, -1) = entire table + - Styles are applied in order; later rules override earlier ones + """ return TableStyle( [ + # Column 0 (labels): blue background with white text ("BACKGROUND", (0, 0), (0, -1), COLOR_BLUE), ("TEXTCOLOR", (0, 0), (0, -1), COLOR_WHITE), ("FONTNAME", (0, 0), (0, -1), "FiraCode"), + # Column 1 (values): light blue background with gray text ("BACKGROUND", (1, 0), (1, -1), COLOR_BG_BLUE), ("TEXTCOLOR", (1, 0), (1, -1), COLOR_GRAY), ("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"), + # Apply to entire table ("ALIGN", (0, 0), (-1, -1), "LEFT"), ("VALIGN", (0, 0), (-1, -1), "TOP"), ("FONTSIZE", (0, 0), (-1, -1), 11), @@ -265,19 +278,30 @@ def _create_info_table_style() -> TableStyle: def _create_header_table_style(header_color: colors.Color = None) -> TableStyle: - """Create a reusable table style for tables with headers.""" + """Create a reusable table style for tables with headers. + + ReportLab TableStyle coordinate system: + - Format: (COMMAND, (start_col, start_row), (end_col, end_row), value) + - (0, 0) to (-1, 0) = entire first row (header row) + - (1, 1) to (-1, -1) = all data cells (excludes header row and first column) + - See _create_info_table_style() for full coordinate system documentation + """ if header_color is None: header_color = COLOR_BLUE return TableStyle( [ + # Header row (row 0): colored background with white text ("BACKGROUND", (0, 0), (-1, 0), header_color), ("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE), ("FONTNAME", (0, 0), (-1, 0), "FiraCode"), ("FONTSIZE", (0, 0), (-1, 0), 10), + # Apply to entire table ("ALIGN", (0, 0), (-1, -1), "CENTER"), ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + # Data cells (excluding header): smaller font ("FONTSIZE", (1, 1), (-1, -1), 9), + # Apply to entire table ("GRID", (0, 0), (-1, -1), 1, COLOR_GRID_GRAY), ("LEFTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM), ("RIGHTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM), @@ -288,18 +312,30 @@ def _create_header_table_style(header_color: colors.Color = None) -> TableStyle: def _create_findings_table_style() -> TableStyle: - """Create a reusable table style for findings tables.""" + """Create a reusable table style for findings tables. + + ReportLab TableStyle coordinate system: + - Format: (COMMAND, (start_col, start_row), (end_col, end_row), value) + - (0, 0) to (-1, 0) = entire first row (header row) + - (0, 0) to (0, 0) = only the top-left cell + - See _create_info_table_style() for full coordinate system documentation + """ return TableStyle( [ + # Header row (row 0): colored background with white text ("BACKGROUND", (0, 0), (-1, 0), COLOR_BLUE), ("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE), ("FONTNAME", (0, 0), (-1, 0), "FiraCode"), + # Only top-left cell centered (for index/number column) ("ALIGN", (0, 0), (0, 0), "CENTER"), + # Apply to entire table ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), ("FONTSIZE", (0, 0), (-1, -1), 9), ("GRID", (0, 0), (-1, -1), 0.1, COLOR_BORDER_GRAY), + # Remove padding only from top-left cell ("LEFTPADDING", (0, 0), (0, 0), 0), ("RIGHTPADDING", (0, 0), (0, 0), 0), + # Apply to entire table ("TOPPADDING", (0, 0), (-1, -1), PADDING_SMALL), ("BOTTOMPADDING", (0, 0), (-1, -1), PADDING_SMALL), ] @@ -1103,11 +1139,15 @@ def generate_threatscore_report( elements.append(Spacer(1, 0.5 * inch)) # Add compliance information table + provider_alias = provider_obj.alias or "N/A" info_data = [ ["Framework:", compliance_framework], ["ID:", compliance_id], ["Name:", Paragraph(compliance_name, normal_center)], ["Version:", compliance_version], + ["Provider:", provider_type.upper()], + ["Account ID:", provider_obj.uid], + ["Alias:", provider_alias], ["Scan ID:", scan_id], ["Description:", Paragraph(compliance_description, normal_center)], ] @@ -2059,12 +2099,15 @@ def generate_ens_report( elements.append(Spacer(1, 0.5 * inch)) # Add compliance information table + provider_alias = provider_obj.alias or "N/A" info_data = [ ["Framework:", compliance_framework], ["ID:", compliance_id], ["Nombre:", Paragraph(compliance_name, normal_center)], ["Versión:", compliance_version], ["Proveedor:", provider_type.upper()], + ["Account ID:", provider_obj.uid], + ["Alias:", provider_alias], ["Scan ID:", scan_id], ["Descripción:", Paragraph(compliance_description, normal_center)], ] @@ -2072,12 +2115,12 @@ def generate_ens_report( info_table.setStyle( TableStyle( [ - ("BACKGROUND", (0, 0), (0, 6), colors.Color(0.2, 0.4, 0.6)), - ("TEXTCOLOR", (0, 0), (0, 6), colors.white), - ("FONTNAME", (0, 0), (0, 6), "FiraCode"), - ("BACKGROUND", (1, 0), (1, 6), colors.Color(0.95, 0.97, 1.0)), - ("TEXTCOLOR", (1, 0), (1, 6), colors.Color(0.2, 0.2, 0.2)), - ("FONTNAME", (1, 0), (1, 6), "PlusJakartaSans"), + ("BACKGROUND", (0, 0), (0, -1), colors.Color(0.2, 0.4, 0.6)), + ("TEXTCOLOR", (0, 0), (0, -1), colors.white), + ("FONTNAME", (0, 0), (0, -1), "FiraCode"), + ("BACKGROUND", (1, 0), (1, -1), colors.Color(0.95, 0.97, 1.0)), + ("TEXTCOLOR", (1, 0), (1, -1), colors.Color(0.2, 0.2, 0.2)), + ("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"), ("ALIGN", (0, 0), (-1, -1), "LEFT"), ("VALIGN", (0, 0), (-1, -1), "TOP"), ("FONTSIZE", (0, 0), (-1, -1), 11), @@ -2238,12 +2281,20 @@ def generate_ens_report( [ "CUMPLE", str(passed_requirements), - f"{(passed_requirements / total_requirements * 100):.1f}%", + ( + f"{(passed_requirements / total_requirements * 100):.1f}%" + if total_requirements > 0 + else "0.0%" + ), ], [ "NO CUMPLE", str(failed_requirements), - f"{(failed_requirements / total_requirements * 100):.1f}%", + ( + f"{(failed_requirements / total_requirements * 100):.1f}%" + if total_requirements > 0 + else "0.0%" + ), ], ["TOTAL", str(total_requirements), "100%"], ] @@ -2989,11 +3040,14 @@ def generate_nis2_report( elements.append(Spacer(1, 0.3 * inch)) # Compliance metadata table + provider_alias = provider_obj.alias or "N/A" metadata_data = [ ["Framework:", compliance_framework], ["Name:", Paragraph(compliance_name, normal_center)], ["Version:", compliance_version or "N/A"], ["Provider:", provider_type.upper()], + ["Account ID:", provider_obj.uid], + ["Alias:", provider_alias], ["Scan ID:", scan_id], ["Description:", Paragraph(compliance_description, normal_center)], ] @@ -3477,6 +3531,7 @@ def generate_compliance_reports( "gcp", "m365", "kubernetes", + "alibabacloud", ]: logger.info( f"Provider {provider_id} ({provider_type}) is not supported for ThreatScore report" diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index c4cf413301..9e40e2df03 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -8,11 +8,16 @@ from collections import defaultdict from datetime import datetime, timezone from typing import Any +import sentry_sdk from celery.utils.log import get_task_logger from config.env import env from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS from django.db import IntegrityError, OperationalError -from django.db.models import Case, Count, IntegerField, Prefetch, Sum, When +from django.db.models import Case, Count, IntegerField, Prefetch, Q, Sum, When +from tasks.jobs.queries import ( + COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL, + COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL, +) from tasks.utils import CustomEncoder from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE @@ -26,8 +31,10 @@ from api.db_utils import ( ) from api.exceptions import ProviderConnectionError from api.models import ( + AttackSurfaceOverview, ComplianceOverviewSummary, ComplianceRequirementOverview, + DailySeveritySummary, Finding, MuteRule, Processor, @@ -37,12 +44,14 @@ from api.models import ( ResourceScanSummary, ResourceTag, Scan, + ScanCategorySummary, ScanSummary, StateChoices, ) from api.models import StatusChoices as FindingStatus from api.utils import initialize_prowler_provider, return_prowler_provider from api.v1.serializers import ScanTaskSerializer +from prowler.lib.check.models import CheckMetadata from prowler.lib.outputs.finding import Finding as ProwlerFinding from prowler.lib.scan.scan import Scan as ProwlerScan @@ -74,6 +83,77 @@ FINDINGS_MICRO_BATCH_SIZE = env.int("DJANGO_FINDINGS_MICRO_BATCH_SIZE", default= # Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=500) +ATTACK_SURFACE_PROVIDER_COMPATIBILITY = { + "internet-exposed": None, # Compatible with all providers + "secrets": None, # Compatible with all providers + "privilege-escalation": ["aws", "kubernetes"], + "ec2-imdsv1": ["aws"], +} + +_ATTACK_SURFACE_MAPPING_CACHE: dict[str, dict] = {} + + +def aggregate_category_counts( + categories: list[str], + severity: str, + status: str, + delta: str | None, + muted: bool, + cache: dict[tuple[str, str], dict[str, int]], +) -> None: + """ + Increment category counters in-place for a finding. + + Args: + categories: List of categories from finding metadata. + severity: Severity level (e.g., "high", "medium"). + status: Finding status as string ("FAIL", "PASS"). + delta: Delta value as string ("new", "changed") or None. + muted: Whether the finding is muted. + cache: Dict {(category, severity): {"total", "failed", "new_failed"}} to update. + """ + is_failed = status == "FAIL" and not muted + is_new_failed = is_failed and delta == "new" + + for cat in categories: + key = (cat, severity) + if key not in cache: + cache[key] = {"total": 0, "failed": 0, "new_failed": 0} + if not muted: + cache[key]["total"] += 1 + if is_failed: + cache[key]["failed"] += 1 + if is_new_failed: + cache[key]["new_failed"] += 1 + + +def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict: + global _ATTACK_SURFACE_MAPPING_CACHE + + if provider_type in _ATTACK_SURFACE_MAPPING_CACHE: + return _ATTACK_SURFACE_MAPPING_CACHE[provider_type] + + attack_surface_check_mappings = { + "internet-exposed": None, + "secrets": None, + "privilege-escalation": { + "iam_policy_allows_privilege_escalation", + "iam_inline_policy_allows_privilege_escalation", + }, + "ec2-imdsv1": { + "ec2_instance_imdsv2_enabled" + }, # AWS only - IMDSv1 enabled findings + } + for category_name, check_ids in attack_surface_check_mappings.items(): + if check_ids is None: + sdk_check_ids = CheckMetadata.list( + provider=provider_type, category=category_name + ) + attack_surface_check_mappings[category_name] = sdk_check_ids + + _ATTACK_SURFACE_MAPPING_CACHE[provider_type] = attack_surface_check_mappings + return attack_surface_check_mappings + def _create_finding_delta( last_status: FindingStatus | None | str, new_status: FindingStatus | None @@ -330,7 +410,7 @@ def _create_compliance_summaries( if summary_objects: with rls_transaction(tenant_id): ComplianceOverviewSummary.objects.bulk_create( - summary_objects, batch_size=500 + summary_objects, batch_size=500, ignore_conflicts=True ) @@ -357,6 +437,7 @@ def _process_finding_micro_batch( unique_resources: set, scan_resource_cache: set, mute_rules_cache: dict, + scan_categories_cache: dict[tuple[str, str], dict[str, int]], ) -> None: """ Process a micro-batch of findings and persist them using bulk operations. @@ -377,6 +458,7 @@ def _process_finding_micro_batch( unique_resources: Set tracking (uid, region) pairs seen in the scan. scan_resource_cache: Set of tuples used to create `ResourceScanSummary` rows. mute_rules_cache: Map of finding UID -> mute reason gathered before the scan. + scan_categories_cache: Dict tracking category counts {(category, severity): {"total", "failed", "new_failed"}}. """ # Accumulate objects for bulk operations findings_to_create = [] @@ -532,11 +614,12 @@ def _process_finding_micro_batch( resource_failed_findings_cache[resource_uid] += 1 # Create finding object (don't save yet) + check_metadata = finding.get_metadata() finding_instance = Finding( tenant_id=tenant_id, uid=finding_uid, delta=delta, - check_metadata=finding.get_metadata(), + check_metadata=check_metadata, status=status, status_extended=finding.status_extended, severity=finding.severity, @@ -549,6 +632,7 @@ def _process_finding_micro_batch( muted_at=datetime.now(tz=timezone.utc) if is_muted else None, muted_reason=muted_reason, compliance=finding.compliance, + categories=check_metadata.get("categories", []) or [], ) findings_to_create.append(finding_instance) resource_denormalized_data.append((finding_instance, resource_instance)) @@ -563,6 +647,16 @@ def _process_finding_micro_batch( ) ) + # Track categories with counts for ScanCategorySummary by (category, severity) + aggregate_category_counts( + categories=check_metadata.get("categories", []) or [], + severity=finding.severity.value, + status=status.value, + delta=delta.value if delta else None, + muted=is_muted, + cache=scan_categories_cache, + ) + # Bulk operations within single transaction with rls_transaction(tenant_id): # Bulk create findings @@ -662,6 +756,7 @@ def perform_prowler_scan( exception = None unique_resources = set() scan_resource_cache: set[tuple[str, str, str, str]] = set() + scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {} start_time = time.time() exc = None @@ -751,6 +846,7 @@ def perform_prowler_scan( unique_resources=unique_resources, scan_resource_cache=scan_resource_cache, mute_rules_cache=mute_rules_cache, + scan_categories_cache=scan_categories_cache, ) # Update scan progress @@ -810,13 +906,33 @@ def perform_prowler_scan( resource_scan_summaries, batch_size=500, ignore_conflicts=True ) except Exception as filter_exception: - import sentry_sdk - sentry_sdk.capture_exception(filter_exception) logger.error( f"Error storing filter values for scan {scan_id}: {filter_exception}" ) + try: + if scan_categories_cache: + category_summaries = [ + ScanCategorySummary( + tenant_id=tenant_id, + scan_id=scan_id, + category=category, + severity=severity, + total_findings=counts["total"], + failed_findings=counts["failed"], + new_failed_findings=counts["new_failed"], + ) + for (category, severity), counts in scan_categories_cache.items() + ] + with rls_transaction(tenant_id): + ScanCategorySummary.objects.bulk_create( + category_summaries, batch_size=500, ignore_conflicts=True + ) + except Exception as cat_exception: + sentry_sdk.capture_exception(cat_exception) + logger.error(f"Error storing categories for scan {scan_id}: {cat_exception}") + serializer = ScanTaskSerializer(instance=scan_instance) return serializer.data @@ -979,11 +1095,14 @@ def _aggregate_findings_by_region( findings_count_by_compliance = {} with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): - # Fetch findings with resources in a single efficient query - # Use select_related for finding fields and prefetch_related for many-to-many resources + # Fetch only PASS/FAIL findings (optimized query reduces data transfer) + # Other statuses are not needed for check_status or ThreatScore calculation findings = ( Finding.all_objects.filter( - tenant_id=tenant_id, scan_id=scan_id, muted=False + tenant_id=tenant_id, + scan_id=scan_id, + muted=False, + status__in=["PASS", "FAIL"], ) .only("id", "check_id", "status", "compliance") .prefetch_related( @@ -1001,6 +1120,8 @@ def _aggregate_findings_by_region( ) for finding in findings: + status = finding.status + for resource in finding.small_resources: region = resource.region @@ -1008,7 +1129,7 @@ def _aggregate_findings_by_region( current_status = check_status_by_region.setdefault(region, {}) # Priority: FAIL > any other status if current_status.get(finding.check_id) != "FAIL": - current_status[finding.check_id] = finding.status + current_status[finding.check_id] = status # Aggregate ThreatScore compliance counts if modeled_threatscore_compliance_id in (finding.compliance or {}): @@ -1023,7 +1144,7 @@ def _aggregate_findings_by_region( requirement_id, {"total": 0, "pass": 0} ) requirement_stats["total"] += 1 - if finding.status == "PASS": + if status == "PASS": requirement_stats["pass"] += 1 return check_status_by_region, findings_count_by_compliance @@ -1191,3 +1312,321 @@ def create_compliance_requirements(tenant_id: str, scan_id: str): except Exception as e: logger.error(f"Error creating compliance requirements for scan {scan_id}: {e}") raise e + + +def aggregate_attack_surface(tenant_id: str, scan_id: str): + """ + Aggregate findings into attack surface overview records. + + Creates one AttackSurfaceOverview record per attack surface type + for the given scan, based on check_id mappings. + + Args: + tenant_id: Tenant that owns the scan. + scan_id: Scan UUID whose findings should be aggregated. + """ + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + scan_instance = Scan.all_objects.select_related("provider").get(pk=scan_id) + provider_type = scan_instance.provider.provider + + provider_attack_surface_mapping = _get_attack_surface_mapping_from_provider( + provider_type=provider_type + ) + + # Filter out attack surfaces that are not compatible or have no resolved check IDs + supported_mappings: dict[str, list[str]] = {} + for attack_surface_type, check_ids in provider_attack_surface_mapping.items(): + compatible_providers = ATTACK_SURFACE_PROVIDER_COMPATIBILITY.get( + attack_surface_type + ) + if ( + compatible_providers is not None + and provider_type not in compatible_providers + ): + logger.info( + f"Skipping {attack_surface_type} - not supported for {provider_type}" + ) + continue + + if not check_ids: + logger.info( + f"Skipping {attack_surface_type} - no check IDs resolved for {provider_type}" + ) + continue + + supported_mappings[attack_surface_type] = list(check_ids) + + if not supported_mappings: + logger.info( + f"No attack surface mappings available for scan {scan_id} and provider {provider_type}" + ) + logger.info(f"No attack surface overview records created for scan {scan_id}") + return + + # Map every check_id to its attack surface, so we can aggregate with a single query + check_id_to_surface: dict[str, str] = {} + for attack_surface_type, check_ids in supported_mappings.items(): + for check_id in check_ids: + check_id_to_surface[check_id] = attack_surface_type + + aggregated_counts = { + attack_surface_type: {"total": 0, "failed": 0, "muted": 0} + for attack_surface_type in supported_mappings.keys() + } + + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + finding_stats = ( + Finding.all_objects.filter( + tenant_id=tenant_id, + scan_id=scan_id, + check_id__in=list(check_id_to_surface.keys()), + ) + .values("check_id") + .annotate( + total=Count("id"), + failed=Count("id", filter=Q(status="FAIL", muted=False)), + muted=Count("id", filter=Q(status="FAIL", muted=True)), + ) + ) + + for stats in finding_stats: + attack_surface_type = check_id_to_surface.get(stats["check_id"]) + if not attack_surface_type: + continue + + aggregated_counts[attack_surface_type]["total"] += stats["total"] or 0 + aggregated_counts[attack_surface_type]["failed"] += stats["failed"] or 0 + aggregated_counts[attack_surface_type]["muted"] += stats["muted"] or 0 + + overview_objects = [] + for attack_surface_type, counts in aggregated_counts.items(): + total = counts["total"] + if not total: + continue + + overview_objects.append( + AttackSurfaceOverview( + tenant_id=tenant_id, + scan_id=scan_id, + attack_surface_type=attack_surface_type, + total_findings=total, + failed_findings=counts["failed"], + muted_failed_findings=counts["muted"], + ) + ) + + # Bulk create overview records + if overview_objects: + with rls_transaction(tenant_id): + AttackSurfaceOverview.objects.bulk_create(overview_objects, batch_size=500) + logger.info( + f"Created {len(overview_objects)} attack surface overview records for scan {scan_id}" + ) + else: + logger.info(f"No attack surface overview records created for scan {scan_id}") + + +def aggregate_daily_severity(tenant_id: str, scan_id: str): + """Aggregate scan severity counts into DailySeveritySummary (one record per provider/day).""" + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + scan = Scan.objects.filter( + tenant_id=tenant_id, + id=scan_id, + state=StateChoices.COMPLETED, + ).first() + + if not scan: + logger.warning(f"Scan {scan_id} not found or not completed") + return {"status": "scan is not completed"} + + provider_id = scan.provider_id + scan_date = scan.completed_at.date() + + severity_totals = ( + ScanSummary.objects.filter( + tenant_id=tenant_id, + scan_id=scan_id, + ) + .values("severity") + .annotate(total_fail=Sum("fail"), total_muted=Sum("muted")) + ) + + severity_data = { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "informational": 0, + "muted": 0, + } + + for row in severity_totals: + severity = row["severity"] + if severity in severity_data: + severity_data[severity] = row["total_fail"] or 0 + severity_data["muted"] += row["total_muted"] or 0 + + with rls_transaction(tenant_id): + summary, created = DailySeveritySummary.objects.update_or_create( + tenant_id=tenant_id, + provider_id=provider_id, + date=scan_date, + defaults={ + "scan_id": scan_id, + "critical": severity_data["critical"], + "high": severity_data["high"], + "medium": severity_data["medium"], + "low": severity_data["low"], + "informational": severity_data["informational"], + "muted": severity_data["muted"], + }, + ) + + action = "created" if created else "updated" + logger.info( + f"Daily severity summary {action} for provider {provider_id} on {scan_date}" + ) + + return { + "status": action, + "provider_id": str(provider_id), + "date": str(scan_date), + "severity_data": severity_data, + } + + +def update_provider_compliance_scores(tenant_id: str, scan_id: str): + """ + Update ProviderComplianceScore with requirement statuses from a completed scan. + + Uses atomic SQL upsert with ON CONFLICT for concurrency safety. Only updates + if the new scan is more recent than existing data. Also cleans up stale + requirements that no longer exist in the new scan. + + Reads from primary DB (not replica) to avoid replication lag issues since + this runs immediately after create_compliance_requirements_task. + + Args: + tenant_id: Tenant that owns the scan. + scan_id: Scan UUID whose compliance data should be materialized. + + Returns: + dict: Statistics about the upsert operation. + """ + with rls_transaction(tenant_id): + scan = ( + Scan.all_objects.filter( + tenant_id=tenant_id, + id=scan_id, + state=StateChoices.COMPLETED, + ) + .select_related("provider") + .first() + ) + + if not scan: + logger.warning( + f"Scan {scan_id} not found or not completed for compliance score update" + ) + return {"status": "skipped", "reason": "scan not completed"} + + if not scan.completed_at: + logger.warning(f"Scan {scan_id} has no completed_at timestamp") + return {"status": "skipped", "reason": "no completed_at"} + + provider_id = str(scan.provider_id) + scan_completed_at = scan.completed_at + + delete_stale_sql = """ + DELETE FROM provider_compliance_scores pcs + WHERE pcs.tenant_id = %s + AND pcs.provider_id = %s + AND pcs.scan_completed_at < %s + AND NOT EXISTS ( + SELECT 1 FROM compliance_requirements_overviews cro + WHERE cro.tenant_id = pcs.tenant_id + AND cro.scan_id = %s + AND cro.compliance_id = pcs.compliance_id + AND cro.requirement_id = pcs.requirement_id + ) + RETURNING compliance_id + """ + + compliance_ids_sql = """ + SELECT DISTINCT compliance_id + FROM compliance_requirements_overviews + WHERE tenant_id = %s AND scan_id = %s + """ + + try: + with psycopg_connection(MainRouter.default_db) as connection: + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id]) + + # Update requirement-level scores per provider + cursor.execute( + COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL, [tenant_id, scan_id] + ) + upserted_count = cursor.rowcount + + cursor.execute(compliance_ids_sql, [tenant_id, scan_id]) + scan_rows = cursor.fetchall() + if not isinstance(scan_rows, (list, tuple)): + scan_rows = [] + scan_compliance_ids = {row[0] for row in scan_rows} + + cursor.execute( + delete_stale_sql, + [tenant_id, provider_id, scan_completed_at, scan_id], + ) + deleted_rows = cursor.fetchall() + if not isinstance(deleted_rows, (list, tuple)): + deleted_rows = [] + deleted_ids = {row[0] for row in deleted_rows} + stale_deleted = len(deleted_ids) + + impacted_compliance_ids = sorted(scan_compliance_ids | deleted_ids) + + if impacted_compliance_ids: + # Advisory lock on tenant to prevent race conditions when + # multiple scans complete simultaneously for the same tenant + cursor.execute( + "SELECT pg_advisory_xact_lock(hashtext(%s))", [tenant_id] + ) + + # Recalculate tenant-level summary (FAIL-dominant across all providers) + cursor.execute( + COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL, + [tenant_id, tenant_id, impacted_compliance_ids], + ) + tenant_summary_count = cursor.rowcount + else: + tenant_summary_count = 0 + + connection.commit() + except Exception: + connection.rollback() + raise + + logger.info( + f"Provider compliance scores updated for scan {scan_id}: " + f"{upserted_count} upserted, {stale_deleted} stale deleted, " + f"{tenant_summary_count} tenant summaries upserted" + ) + + return { + "status": "completed", + "scan_id": str(scan_id), + "provider_id": provider_id, + "upserted": upserted_count, + "stale_deleted": stale_deleted, + "tenant_summary_count": tenant_summary_count, + } + + except Exception as e: + logger.error( + f"Error updating provider compliance scores for scan {scan_id}: {e}" + ) + raise diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index 0221996e1b..4e97c581f7 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -11,7 +11,7 @@ from django_celery_beat.models import PeriodicTask from api.compliance import get_compliance_frameworks from api.db_router import READ_REPLICA_ALIAS from api.db_utils import rls_transaction -from api.decorators import set_tenant +from api.decorators import handle_provider_deletion, set_tenant from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices from api.utils import initialize_prowler_provider from api.v1.serializers import ScanTaskSerializer @@ -23,7 +23,10 @@ from prowler.lib.outputs.finding import Finding as FindingOutput from tasks.jobs.attack_paths import attack_paths_scan from tasks.jobs.backfill import ( backfill_compliance_summaries, + backfill_daily_severity_summaries, + backfill_provider_compliance_scores, backfill_resource_scan_summaries, + backfill_scan_category_summaries, ) from tasks.jobs.connection import ( check_integration_connection, @@ -50,15 +53,70 @@ from tasks.jobs.lighthouse_providers import ( from tasks.jobs.muting import mute_historical_findings from tasks.jobs.report import generate_compliance_reports_job from tasks.jobs.scan import ( + aggregate_attack_surface, + aggregate_daily_severity, aggregate_findings, create_compliance_requirements, perform_prowler_scan, + update_provider_compliance_scores, ) from tasks.utils import batched, get_next_execution_datetime logger = get_task_logger(__name__) +def _cleanup_orphan_scheduled_scans( + tenant_id: str, + provider_id: str, + scheduler_task_id: int, +) -> int: + """ + TEMPORARY WORKAROUND: Clean up orphan AVAILABLE scans. + + Detects and removes AVAILABLE scans that were never used due to an + issue during the first scheduled scan setup. + + An AVAILABLE scan is considered orphan if there's also a SCHEDULED scan for + the same provider with the same scheduler_task_id. This situation indicates + that the first scan execution didn't find the AVAILABLE scan (because it + wasn't committed yet, probably) and created a new one, leaving the AVAILABLE orphaned. + + Args: + tenant_id: The tenant ID. + provider_id: The provider ID. + scheduler_task_id: The PeriodicTask ID that triggers these scans. + + Returns: + Number of orphan scans deleted (0 if none found). + """ + orphan_available_scans = Scan.objects.filter( + tenant_id=tenant_id, + provider_id=provider_id, + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=scheduler_task_id, + ) + + scheduled_scan_exists = Scan.objects.filter( + tenant_id=tenant_id, + provider_id=provider_id, + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.SCHEDULED, + scheduler_task_id=scheduler_task_id, + ).exists() + + if scheduled_scan_exists and orphan_available_scans.exists(): + orphan_count = orphan_available_scans.count() + logger.warning( + f"[WORKAROUND] Found {orphan_count} orphan AVAILABLE scan(s) for " + f"provider {provider_id} alongside a SCHEDULED scan. Cleaning up orphans..." + ) + orphan_available_scans.delete() + return orphan_count + + return 0 + + def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str): """ Helper function to perform tasks after a scan is completed. @@ -68,13 +126,20 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str) scan_id (str): The ID of the scan that was performed. provider_id (str): The primary key of the Provider instance that was scanned. """ - create_compliance_requirements_task.apply_async( + chain( + create_compliance_requirements_task.si(tenant_id=tenant_id, scan_id=scan_id), + update_provider_compliance_scores_task.si(tenant_id=tenant_id, scan_id=scan_id), + ).apply_async() + aggregate_attack_surface_task.apply_async( kwargs={"tenant_id": tenant_id, "scan_id": scan_id} ) chain( perform_scan_summary_task.si(tenant_id=tenant_id, scan_id=scan_id), - generate_outputs_task.si( - scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id + group( + aggregate_daily_severity_task.si(tenant_id=tenant_id, scan_id=scan_id), + generate_outputs_task.si( + scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id + ), ), group( # Use optimized task that generates both reports with shared queries @@ -145,6 +210,7 @@ def delete_provider_task(provider_id: str, tenant_id: str): @shared_task(base=RLSTask, name="scan-perform", queue="scans") +@handle_provider_deletion def perform_scan_task( tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None ): @@ -177,6 +243,7 @@ def perform_scan_task( @shared_task(base=RLSTask, bind=True, name="scan-perform-scheduled", queue="scans") +@handle_provider_deletion def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str): """ Task to perform a scheduled Prowler scan on a given provider. @@ -240,6 +307,14 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str): return serializer.data next_scan_datetime = get_next_execution_datetime(task_id, provider_id) + + # TEMPORARY WORKAROUND: Clean up orphan scans from transaction isolation issue + _cleanup_orphan_scheduled_scans( + tenant_id=tenant_id, + provider_id=provider_id, + scheduler_task_id=periodic_task_instance.id, + ) + scan_instance, _ = Scan.objects.get_or_create( tenant_id=tenant_id, provider_id=provider_id, @@ -282,6 +357,7 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str): @shared_task(name="scan-summary", queue="overview") +@handle_provider_deletion def perform_scan_summary_task(tenant_id: str, scan_id: str): return aggregate_findings(tenant_id=tenant_id, scan_id=scan_id) @@ -316,6 +392,7 @@ def delete_tenant_task(tenant_id: str): queue="scan-reports", ) @set_tenant(keep_tenant=True) +@handle_provider_deletion def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str): """ Process findings in batches and generate output files in multiple formats. @@ -511,6 +588,7 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str): @shared_task(name="backfill-scan-resource-summaries", queue="backfill") +@handle_provider_deletion def backfill_scan_resource_summaries_task(tenant_id: str, scan_id: str): """ Tries to backfill the resource scan summaries table for a given scan. @@ -523,6 +601,7 @@ def backfill_scan_resource_summaries_task(tenant_id: str, scan_id: str): @shared_task(name="backfill-compliance-summaries", queue="backfill") +@handle_provider_deletion def backfill_compliance_summaries_task(tenant_id: str, scan_id: str): """ Tries to backfill compliance overview summaries for a completed scan. @@ -537,7 +616,43 @@ def backfill_compliance_summaries_task(tenant_id: str, scan_id: str): return backfill_compliance_summaries(tenant_id=tenant_id, scan_id=scan_id) +@shared_task(name="backfill-daily-severity-summaries", queue="backfill") +def backfill_daily_severity_summaries_task(tenant_id: str, days: int = None): + """Backfill DailySeveritySummary from historical scans. Use days param to limit scope.""" + return backfill_daily_severity_summaries(tenant_id=tenant_id, days=days) + + +@shared_task(name="backfill-scan-category-summaries", queue="backfill") +@handle_provider_deletion +def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str): + """ + Backfill ScanCategorySummary for a completed scan. + + Aggregates unique categories from findings and creates a summary row. + + Args: + tenant_id (str): The tenant identifier. + scan_id (str): The scan identifier. + """ + return backfill_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id) + + +@shared_task(name="backfill-provider-compliance-scores", queue="backfill") +def backfill_provider_compliance_scores_task(tenant_id: str): + """ + Backfill ProviderComplianceScore from latest completed scan per provider. + + Used to populate the compliance watchlist materialized table for tenants + that had scans before the feature was deployed. + + Args: + tenant_id: Target tenant UUID. + """ + return backfill_provider_compliance_scores(tenant_id=tenant_id) + + @shared_task(base=RLSTask, name="scan-compliance-overviews", queue="compliance") +@handle_provider_deletion def create_compliance_requirements_task(tenant_id: str, scan_id: str): """ Creates detailed compliance requirement records for a scan. @@ -553,6 +668,44 @@ def create_compliance_requirements_task(tenant_id: str, scan_id: str): return create_compliance_requirements(tenant_id=tenant_id, scan_id=scan_id) +@shared_task(name="scan-attack-surface-overviews", queue="overview") +@handle_provider_deletion +def aggregate_attack_surface_task(tenant_id: str, scan_id: str): + """ + Creates attack surface overview records for a scan. + + This task processes findings and aggregates them into attack surface categories + (internet-exposed, secrets, privilege-escalation, ec2-imdsv1) for quick overview queries. + + Args: + tenant_id (str): The tenant ID for which to create records. + scan_id (str): The ID of the scan for which to create records. + """ + return aggregate_attack_surface(tenant_id=tenant_id, scan_id=scan_id) + + +@shared_task(name="scan-provider-compliance-scores", queue="compliance") +def update_provider_compliance_scores_task(tenant_id: str, scan_id: str): + """ + Update provider compliance scores from a completed scan. + + This task materializes compliance requirement statuses into ProviderComplianceScore + for efficient watchlist queries. Uses atomic upsert with concurrency protection. + + Args: + tenant_id (str): The tenant ID for which to update scores. + scan_id (str): The ID of the scan whose data should be materialized. + """ + return update_provider_compliance_scores(tenant_id=tenant_id, scan_id=scan_id) + + +@shared_task(name="scan-daily-severity", queue="overview") +@handle_provider_deletion +def aggregate_daily_severity_task(tenant_id: str, scan_id: str): + """Aggregate scan severity into DailySeveritySummary for findings_severity/timeseries endpoint.""" + return aggregate_daily_severity(tenant_id=tenant_id, scan_id=scan_id) + + @shared_task(base=RLSTask, name="lighthouse-connection-check") @set_tenant def check_lighthouse_connection_task(lighthouse_config_id: str, tenant_id: str = None): @@ -591,6 +744,7 @@ def refresh_lighthouse_provider_models_task( @shared_task(name="integration-check") +@handle_provider_deletion def check_integrations_task(tenant_id: str, provider_id: str, scan_id: str = None): """ Check and execute all configured integrations for a provider. @@ -655,6 +809,7 @@ def check_integrations_task(tenant_id: str, provider_id: str, scan_id: str = Non name="integration-s3", queue="integrations", ) +@handle_provider_deletion def s3_integration_task( tenant_id: str, provider_id: str, @@ -714,6 +869,7 @@ def jira_integration_task( name="scan-compliance-reports", queue="scan-reports", ) +@handle_provider_deletion def generate_compliance_reports_task(tenant_id: str, scan_id: str, provider_id: str): """ Optimized task to generate ThreatScore, ENS, and NIS2 reports with shared queries. diff --git a/api/src/backend/tasks/tests/test_backfill.py b/api/src/backend/tasks/tests/test_backfill.py index 13fa481b2e..4c9780d101 100644 --- a/api/src/backend/tasks/tests/test_backfill.py +++ b/api/src/backend/tasks/tests/test_backfill.py @@ -1,17 +1,25 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch from uuid import uuid4 import pytest from tasks.jobs.backfill import ( backfill_compliance_summaries, + backfill_provider_compliance_scores, backfill_resource_scan_summaries, + backfill_scan_category_summaries, ) from api.models import ( ComplianceOverviewSummary, + Finding, ResourceScanSummary, Scan, + ScanCategorySummary, StateChoices, ) +from prowler.lib.check.models import Severity +from prowler.lib.outputs.finding import Status @pytest.fixture(scope="function") @@ -46,6 +54,45 @@ def get_not_completed_scans(providers_fixture): return scan_1, scan_2 +@pytest.fixture(scope="function") +def findings_with_categories_fixture(scans_fixture, resources_fixture): + scan = scans_fixture[0] + resource = resources_fixture[0] + + finding = Finding.objects.create( + tenant_id=scan.tenant_id, + uid="finding_with_categories", + scan=scan, + delta="new", + status=Status.FAIL, + status_extended="test status", + impact=Severity.critical, + impact_extended="test impact", + severity=Severity.critical, + raw_result={"status": Status.FAIL}, + check_id="test_check", + check_metadata={"CheckId": "test_check"}, + categories=["gen-ai", "security"], + first_seen_at="2024-01-02T00:00:00Z", + ) + finding.add_resources([resource]) + return finding + + +@pytest.fixture(scope="function") +def scan_category_summary_fixture(scans_fixture): + scan = scans_fixture[0] + return ScanCategorySummary.objects.create( + tenant_id=scan.tenant_id, + scan=scan, + category="existing-category", + severity=Severity.critical, + total_findings=1, + failed_findings=0, + new_failed_findings=0, + ) + + @pytest.mark.django_db class TestBackfillResourceScanSummaries: def test_already_backfilled(self, resource_scan_summary_data): @@ -172,3 +219,106 @@ class TestBackfillComplianceSummaries: assert summary.requirements_failed == expected_counts["requirements_failed"] assert summary.requirements_manual == expected_counts["requirements_manual"] assert summary.total_requirements == expected_counts["total_requirements"] + + +@pytest.mark.django_db +class TestBackfillScanCategorySummaries: + def test_already_backfilled(self, scan_category_summary_fixture): + tenant_id = scan_category_summary_fixture.tenant_id + scan_id = scan_category_summary_fixture.scan_id + + result = backfill_scan_category_summaries(str(tenant_id), str(scan_id)) + + assert result == {"status": "already backfilled"} + + def test_not_completed_scan(self, get_not_completed_scans): + for scan in get_not_completed_scans: + result = backfill_scan_category_summaries(str(scan.tenant_id), str(scan.id)) + assert result == {"status": "scan is not completed"} + + def test_no_categories_to_backfill(self, scans_fixture): + scan = scans_fixture[1] # Failed scan with no findings + result = backfill_scan_category_summaries(str(scan.tenant_id), str(scan.id)) + assert result == {"status": "no categories to backfill"} + + def test_successful_backfill(self, findings_with_categories_fixture): + finding = findings_with_categories_fixture + tenant_id = str(finding.tenant_id) + scan_id = str(finding.scan_id) + + result = backfill_scan_category_summaries(tenant_id, scan_id) + + # 2 categories × 1 severity = 2 rows + assert result == {"status": "backfilled", "categories_count": 2} + + summaries = ScanCategorySummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ) + assert summaries.count() == 2 + categories = set(summaries.values_list("category", flat=True)) + assert categories == {"gen-ai", "security"} + + for summary in summaries: + assert summary.severity == Severity.critical + assert summary.total_findings == 1 + assert summary.failed_findings == 1 + assert summary.new_failed_findings == 1 + + +@pytest.mark.django_db +class TestBackfillProviderComplianceScores: + def test_no_completed_scans(self, tenants_fixture): + tenant = tenants_fixture[2] + result = backfill_provider_compliance_scores(str(tenant.id)) + assert result == {"status": "no completed scans"} + + def test_no_scans_to_process(self, tenants_fixture, scans_fixture): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + scan.completed_at = None + scan.save() + + result = backfill_provider_compliance_scores(str(tenant.id)) + assert result == {"status": "no completed scans"} + + @patch("tasks.jobs.backfill.psycopg_connection") + def test_successful_backfill_executes_sql_queries( + self, + mock_psycopg_connection, + tenants_fixture, + scans_fixture, + settings, + ): + """Test successful backfill executes SQL queries and returns correct stats.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + # Set completed_at to make the scan eligible for backfill + scan.completed_at = datetime.now(timezone.utc) + scan.save() + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + connection.autocommit = True + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + cursor.rowcount = 5 + + result = backfill_provider_compliance_scores(str(tenant.id)) + + assert result["status"] == "backfilled" + assert result["providers_processed"] == 1 + assert result["providers_skipped"] == 0 + assert result["total_upserted"] == 5 + assert result["tenant_summary_count"] == 5 diff --git a/api/src/backend/tasks/tests/test_beat.py b/api/src/backend/tasks/tests/test_beat.py index 7a1656553f..5c25e97340 100644 --- a/api/src/backend/tasks/tests/test_beat.py +++ b/api/src/backend/tasks/tests/test_beat.py @@ -28,6 +28,7 @@ class TestScheduleProviderScan: "tenant_id": str(provider_instance.tenant_id), "provider_id": str(provider_instance.id), }, + countdown=5, ) task_name = f"scan-perform-scheduled-{provider_instance.id}" diff --git a/api/src/backend/tasks/tests/test_integrations.py b/api/src/backend/tasks/tests/test_integrations.py index 75f07677a0..d37b27e320 100644 --- a/api/src/backend/tasks/tests/test_integrations.py +++ b/api/src/backend/tasks/tests/test_integrations.py @@ -1199,9 +1199,6 @@ class TestSecurityHubIntegrationUploads: ) assert result is False - # Integration should be marked as disconnected - integration.save.assert_called_once() - assert integration.connected is False @patch("tasks.jobs.integrations.ASFF") @patch("tasks.jobs.integrations.FindingOutput") diff --git a/api/src/backend/tasks/tests/test_scan.py b/api/src/backend/tasks/tests/test_scan.py index f2724de4e6..8902b17b54 100644 --- a/api/src/backend/tasks/tests/test_scan.py +++ b/api/src/backend/tasks/tests/test_scan.py @@ -9,17 +9,22 @@ from unittest.mock import MagicMock, patch import pytest from tasks.jobs.scan import ( + _ATTACK_SURFACE_MAPPING_CACHE, _aggregate_findings_by_region, _copy_compliance_requirement_rows, _create_compliance_summaries, _create_finding_delta, + _get_attack_surface_mapping_from_provider, _normalized_compliance_key, _persist_compliance_requirement_rows, _process_finding_micro_batch, _store_resources, + aggregate_attack_surface, + aggregate_category_counts, aggregate_findings, create_compliance_requirements, perform_prowler_scan, + update_provider_compliance_scores, ) from tasks.utils import CustomEncoder @@ -1374,6 +1379,7 @@ class TestProcessFindingMicroBatch: unique_resources: set[tuple[str, str]] = set() scan_resource_cache: set[tuple[str, str, str, str]] = set() mute_rules_cache = {} + scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {} with ( patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction), @@ -1391,6 +1397,7 @@ class TestProcessFindingMicroBatch: unique_resources, scan_resource_cache, mute_rules_cache, + scan_categories_cache, ) created_finding = Finding.objects.get(uid=finding.uid) @@ -1483,6 +1490,7 @@ class TestProcessFindingMicroBatch: unique_resources: set[tuple[str, str]] = set() scan_resource_cache: set[tuple[str, str, str, str]] = set() mute_rules_cache = {finding.uid: "Muted via rule"} + scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {} with ( patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction), @@ -1500,6 +1508,7 @@ class TestProcessFindingMicroBatch: unique_resources, scan_resource_cache, mute_rules_cache, + scan_categories_cache, ) existing_resource.refresh_from_db() @@ -1607,6 +1616,7 @@ class TestProcessFindingMicroBatch: unique_resources: set[tuple[str, str]] = set() scan_resource_cache: set[tuple[str, str, str, str]] = set() mute_rules_cache = {} + scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {} with ( patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction), @@ -1625,6 +1635,7 @@ class TestProcessFindingMicroBatch: unique_resources, scan_resource_cache, mute_rules_cache, + scan_categories_cache, ) # Verify the long UID finding was NOT created @@ -1645,6 +1656,118 @@ class TestProcessFindingMicroBatch: for call in warning_calls ) + def test_process_finding_micro_batch_tracks_categories( + self, tenants_fixture, scans_fixture + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = scan.provider + + finding1 = FakeFinding( + uid="finding-cat-1", + status=StatusChoices.PASS, + status_extended="all good", + severity=Severity.low, + check_id="genai_check", + resource_uid="arn:aws:bedrock:::model/test", + resource_name="test-model", + region="us-east-1", + service_name="bedrock", + resource_type="model", + resource_tags={}, + resource_metadata={}, + resource_details={}, + partition="aws", + raw={}, + compliance={}, + metadata={"categories": ["gen-ai", "security"]}, + muted=False, + ) + + finding2 = FakeFinding( + uid="finding-cat-2", + status=StatusChoices.FAIL, + status_extended="bad", + severity=Severity.high, + check_id="iam_check", + resource_uid="arn:aws:iam:::user/test", + resource_name="test-user", + region="us-east-1", + service_name="iam", + resource_type="user", + resource_tags={}, + resource_metadata={}, + resource_details={}, + partition="aws", + raw={}, + compliance={}, + metadata={"categories": ["security", "iam"]}, + muted=False, + ) + + resource_cache = {} + tag_cache = {} + last_status_cache = {} + resource_failed_findings_cache = {} + unique_resources: set[tuple[str, str]] = set() + scan_resource_cache: set[tuple[str, str, str, str]] = set() + mute_rules_cache = {} + scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {} + + with ( + patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction), + patch("api.db_utils.rls_transaction", new=noop_rls_transaction), + ): + _process_finding_micro_batch( + str(tenant.id), + [finding1, finding2], + scan, + provider, + resource_cache, + tag_cache, + last_status_cache, + resource_failed_findings_cache, + unique_resources, + scan_resource_cache, + mute_rules_cache, + scan_categories_cache, + ) + + # finding1: PASS, severity=low, categories=["gen-ai", "security"] + # finding2: FAIL, severity=high, categories=["security", "iam"] + # Keys are (category, severity) tuples + assert set(scan_categories_cache.keys()) == { + ("gen-ai", "low"), + ("security", "low"), + ("security", "high"), + ("iam", "high"), + } + assert scan_categories_cache[("gen-ai", "low")] == { + "total": 1, + "failed": 0, + "new_failed": 0, + } + assert scan_categories_cache[("security", "low")] == { + "total": 1, + "failed": 0, + "new_failed": 0, + } + assert scan_categories_cache[("security", "high")] == { + "total": 1, + "failed": 1, + "new_failed": 1, + } + assert scan_categories_cache[("iam", "high")] == { + "total": 1, + "failed": 1, + "new_failed": 1, + } + + created_finding1 = Finding.objects.get(uid="finding-cat-1") + created_finding2 = Finding.objects.get(uid="finding-cat-2") + assert set(created_finding1.categories) == {"gen-ai", "security"} + assert set(created_finding2.categories) == {"security", "iam"} + @pytest.mark.django_db class TestCreateComplianceRequirements: @@ -3338,7 +3461,10 @@ class TestAggregateFindingsByRegion: # Verify filter was called with muted=False mock_findings_filter.assert_called_once_with( - tenant_id=tenant_id, scan_id=scan_id, muted=False + tenant_id=tenant_id, + scan_id=scan_id, + muted=False, + status__in=["PASS", "FAIL"], ) @patch("tasks.jobs.scan.Finding.all_objects.filter") @@ -3471,3 +3597,549 @@ class TestAggregateFindingsByRegion: assert check_status_by_region == {} assert findings_count_by_compliance == {} + + +@pytest.mark.django_db +class TestAggregateAttackSurface: + """Test aggregate_attack_surface function and related caching.""" + + def setup_method(self): + """Clear cache before each test.""" + _ATTACK_SURFACE_MAPPING_CACHE.clear() + + def teardown_method(self): + """Clear cache after each test.""" + _ATTACK_SURFACE_MAPPING_CACHE.clear() + + @patch("tasks.jobs.scan.CheckMetadata.list") + def test_get_attack_surface_mapping_caches_result(self, mock_check_metadata_list): + """Test that _get_attack_surface_mapping_from_provider caches results.""" + mock_check_metadata_list.return_value = {"check_internet_exposed_1"} + + # First call should hit CheckMetadata.list + result1 = _get_attack_surface_mapping_from_provider("aws") + assert mock_check_metadata_list.call_count == 2 # internet-exposed, secrets + + # Second call should use cache + result2 = _get_attack_surface_mapping_from_provider("aws") + assert mock_check_metadata_list.call_count == 2 # No additional calls + + assert result1 is result2 + assert "aws" in _ATTACK_SURFACE_MAPPING_CACHE + + @patch("tasks.jobs.scan.CheckMetadata.list") + def test_get_attack_surface_mapping_different_providers( + self, mock_check_metadata_list + ): + """Test caching works independently for different providers.""" + mock_check_metadata_list.return_value = {"check_1"} + + _get_attack_surface_mapping_from_provider("aws") + aws_call_count = mock_check_metadata_list.call_count + + _get_attack_surface_mapping_from_provider("gcp") + gcp_call_count = mock_check_metadata_list.call_count + + # Both providers should have made calls + assert gcp_call_count > aws_call_count + assert "aws" in _ATTACK_SURFACE_MAPPING_CACHE + assert "gcp" in _ATTACK_SURFACE_MAPPING_CACHE + + @patch("tasks.jobs.scan.CheckMetadata.list") + def test_get_attack_surface_mapping_returns_hardcoded_checks( + self, mock_check_metadata_list + ): + """Test that hardcoded check IDs are returned for privilege-escalation and ec2-imdsv1.""" + mock_check_metadata_list.return_value = set() + + result = _get_attack_surface_mapping_from_provider("aws") + + # Hardcoded checks should be present + assert ( + "iam_policy_allows_privilege_escalation" in result["privilege-escalation"] + ) + assert ( + "iam_inline_policy_allows_privilege_escalation" + in result["privilege-escalation"] + ) + assert "ec2_instance_imdsv2_enabled" in result["ec2-imdsv1"] + + @patch("tasks.jobs.scan.AttackSurfaceOverview.objects.bulk_create") + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan._get_attack_surface_mapping_from_provider") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_attack_surface_creates_overview_records( + self, + mock_rls_transaction, + mock_get_mapping, + mock_findings_filter, + mock_bulk_create, + tenants_fixture, + scans_fixture, + ): + """Test that aggregate_attack_surface creates AttackSurfaceOverview records.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + scan.provider.provider = "aws" + scan.provider.save() + + mock_get_mapping.return_value = { + "internet-exposed": {"check_internet_1", "check_internet_2"}, + "secrets": {"check_secrets_1"}, + "privilege-escalation": {"check_privesc_1"}, + "ec2-imdsv1": {"check_imdsv1_1"}, + } + + # Mock findings aggregation + mock_queryset = MagicMock() + mock_queryset.values.return_value = mock_queryset + mock_queryset.annotate.return_value = [ + {"check_id": "check_internet_1", "total": 10, "failed": 3, "muted": 1}, + {"check_id": "check_secrets_1", "total": 5, "failed": 2, "muted": 0}, + ] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + aggregate_attack_surface(str(tenant.id), str(scan.id)) + + mock_bulk_create.assert_called_once() + args, kwargs = mock_bulk_create.call_args + objects = args[0] + + # Should create records for internet-exposed and secrets (the ones with findings) + assert len(objects) == 2 + assert kwargs["batch_size"] == 500 + + @patch("tasks.jobs.scan.AttackSurfaceOverview.objects.bulk_create") + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan._get_attack_surface_mapping_from_provider") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_attack_surface_skips_unsupported_provider( + self, + mock_rls_transaction, + mock_get_mapping, + mock_findings_filter, + mock_bulk_create, + tenants_fixture, + scans_fixture, + ): + """Test that ec2-imdsv1 is skipped for non-AWS providers.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + scan.provider.provider = "gcp" + scan.provider.uid = "gcp-test-project-id" + scan.provider.save() + + mock_get_mapping.return_value = { + "internet-exposed": {"check_internet_1"}, + "secrets": {"check_secrets_1"}, + "privilege-escalation": set(), # Not supported for GCP + "ec2-imdsv1": {"check_imdsv1_1"}, # Should be skipped for GCP + } + + mock_queryset = MagicMock() + mock_queryset.values.return_value = mock_queryset + mock_queryset.annotate.return_value = [ + {"check_id": "check_internet_1", "total": 5, "failed": 1, "muted": 0}, + ] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + aggregate_attack_surface(str(tenant.id), str(scan.id)) + + # ec2-imdsv1 check_ids should not be in the filter + filter_call = mock_findings_filter.call_args + check_ids_in_filter = filter_call[1]["check_id__in"] + assert "check_imdsv1_1" not in check_ids_in_filter + + @patch("tasks.jobs.scan.AttackSurfaceOverview.objects.bulk_create") + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan._get_attack_surface_mapping_from_provider") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_attack_surface_no_findings( + self, + mock_rls_transaction, + mock_get_mapping, + mock_findings_filter, + mock_bulk_create, + tenants_fixture, + scans_fixture, + ): + """Test that no records are created when there are no findings.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + mock_get_mapping.return_value = { + "internet-exposed": {"check_1"}, + "secrets": {"check_2"}, + "privilege-escalation": set(), + "ec2-imdsv1": set(), + } + + mock_queryset = MagicMock() + mock_queryset.values.return_value = mock_queryset + mock_queryset.annotate.return_value = [] # No findings + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + aggregate_attack_surface(str(tenant.id), str(scan.id)) + + mock_bulk_create.assert_not_called() + + @patch("tasks.jobs.scan.AttackSurfaceOverview.objects.bulk_create") + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan._get_attack_surface_mapping_from_provider") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_attack_surface_aggregates_counts_correctly( + self, + mock_rls_transaction, + mock_get_mapping, + mock_findings_filter, + mock_bulk_create, + tenants_fixture, + scans_fixture, + ): + """Test that counts from multiple check_ids are aggregated per attack surface type.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + scan.provider.provider = "aws" + scan.provider.save() + + mock_get_mapping.return_value = { + "internet-exposed": {"check_internet_1", "check_internet_2"}, + "secrets": set(), + "privilege-escalation": set(), + "ec2-imdsv1": set(), + } + + mock_queryset = MagicMock() + mock_queryset.values.return_value = mock_queryset + mock_queryset.annotate.return_value = [ + {"check_id": "check_internet_1", "total": 10, "failed": 3, "muted": 1}, + {"check_id": "check_internet_2", "total": 5, "failed": 2, "muted": 0}, + ] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + aggregate_attack_surface(str(tenant.id), str(scan.id)) + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + + assert len(objects) == 1 + overview = objects[0] + assert overview.attack_surface_type == "internet-exposed" + assert overview.total_findings == 15 # 10 + 5 + assert overview.failed_findings == 5 # 3 + 2 + assert overview.muted_failed_findings == 1 # 1 + 0 + + @patch("tasks.jobs.scan.Scan.all_objects.select_related") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_attack_surface_uses_select_related( + self, mock_rls_transaction, mock_select_related, tenants_fixture, scans_fixture + ): + """Test that select_related is used to avoid N+1 query.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + mock_scan = MagicMock() + mock_scan.provider.provider = "aws" + + mock_select_related.return_value.get.return_value = mock_scan + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + with patch( + "tasks.jobs.scan._get_attack_surface_mapping_from_provider" + ) as mock_map: + mock_map.return_value = {} + + aggregate_attack_surface(str(tenant.id), str(scan.id)) + + mock_select_related.assert_called_once_with("provider") + + +class TestAggregateCategoryCounts: + """Test aggregate_category_counts helper function.""" + + def test_aggregate_category_counts_basic(self): + """Test basic category counting for a non-muted PASS finding.""" + cache: dict[tuple[str, str], dict[str, int]] = {} + aggregate_category_counts( + categories=["security", "iam"], + severity="high", + status="PASS", + delta=None, + muted=False, + cache=cache, + ) + + assert ("security", "high") in cache + assert ("iam", "high") in cache + assert cache[("security", "high")] == {"total": 1, "failed": 0, "new_failed": 0} + assert cache[("iam", "high")] == {"total": 1, "failed": 0, "new_failed": 0} + + def test_aggregate_category_counts_fail_not_muted(self): + """Test category counting for a non-muted FAIL finding.""" + cache: dict[tuple[str, str], dict[str, int]] = {} + aggregate_category_counts( + categories=["security"], + severity="critical", + status="FAIL", + delta=None, + muted=False, + cache=cache, + ) + + assert cache[("security", "critical")] == { + "total": 1, + "failed": 1, + "new_failed": 0, + } + + def test_aggregate_category_counts_new_fail(self): + """Test category counting for a new FAIL finding (delta='new').""" + cache: dict[tuple[str, str], dict[str, int]] = {} + aggregate_category_counts( + categories=["gen-ai"], + severity="high", + status="FAIL", + delta="new", + muted=False, + cache=cache, + ) + + assert cache[("gen-ai", "high")] == {"total": 1, "failed": 1, "new_failed": 1} + + def test_aggregate_category_counts_muted_finding(self): + """Test that muted findings are excluded from all counts.""" + cache: dict[tuple[str, str], dict[str, int]] = {} + aggregate_category_counts( + categories=["security"], + severity="high", + status="FAIL", + delta="new", + muted=True, + cache=cache, + ) + + assert cache[("security", "high")] == {"total": 0, "failed": 0, "new_failed": 0} + + def test_aggregate_category_counts_accumulates(self): + """Test that multiple calls accumulate counts.""" + cache: dict[tuple[str, str], dict[str, int]] = {} + + # First finding: PASS + aggregate_category_counts( + categories=["security"], + severity="high", + status="PASS", + delta=None, + muted=False, + cache=cache, + ) + + # Second finding: FAIL (new) + aggregate_category_counts( + categories=["security"], + severity="high", + status="FAIL", + delta="new", + muted=False, + cache=cache, + ) + + # Third finding: FAIL (changed) + aggregate_category_counts( + categories=["security"], + severity="high", + status="FAIL", + delta="changed", + muted=False, + cache=cache, + ) + + assert cache[("security", "high")] == {"total": 3, "failed": 2, "new_failed": 1} + + def test_aggregate_category_counts_empty_categories(self): + """Test with empty categories list.""" + cache: dict[tuple[str, str], dict[str, int]] = {} + aggregate_category_counts( + categories=[], + severity="high", + status="FAIL", + delta="new", + muted=False, + cache=cache, + ) + + assert cache == {} + + def test_aggregate_category_counts_changed_delta(self): + """Test that changed delta increments failed but not new_failed.""" + cache: dict[tuple[str, str], dict[str, int]] = {} + aggregate_category_counts( + categories=["iam"], + severity="medium", + status="FAIL", + delta="changed", + muted=False, + cache=cache, + ) + + assert cache[("iam", "medium")] == {"total": 1, "failed": 1, "new_failed": 0} + + def test_aggregate_category_counts_multiple_categories_single_finding(self): + """Test single finding with multiple categories.""" + cache: dict[tuple[str, str], dict[str, int]] = {} + aggregate_category_counts( + categories=["security", "compliance", "data-protection"], + severity="low", + status="FAIL", + delta="new", + muted=False, + cache=cache, + ) + + assert len(cache) == 3 + for cat in ["security", "compliance", "data-protection"]: + assert cache[(cat, "low")] == {"total": 1, "failed": 1, "new_failed": 1} + + +@pytest.mark.django_db +class TestUpdateProviderComplianceScores: + @patch("tasks.jobs.scan.psycopg_connection") + def test_update_provider_compliance_scores_basic( + self, + mock_psycopg_connection, + tenants_fixture, + scans_fixture, + settings, + ): + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + tenant = tenants_fixture[0] + scan = scans_fixture[0] + tenant_id = str(tenant.id) + scan_id = str(scan.id) + + scan.state = StateChoices.COMPLETED + scan.completed_at = datetime.now(timezone.utc) + scan.save() + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + connection.autocommit = True + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + cursor.rowcount = 2 + + result = update_provider_compliance_scores(tenant_id, scan_id) + + assert result["status"] == "completed" + assert result["upserted"] == 2 + assert cursor.execute.call_count >= 3 + connection.commit.assert_called_once() + + def test_update_provider_compliance_scores_skips_incomplete_scan( + self, tenants_fixture, scans_fixture + ): + tenant = tenants_fixture[0] + scan = scans_fixture[1] + tenant_id = str(tenant.id) + scan_id = str(scan.id) + + result = update_provider_compliance_scores(tenant_id, scan_id) + + assert result["status"] == "skipped" + assert result["reason"] == "scan not completed" + + def test_update_provider_compliance_scores_skips_no_completed_at( + self, tenants_fixture, scans_fixture + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + tenant_id = str(tenant.id) + scan_id = str(scan.id) + + scan.state = StateChoices.COMPLETED + scan.completed_at = None + scan.save() + + result = update_provider_compliance_scores(tenant_id, scan_id) + + assert result["status"] == "skipped" + assert result["reason"] == "no completed_at" + + @patch("tasks.jobs.scan.psycopg_connection") + def test_update_provider_compliance_scores_executes_sql_queries( + self, + mock_psycopg_connection, + tenants_fixture, + providers_fixture, + scans_fixture, + settings, + ): + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + tenant = tenants_fixture[0] + scan = scans_fixture[0] + tenant_id = str(tenant.id) + scan_id = str(scan.id) + + scan.state = StateChoices.COMPLETED + scan.completed_at = datetime.now(timezone.utc) + scan.save() + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + cursor.rowcount = 1 + cursor.fetchall.side_effect = [[("aws_cis_2.0",)], []] + + result = update_provider_compliance_scores(tenant_id, scan_id) + + assert result["status"] == "completed" + + calls = [str(c) for c in cursor.execute.call_args_list] + assert any("provider_compliance_scores" in c for c in calls) + assert any("tenant_compliance_summaries" in c for c in calls) + assert any("pg_advisory_xact_lock" in c for c in calls) diff --git a/api/src/backend/tasks/tests/test_tasks.py b/api/src/backend/tasks/tests/test_tasks.py index 98926937fe..bdda0f6bcb 100644 --- a/api/src/backend/tasks/tests/test_tasks.py +++ b/api/src/backend/tasks/tests/test_tasks.py @@ -7,13 +7,21 @@ import openai import pytest from botocore.exceptions import ClientError +from django_celery_beat.models import IntervalSchedule, PeriodicTask from api.models import ( Integration, LighthouseProviderConfiguration, LighthouseProviderModels, + Scan, + StateChoices, +) +from tasks.jobs.lighthouse_providers import ( + _create_bedrock_client, + _extract_bedrock_credentials, ) from tasks.tasks import ( + _cleanup_orphan_scheduled_scans, _perform_scan_complete_tasks, check_integrations_task, check_lighthouse_provider_connection_task, @@ -25,6 +33,198 @@ from tasks.tasks import ( ) +@pytest.mark.django_db +class TestExtractBedrockCredentials: + """Unit tests for _extract_bedrock_credentials helper function.""" + + def test_extract_access_key_credentials(self, tenants_fixture): + """Test extraction of access key + secret key credentials.""" + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + is_active=True, + ) + provider_cfg.credentials_decoded = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + } + provider_cfg.save() + + result = _extract_bedrock_credentials(provider_cfg) + + assert result is not None + assert result["access_key_id"] == "AKIAIOSFODNN7EXAMPLE" + assert result["secret_access_key"] == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + assert result["region"] == "us-east-1" + assert "api_key" not in result + + def test_extract_api_key_credentials(self, tenants_fixture): + """Test extraction of API key (bearer token) credentials.""" + valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110) + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + is_active=True, + ) + provider_cfg.credentials_decoded = { + "api_key": valid_api_key, + "region": "us-west-2", + } + provider_cfg.save() + + result = _extract_bedrock_credentials(provider_cfg) + + assert result is not None + assert result["api_key"] == valid_api_key + assert result["region"] == "us-west-2" + assert "access_key_id" not in result + assert "secret_access_key" not in result + + def test_api_key_takes_precedence_over_access_keys(self, tenants_fixture): + """Test that API key is preferred when both auth methods are present.""" + valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("B" * 110) + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + is_active=True, + ) + provider_cfg.credentials_decoded = { + "api_key": valid_api_key, + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "eu-west-1", + } + provider_cfg.save() + + result = _extract_bedrock_credentials(provider_cfg) + + assert result is not None + assert result["api_key"] == valid_api_key + assert result["region"] == "eu-west-1" + assert "access_key_id" not in result + + def test_missing_region_returns_none(self, tenants_fixture): + """Test that missing region returns None.""" + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + is_active=True, + ) + provider_cfg.credentials_decoded = { + "api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110), + } + provider_cfg.save() + + result = _extract_bedrock_credentials(provider_cfg) + + assert result is None + + def test_empty_credentials_returns_none(self, tenants_fixture): + """Test that empty credentials dict returns None (region only is not enough).""" + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + is_active=True, + ) + # Only region, no auth credentials - should return None + provider_cfg.credentials_decoded = { + "region": "us-east-1", + } + provider_cfg.save() + + result = _extract_bedrock_credentials(provider_cfg) + + assert result is None + + def test_non_dict_credentials_returns_none(self, tenants_fixture): + """Test that non-dict credentials returns None.""" + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + is_active=True, + ) + # Store valid credentials first to pass model validation + provider_cfg.credentials_decoded = { + "api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110), + "region": "us-east-1", + } + provider_cfg.save() + + # Mock the credentials_decoded property to return a non-dict value + # This simulates corrupted/invalid stored data + with patch.object( + type(provider_cfg), + "credentials_decoded", + new_callable=lambda: property(lambda self: "invalid"), + ): + result = _extract_bedrock_credentials(provider_cfg) + + assert result is None + + +class TestCreateBedrockClient: + """Unit tests for _create_bedrock_client helper function.""" + + @patch("tasks.jobs.lighthouse_providers.boto3.client") + def test_create_client_with_access_keys(self, mock_boto_client): + """Test creating client with access key authentication.""" + mock_client = MagicMock() + mock_boto_client.return_value = mock_client + + creds = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + } + + result = _create_bedrock_client(creds) + + assert result == mock_client + mock_boto_client.assert_called_once_with( + service_name="bedrock", + region_name="us-east-1", + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + ) + + @patch("tasks.jobs.lighthouse_providers.Config") + @patch("tasks.jobs.lighthouse_providers.boto3.client") + def test_create_client_with_api_key(self, mock_boto_client, mock_config): + """Test creating client with API key authentication.""" + mock_client = MagicMock() + mock_events = MagicMock() + mock_client.meta.events = mock_events + mock_boto_client.return_value = mock_client + mock_config_instance = MagicMock() + mock_config.return_value = mock_config_instance + valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110) + + creds = { + "api_key": valid_api_key, + "region": "us-west-2", + } + + result = _create_bedrock_client(creds) + + assert result == mock_client + mock_boto_client.assert_called_once_with( + service_name="bedrock", + region_name="us-west-2", + config=mock_config_instance, + ) + mock_events.register.assert_called_once() + call_args = mock_events.register.call_args + assert call_args[0][0] == "before-send.*.*" + + # Verify handler injects bearer token + handler_fn = call_args[0][1] + mock_request = MagicMock() + mock_request.headers = {} + handler_fn(mock_request) + assert mock_request.headers["Authorization"] == f"Bearer {valid_api_key}" + + # TODO Move this to outputs/reports jobs @pytest.mark.django_db class TestGenerateOutputs: @@ -533,8 +733,15 @@ class TestGenerateOutputs: class TestScanCompleteTasks: +<<<<<<< HEAD @patch("tasks.tasks.perform_attack_paths_scan_task.apply_async") @patch("tasks.tasks.create_compliance_requirements_task.apply_async") +======= + @patch("tasks.tasks.aggregate_attack_surface_task.apply_async") + @patch("tasks.tasks.chain") + @patch("tasks.tasks.create_compliance_requirements_task.si") + @patch("tasks.tasks.update_provider_compliance_scores_task.si") +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 @patch("tasks.tasks.perform_scan_summary_task.si") @patch("tasks.tasks.generate_outputs_task.si") @patch("tasks.tasks.generate_compliance_reports_task.si") @@ -545,14 +752,30 @@ class TestScanCompleteTasks: mock_compliance_reports_task, mock_outputs_task, mock_scan_summary_task, + mock_update_compliance_scores_task, mock_compliance_requirements_task, +<<<<<<< HEAD mock_attack_paths_task, +======= + mock_chain, + mock_attack_surface_task, +>>>>>>> 1bf49747adaefcb19db66274478f6933342112c1 ): """Test that scan complete tasks are properly orchestrated with optimized reports.""" _perform_scan_complete_tasks("tenant-id", "scan-id", "provider-id") - # Verify compliance requirements task is called + # Verify compliance requirements task is called via chain mock_compliance_requirements_task.assert_called_once_with( + tenant_id="tenant-id", scan_id="scan-id" + ) + + # Verify update provider compliance scores task is called via chain + mock_update_compliance_scores_task.assert_called_once_with( + tenant_id="tenant-id", scan_id="scan-id" + ) + + # Verify attack surface task is called + mock_attack_surface_task.assert_called_once_with( kwargs={"tenant_id": "tenant-id", "scan_id": "scan-id"}, ) @@ -1213,6 +1436,16 @@ class TestCheckLighthouseProviderConnectionTask: None, {"connected": True, "error": None}, ), + # Bedrock API key authentication + ( + LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + { + "api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110), + "region": "us-east-1", + }, + None, + {"connected": True, "error": None}, + ), ], ) def test_check_connection_success_all_providers( @@ -1281,6 +1514,24 @@ class TestCheckLighthouseProviderConnectionTask: "list_foundation_models", ), ), + # Bedrock API key authentication failure + ( + LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + { + "api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("X" * 110), + "region": "us-east-1", + }, + None, + ClientError( + { + "Error": { + "Code": "UnrecognizedClientException", + "Message": "Invalid API key", + } + }, + "list_foundation_models", + ), + ), ], ) def test_check_connection_api_failure( @@ -1405,6 +1656,17 @@ class TestRefreshLighthouseProviderModelsTask: {"openai.gpt-oss-120b-1:0": "gpt-oss-120b"}, 1, ), + # Bedrock API key authentication + ( + LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + { + "api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110), + "region": "us-east-1", + }, + None, + {"anthropic.claude-v3": "Claude 3"}, + 1, + ), ], ) def test_refresh_models_create_new( @@ -1541,3 +1803,343 @@ class TestRefreshLighthouseProviderModelsTask: assert result["deleted"] == 0 assert "error" in result assert result["error"] is not None + + +@pytest.mark.django_db +class TestCleanupOrphanScheduledScans: + """Unit tests for _cleanup_orphan_scheduled_scans helper function.""" + + def _create_periodic_task(self, provider_id, tenant_id): + """Helper to create a PeriodicTask for testing.""" + interval, _ = IntervalSchedule.objects.get_or_create(every=24, period="hours") + return PeriodicTask.objects.create( + name=f"scan-perform-scheduled-{provider_id}", + task="scan-perform-scheduled", + interval=interval, + kwargs=f'{{"tenant_id": "{tenant_id}", "provider_id": "{provider_id}"}}', + enabled=True, + ) + + def test_cleanup_deletes_orphan_when_both_available_and_scheduled_exist( + self, tenants_fixture, providers_fixture + ): + """Test that AVAILABLE scan is deleted when SCHEDULED also exists.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + periodic_task = self._create_periodic_task(provider.id, tenant.id) + + # Create orphan AVAILABLE scan + orphan_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=periodic_task.id, + ) + + # Create SCHEDULED scan (next execution) + scheduled_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.SCHEDULED, + scheduler_task_id=periodic_task.id, + ) + + # Execute cleanup + deleted_count = _cleanup_orphan_scheduled_scans( + tenant_id=str(tenant.id), + provider_id=str(provider.id), + scheduler_task_id=periodic_task.id, + ) + + # Verify orphan was deleted + assert deleted_count == 1 + assert not Scan.objects.filter(id=orphan_scan.id).exists() + assert Scan.objects.filter(id=scheduled_scan.id).exists() + + def test_cleanup_does_not_delete_when_only_available_exists( + self, tenants_fixture, providers_fixture + ): + """Test that AVAILABLE scan is NOT deleted when no SCHEDULED exists.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + periodic_task = self._create_periodic_task(provider.id, tenant.id) + + # Create only AVAILABLE scan (normal first scan scenario) + available_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=periodic_task.id, + ) + + # Execute cleanup + deleted_count = _cleanup_orphan_scheduled_scans( + tenant_id=str(tenant.id), + provider_id=str(provider.id), + scheduler_task_id=periodic_task.id, + ) + + # Verify nothing was deleted + assert deleted_count == 0 + assert Scan.objects.filter(id=available_scan.id).exists() + + def test_cleanup_does_not_delete_when_only_scheduled_exists( + self, tenants_fixture, providers_fixture + ): + """Test that nothing is deleted when only SCHEDULED exists.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + periodic_task = self._create_periodic_task(provider.id, tenant.id) + + # Create only SCHEDULED scan (normal subsequent scan scenario) + scheduled_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.SCHEDULED, + scheduler_task_id=periodic_task.id, + ) + + # Execute cleanup + deleted_count = _cleanup_orphan_scheduled_scans( + tenant_id=str(tenant.id), + provider_id=str(provider.id), + scheduler_task_id=periodic_task.id, + ) + + # Verify nothing was deleted + assert deleted_count == 0 + assert Scan.objects.filter(id=scheduled_scan.id).exists() + + def test_cleanup_returns_zero_when_no_scans_exist( + self, tenants_fixture, providers_fixture + ): + """Test that cleanup returns 0 when no scans exist.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + periodic_task = self._create_periodic_task(provider.id, tenant.id) + + # Execute cleanup with no scans + deleted_count = _cleanup_orphan_scheduled_scans( + tenant_id=str(tenant.id), + provider_id=str(provider.id), + scheduler_task_id=periodic_task.id, + ) + + assert deleted_count == 0 + + def test_cleanup_deletes_multiple_orphan_available_scans( + self, tenants_fixture, providers_fixture + ): + """Test that multiple AVAILABLE orphan scans are all deleted.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + periodic_task = self._create_periodic_task(provider.id, tenant.id) + + # Create multiple orphan AVAILABLE scans + orphan_scan_1 = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=periodic_task.id, + ) + orphan_scan_2 = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=periodic_task.id, + ) + + # Create SCHEDULED scan + scheduled_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.SCHEDULED, + scheduler_task_id=periodic_task.id, + ) + + # Execute cleanup + deleted_count = _cleanup_orphan_scheduled_scans( + tenant_id=str(tenant.id), + provider_id=str(provider.id), + scheduler_task_id=periodic_task.id, + ) + + # Verify all orphans were deleted + assert deleted_count == 2 + assert not Scan.objects.filter(id=orphan_scan_1.id).exists() + assert not Scan.objects.filter(id=orphan_scan_2.id).exists() + assert Scan.objects.filter(id=scheduled_scan.id).exists() + + def test_cleanup_does_not_affect_different_provider( + self, tenants_fixture, providers_fixture + ): + """Test that cleanup only affects scans for the specified provider.""" + tenant = tenants_fixture[0] + provider1 = providers_fixture[0] + provider2 = providers_fixture[1] + periodic_task1 = self._create_periodic_task(provider1.id, tenant.id) + periodic_task2 = self._create_periodic_task(provider2.id, tenant.id) + + # Create orphan scenario for provider1 + orphan_scan_p1 = Scan.objects.create( + tenant_id=tenant.id, + provider=provider1, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=periodic_task1.id, + ) + scheduled_scan_p1 = Scan.objects.create( + tenant_id=tenant.id, + provider=provider1, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.SCHEDULED, + scheduler_task_id=periodic_task1.id, + ) + + # Create AVAILABLE scan for provider2 (should not be affected) + available_scan_p2 = Scan.objects.create( + tenant_id=tenant.id, + provider=provider2, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=periodic_task2.id, + ) + + # Execute cleanup for provider1 only + deleted_count = _cleanup_orphan_scheduled_scans( + tenant_id=str(tenant.id), + provider_id=str(provider1.id), + scheduler_task_id=periodic_task1.id, + ) + + # Verify only provider1's orphan was deleted + assert deleted_count == 1 + assert not Scan.objects.filter(id=orphan_scan_p1.id).exists() + assert Scan.objects.filter(id=scheduled_scan_p1.id).exists() + assert Scan.objects.filter(id=available_scan_p2.id).exists() + + def test_cleanup_does_not_affect_manual_scans( + self, tenants_fixture, providers_fixture + ): + """Test that cleanup only affects SCHEDULED trigger scans, not MANUAL.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + periodic_task = self._create_periodic_task(provider.id, tenant.id) + + # Create orphan AVAILABLE scheduled scan + orphan_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=periodic_task.id, + ) + + # Create SCHEDULED scan + scheduled_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.SCHEDULED, + scheduler_task_id=periodic_task.id, + ) + + # Create AVAILABLE manual scan (should not be affected) + manual_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Manual scan", + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.AVAILABLE, + ) + + # Execute cleanup + deleted_count = _cleanup_orphan_scheduled_scans( + tenant_id=str(tenant.id), + provider_id=str(provider.id), + scheduler_task_id=periodic_task.id, + ) + + # Verify only scheduled orphan was deleted + assert deleted_count == 1 + assert not Scan.objects.filter(id=orphan_scan.id).exists() + assert Scan.objects.filter(id=scheduled_scan.id).exists() + assert Scan.objects.filter(id=manual_scan.id).exists() + + def test_cleanup_does_not_affect_different_scheduler_task( + self, tenants_fixture, providers_fixture + ): + """Test that cleanup only affects scans with the specified scheduler_task_id.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + periodic_task1 = self._create_periodic_task(provider.id, tenant.id) + + # Create another periodic task + interval, _ = IntervalSchedule.objects.get_or_create(every=24, period="hours") + periodic_task2 = PeriodicTask.objects.create( + name=f"scan-perform-scheduled-other-{provider.id}", + task="scan-perform-scheduled", + interval=interval, + kwargs=f'{{"tenant_id": "{tenant.id}", "provider_id": "{provider.id}"}}', + enabled=True, + ) + + # Create orphan scenario for periodic_task1 + orphan_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=periodic_task1.id, + ) + scheduled_scan = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.SCHEDULED, + scheduler_task_id=periodic_task1.id, + ) + + # Create AVAILABLE scan for periodic_task2 (should not be affected) + available_scan_other_task = Scan.objects.create( + tenant_id=tenant.id, + provider=provider, + name="Daily scheduled scan", + trigger=Scan.TriggerChoices.SCHEDULED, + state=StateChoices.AVAILABLE, + scheduler_task_id=periodic_task2.id, + ) + + # Execute cleanup for periodic_task1 only + deleted_count = _cleanup_orphan_scheduled_scans( + tenant_id=str(tenant.id), + provider_id=str(provider.id), + scheduler_task_id=periodic_task1.id, + ) + + # Verify only periodic_task1's orphan was deleted + assert deleted_count == 1 + assert not Scan.objects.filter(id=orphan_scan.id).exists() + assert Scan.objects.filter(id=scheduled_scan.id).exists() + assert Scan.objects.filter(id=available_scan_other_task.id).exists() diff --git a/dashboard/assets/images/providers/alibabacloud_provider.png b/dashboard/assets/images/providers/alibabacloud_provider.png new file mode 100644 index 0000000000..7b14b0c33b Binary files /dev/null and b/dashboard/assets/images/providers/alibabacloud_provider.png differ diff --git a/dashboard/compliance/cis_1_12_kubernetes.py b/dashboard/compliance/cis_1_12_kubernetes.py new file mode 100644 index 0000000000..94558f33ad --- /dev/null +++ b/dashboard/compliance/cis_1_12_kubernetes.py @@ -0,0 +1,24 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/cis_2_0_alibabacloud.py b/dashboard/compliance/cis_2_0_alibabacloud.py new file mode 100644 index 0000000000..94558f33ad --- /dev/null +++ b/dashboard/compliance/cis_2_0_alibabacloud.py @@ -0,0 +1,24 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/cis_5_0_azure.py b/dashboard/compliance/cis_5_0_azure.py new file mode 100644 index 0000000000..9d33cc67a8 --- /dev/null +++ b/dashboard/compliance/cis_5_0_azure.py @@ -0,0 +1,25 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/cis_6_0_m365.py b/dashboard/compliance/cis_6_0_m365.py new file mode 100644 index 0000000000..94558f33ad --- /dev/null +++ b/dashboard/compliance/cis_6_0_m365.py @@ -0,0 +1,24 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/prowler_threatscore_alibabacloud.py b/dashboard/compliance/prowler_threatscore_alibabacloud.py new file mode 100644 index 0000000000..d86a13fd01 --- /dev/null +++ b/dashboard/compliance/prowler_threatscore_alibabacloud.py @@ -0,0 +1,28 @@ +import warnings + +from dashboard.common_methods import get_section_containers_threatscore + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_threatscore( + aux, + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "REQUIREMENTS_ID", + ) diff --git a/dashboard/lib/dropdowns.py b/dashboard/lib/dropdowns.py index 3e92759042..421801bc27 100644 --- a/dashboard/lib/dropdowns.py +++ b/dashboard/lib/dropdowns.py @@ -312,3 +312,28 @@ def create_table_row_dropdown(table_rows: list) -> html.Div: ), ], ) + + +def create_category_dropdown(categories: list) -> html.Div: + """ + Dropdown to select the category. + Args: + categories (list): List of categories. + Returns: + html.Div: Dropdown to select the category. + """ + return html.Div( + [ + html.Label( + "Category:", className="text-prowler-stone-900 font-bold text-sm" + ), + dcc.Dropdown( + id="category-filter", + options=[{"label": i, "value": i} for i in categories], + value=["All"], + clearable=False, + multi=True, + style={"color": "#000000"}, + ), + ], + ) diff --git a/dashboard/lib/layouts.py b/dashboard/lib/layouts.py index e054731684..930432b6c4 100644 --- a/dashboard/lib/layouts.py +++ b/dashboard/lib/layouts.py @@ -12,6 +12,7 @@ def create_layout_overview( provider_dropdown: html.Div, table_row_dropdown: html.Div, status_dropdown: html.Div, + category_dropdown: html.Div, table_div_header: html.Div, amount_providers: int, ) -> html.Div: @@ -51,8 +52,9 @@ def create_layout_overview( html.Div([service_dropdown], className=""), html.Div([provider_dropdown], className=""), html.Div([status_dropdown], className=""), + html.Div([category_dropdown], className=""), ], - className="grid gap-x-4 mb-[30px] sm:grid-cols-2 lg:grid-cols-4", + className="grid gap-x-4 mb-[30px] sm:grid-cols-2 lg:grid-cols-5", ), html.Div( [ @@ -61,6 +63,7 @@ def create_layout_overview( html.Div(className="flex", id="gcp_card", n_clicks=0), html.Div(className="flex", id="k8s_card", n_clicks=0), html.Div(className="flex", id="m365_card", n_clicks=0), + html.Div(className="flex", id="alibabacloud_card", n_clicks=0), ], className=f"grid gap-x-4 mb-[30px] sm:grid-cols-2 lg:grid-cols-{amount_providers}", ), diff --git a/dashboard/pages/compliance.py b/dashboard/pages/compliance.py index 33a2b0c268..f944f7f098 100644 --- a/dashboard/pages/compliance.py +++ b/dashboard/pages/compliance.py @@ -78,6 +78,8 @@ def load_csv_files(csv_files): result = result.replace("_KUBERNETES", " - KUBERNETES") if "M65" in result: result = result.replace("_M65", " - M65") + if "ALIBABACLOUD" in result: + result = result.replace("_ALIBABACLOUD", " - ALIBABACLOUD") results.append(result) unique_results = set(results) @@ -125,7 +127,7 @@ if data is None: ) else: - data["ASSESSMENTDATE"] = pd.to_datetime(data["ASSESSMENTDATE"]) + data["ASSESSMENTDATE"] = pd.to_datetime(data["ASSESSMENTDATE"], format="mixed") data["ASSESSMENT_TIME"] = data["ASSESSMENTDATE"].dt.strftime("%Y-%m-%d %H:%M:%S") data_values = data["ASSESSMENT_TIME"].unique() @@ -278,9 +280,13 @@ def display_data( data["REQUIREMENTS_ATTRIBUTES_PROFILE"] = data[ "REQUIREMENTS_ATTRIBUTES_PROFILE" ].apply(lambda x: x.split(" - ")[0]) + + # Rename the column LOCATION to REGION for Alibaba Cloud + if "alibabacloud" in analytics_input: + data = data.rename(columns={"LOCATION": "REGION"}) # Filter the chosen level of the CIS if is_level_1: - data = data[data["REQUIREMENTS_ATTRIBUTES_PROFILE"] == "Level 1"] + data = data[data["REQUIREMENTS_ATTRIBUTES_PROFILE"].str.contains("Level 1")] # Rename the column PROJECTID to ACCOUNTID for GCP if data.columns.str.contains("PROJECTID").any(): @@ -401,9 +407,11 @@ def display_data( compliance_module = importlib.import_module( f"dashboard.compliance.{current}" ) - data = data.drop_duplicates( - subset=["CHECKID", "STATUS", "MUTED", "RESOURCEID", "STATUSEXTENDED"] - ) + # Build subset list based on available columns + dedup_columns = ["CHECKID", "STATUS", "RESOURCEID", "STATUSEXTENDED"] + if "MUTED" in data.columns: + dedup_columns.insert(2, "MUTED") + data = data.drop_duplicates(subset=dedup_columns) if "threatscore" in analytics_input: data = get_threatscore_mean_by_pillar(data) @@ -646,6 +654,7 @@ def get_table(current_compliance, table): def get_threatscore_mean_by_pillar(df): score_per_pillar = {} max_score_per_pillar = {} + counted_findings_per_pillar = {} for _, row in df.iterrows(): pillar = ( @@ -657,6 +666,18 @@ def get_threatscore_mean_by_pillar(df): if pillar not in score_per_pillar: score_per_pillar[pillar] = 0 max_score_per_pillar[pillar] = 0 + counted_findings_per_pillar[pillar] = set() + + # Skip muted findings for score calculation + is_muted = "MUTED" in df.columns and row.get("MUTED") == "True" + if is_muted: + continue + + # Create unique finding identifier to avoid counting duplicates + finding_id = f"{row.get('CHECKID', '')}_{row.get('RESOURCEID', '')}" + if finding_id in counted_findings_per_pillar[pillar]: + continue + counted_findings_per_pillar[pillar].add(finding_id) level_of_risk = pd.to_numeric( row["REQUIREMENTS_ATTRIBUTES_LEVELOFRISK"], errors="coerce" @@ -700,6 +721,10 @@ def get_table_prowler_threatscore(df): score_per_pillar = {} max_score_per_pillar = {} pillars = {} + counted_findings_per_pillar = {} + counted_pass = set() + counted_fail = set() + counted_muted = set() df_copy = df.copy() @@ -714,6 +739,24 @@ def get_table_prowler_threatscore(df): pillars[pillar] = {"FAIL": 0, "PASS": 0, "MUTED": 0} score_per_pillar[pillar] = 0 max_score_per_pillar[pillar] = 0 + counted_findings_per_pillar[pillar] = set() + + # Create unique finding identifier + finding_id = f"{row.get('CHECKID', '')}_{row.get('RESOURCEID', '')}" + + # Check if muted + is_muted = "MUTED" in df_copy.columns and row.get("MUTED") == "True" + + # Count muted findings (separate from score calculation) + if is_muted and finding_id not in counted_muted: + counted_muted.add(finding_id) + pillars[pillar]["MUTED"] += 1 + continue # Skip muted findings for score calculation + + # Skip if already counted for this pillar + if finding_id in counted_findings_per_pillar[pillar]: + continue + counted_findings_per_pillar[pillar].add(finding_id) level_of_risk = pd.to_numeric( row["REQUIREMENTS_ATTRIBUTES_LEVELOFRISK"], errors="coerce" @@ -732,13 +775,14 @@ def get_table_prowler_threatscore(df): max_score_per_pillar[pillar] += level_of_risk * weight if row["STATUS"] == "PASS": - pillars[pillar]["PASS"] += 1 + if finding_id not in counted_pass: + counted_pass.add(finding_id) + pillars[pillar]["PASS"] += 1 score_per_pillar[pillar] += level_of_risk * weight elif row["STATUS"] == "FAIL": - pillars[pillar]["FAIL"] += 1 - - if "MUTED" in row and row["MUTED"] == "True": - pillars[pillar]["MUTED"] += 1 + if finding_id not in counted_fail: + counted_fail.add(finding_id) + pillars[pillar]["FAIL"] += 1 result_df = [] diff --git a/dashboard/pages/overview.py b/dashboard/pages/overview.py index 4ce749a9e4..ada06b8282 100644 --- a/dashboard/pages/overview.py +++ b/dashboard/pages/overview.py @@ -35,6 +35,7 @@ from dashboard.config import ( from dashboard.lib.cards import create_provider_card from dashboard.lib.dropdowns import ( create_account_dropdown, + create_category_dropdown, create_date_dropdown, create_provider_dropdown, create_region_dropdown, @@ -79,6 +80,9 @@ ks8_provider_logo = html.Img( m365_provider_logo = html.Img( src="assets/images/providers/m365_provider.png", alt="m365 provider" ) +alibabacloud_provider_logo = html.Img( + src="assets/images/providers/alibabacloud_provider.png", alt="alibabacloud provider" +) def load_csv_files(csv_files): @@ -253,6 +257,8 @@ else: accounts.append(account + " - AWS") if "kubernetes" in list(data[data["ACCOUNT_UID"] == account]["PROVIDER"]): accounts.append(account + " - K8S") + if "alibabacloud" in list(data[data["ACCOUNT_UID"] == account]["PROVIDER"]): + accounts.append(account + " - ALIBABACLOUD") account_dropdown = create_account_dropdown(accounts) @@ -298,6 +304,8 @@ else: services.append(service + " - GCP") if "m365" in list(data[data["SERVICE_NAME"] == service]["PROVIDER"]): services.append(service + " - M365") + if "alibabacloud" in list(data[data["SERVICE_NAME"] == service]["PROVIDER"]): + services.append(service + " - ALIBABACLOUD") services = ["All"] + services services = [ @@ -336,6 +344,18 @@ else: status = [x for x in status if str(x) != "nan" and x.__class__.__name__ == "str"] status_dropdown = create_status_dropdown(status) + + # Create the category dropdown + categories = [] + if "CATEGORIES" in data.columns: + for cat_list in data["CATEGORIES"].dropna().unique(): + if cat_list and str(cat_list) != "nan": + for cat in str(cat_list).split(","): + cat = cat.strip() + if cat and cat not in categories: + categories.append(cat) + categories = ["All"] + sorted(categories) + category_dropdown = create_category_dropdown(categories) table_div_header = [] table_div_header.append( html.Div( @@ -497,6 +517,7 @@ else: provider_dropdown, table_row_dropdown, status_dropdown, + category_dropdown, table_div_header, len(data["PROVIDER"].unique()), ) @@ -520,6 +541,7 @@ else: Output("gcp_card", "children"), Output("k8s_card", "children"), Output("m365_card", "children"), + Output("alibabacloud_card", "children"), Output("subscribe_card", "children"), Output("info-file-over", "title"), Output("severity-filter", "value"), @@ -532,11 +554,14 @@ else: Output("table-rows", "options"), Output("status-filter", "value"), Output("status-filter", "options"), + Output("category-filter", "value"), + Output("category-filter", "options"), Output("aws_card", "n_clicks"), Output("azure_card", "n_clicks"), Output("gcp_card", "n_clicks"), Output("k8s_card", "n_clicks"), Output("m365_card", "n_clicks"), + Output("alibabacloud_card", "n_clicks"), ], Input("cloud-account-filter", "value"), Input("region-filter", "value"), @@ -548,6 +573,7 @@ else: Input("provider-filter", "value"), Input("table-rows", "value"), Input("status-filter", "value"), + Input("category-filter", "value"), Input("search-input", "value"), Input("aws_card", "n_clicks"), Input("azure_card", "n_clicks"), @@ -560,6 +586,7 @@ else: Input("sort_button_region", "n_clicks"), Input("sort_button_service", "n_clicks"), Input("sort_button_account", "n_clicks"), + Input("alibabacloud_card", "n_clicks"), ) def filter_data( cloud_account_values, @@ -572,6 +599,7 @@ def filter_data( provider_values, table_row_values, status_values, + category_values, search_value, aws_clicks, azure_clicks, @@ -584,6 +612,7 @@ def filter_data( sort_button_region, sort_button_service, sort_button_account, + alibabacloud_clicks, ): # Use n_clicks for vulture n_clicks_csv = n_clicks_csv @@ -599,6 +628,7 @@ def filter_data( gcp_clicks = 0 k8s_clicks = 0 m365_clicks = 0 + alibabacloud_clicks = 0 if azure_clicks > 0: filtered_data = data.copy() if azure_clicks % 2 != 0 and "azure" in list(data["PROVIDER"]): @@ -607,6 +637,7 @@ def filter_data( gcp_clicks = 0 k8s_clicks = 0 m365_clicks = 0 + alibabacloud_clicks = 0 if gcp_clicks > 0: filtered_data = data.copy() if gcp_clicks % 2 != 0 and "gcp" in list(data["PROVIDER"]): @@ -615,6 +646,7 @@ def filter_data( azure_clicks = 0 k8s_clicks = 0 m365_clicks = 0 + alibabacloud_clicks = 0 if k8s_clicks > 0: filtered_data = data.copy() if k8s_clicks % 2 != 0 and "kubernetes" in list(data["PROVIDER"]): @@ -623,6 +655,7 @@ def filter_data( azure_clicks = 0 gcp_clicks = 0 m365_clicks = 0 + alibabacloud_clicks = 0 if m365_clicks > 0: filtered_data = data.copy() if m365_clicks % 2 != 0 and "m365" in list(data["PROVIDER"]): @@ -631,7 +664,16 @@ def filter_data( azure_clicks = 0 gcp_clicks = 0 k8s_clicks = 0 - + alibabacloud_clicks = 0 + if alibabacloud_clicks > 0: + filtered_data = data.copy() + if alibabacloud_clicks % 2 != 0 and "alibabacloud" in list(data["PROVIDER"]): + filtered_data = filtered_data[filtered_data["PROVIDER"] == "alibabacloud"] + aws_clicks = 0 + azure_clicks = 0 + gcp_clicks = 0 + k8s_clicks = 0 + m365_clicks = 0 # For all the data, we will add to the status column the value 'MUTED (FAIL)' and 'MUTED (PASS)' depending on the value of the column 'STATUS' and 'MUTED' if "MUTED" in filtered_data.columns: filtered_data["STATUS"] = filtered_data.apply( @@ -723,6 +765,8 @@ def filter_data( all_account_ids.append(account) if "kubernetes" in list(data[data["ACCOUNT_UID"] == account]["PROVIDER"]): all_account_ids.append(account) + if "alibabacloud" in list(data[data["ACCOUNT_UID"] == account]["PROVIDER"]): + all_account_ids.append(account) all_account_names = [] if "ACCOUNT_NAME" in filtered_data.columns: @@ -745,6 +789,10 @@ def filter_data( cloud_accounts_options.append(item + " - AWS") if "kubernetes" in list(data[data["ACCOUNT_UID"] == item]["PROVIDER"]): cloud_accounts_options.append(item + " - K8S") + if "alibabacloud" in list( + data[data["ACCOUNT_UID"] == item]["PROVIDER"] + ): + cloud_accounts_options.append(item + " - ALIBABACLOUD") if "ACCOUNT_NAME" in filtered_data.columns: if "azure" in list(data[data["ACCOUNT_NAME"] == item]["PROVIDER"]): cloud_accounts_options.append(item + " - AZURE") @@ -873,6 +921,10 @@ def filter_data( filtered_data[filtered_data["SERVICE_NAME"] == item]["PROVIDER"] ): service_filter_options.append(item + " - M365") + if "alibabacloud" in list( + filtered_data[filtered_data["SERVICE_NAME"] == item]["PROVIDER"] + ): + service_filter_options.append(item + " - ALIBABACLOUD") # Filter Service if service_values == ["All"]: @@ -931,6 +983,41 @@ def filter_data( status_filter_options = ["All"] + list(filtered_data["STATUS"].unique()) + # Filter Category + if "CATEGORIES" in filtered_data.columns: + if category_values == ["All"]: + updated_category_values = None + elif "All" in category_values and len(category_values) > 1: + category_values.remove("All") + updated_category_values = category_values + elif len(category_values) == 0: + updated_category_values = None + category_values = ["All"] + else: + updated_category_values = category_values + + if updated_category_values: + filtered_data = filtered_data[ + filtered_data["CATEGORIES"].apply( + lambda x: any( + cat.strip() in updated_category_values + for cat in str(x).split(",") + if str(x) != "nan" + ) + ) + ] + + category_filter_options = ["All"] + for cat_list in filtered_data["CATEGORIES"].dropna().unique(): + if cat_list and str(cat_list) != "nan": + for cat in str(cat_list).split(","): + cat = cat.strip() + if cat and cat not in category_filter_options: + category_filter_options.append(cat) + category_filter_options = sorted(category_filter_options) + else: + category_filter_options = ["All"] + if len(filtered_data_sp) == 0: fig = px.pie() fig.update_layout( @@ -1324,6 +1411,12 @@ def filter_data( filtered_data.loc[ filtered_data["ACCOUNT_UID"] == account, "ACCOUNT_UID" ] = (account + " - M365") + if "alibabacloud" in list( + data[data["ACCOUNT_UID"] == account]["PROVIDER"] + ): + filtered_data.loc[ + filtered_data["ACCOUNT_UID"] == account, "ACCOUNT_UID" + ] = (account + " - ALIBABACLOUD") table_collapsible = [] for item in filtered_data.to_dict("records"): @@ -1410,6 +1503,13 @@ def filter_data( else: m365_card = None + if "alibabacloud" in list(data["PROVIDER"].unique()): + alibabacloud_card = create_provider_card( + "alibabacloud", alibabacloud_provider_logo, "Accounts", full_filtered_data + ) + else: + alibabacloud_card = None + # Subscribe to Prowler Cloud card subscribe_card = [ html.Div( @@ -1454,6 +1554,7 @@ def filter_data( gcp_card, k8s_card, m365_card, + alibabacloud_card, subscribe_card, list_files, severity_values, @@ -1464,11 +1565,14 @@ def filter_data( table_row_options, status_values, status_filter_options, + category_values, + category_filter_options, aws_clicks, azure_clicks, gcp_clicks, k8s_clicks, m365_clicks, + alibabacloud_clicks, ) else: return ( @@ -1487,6 +1591,7 @@ def filter_data( gcp_card, k8s_card, m365_card, + alibabacloud_card, subscribe_card, list_files, severity_values, @@ -1499,11 +1604,14 @@ def filter_data( table_row_options, status_values, status_filter_options, + category_values, + category_filter_options, aws_clicks, azure_clicks, gcp_clicks, k8s_clicks, m365_clicks, + alibabacloud_clicks, ) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 36b0942bdf..90ab1b6390 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -44,6 +44,9 @@ services: volumes: - "./ui:/app" - "/app/node_modules" + depends_on: + mcp-server: + condition: service_healthy postgres: image: postgres:16.3-alpine3.20 @@ -60,7 +63,11 @@ services: ports: - "${POSTGRES_PORT:-5432}:${POSTGRES_PORT:-5432}" healthcheck: - test: ["CMD-SHELL", "sh -c 'pg_isready -U ${POSTGRES_ADMIN_USER} -d ${POSTGRES_DB}'"] + test: + [ + "CMD-SHELL", + "sh -c 'pg_isready -U ${POSTGRES_ADMIN_USER} -d ${POSTGRES_DB}'", + ] interval: 5s timeout: 5s retries: 5 @@ -163,6 +170,32 @@ services: - "../docker-entrypoint.sh" - "beat" + mcp-server: + build: + context: ./mcp_server + dockerfile: Dockerfile + environment: + - PROWLER_MCP_TRANSPORT_MODE=http + env_file: + - path: .env + required: false + ports: + - "8000:8000" + volumes: + - ./mcp_server/prowler_mcp_server:/app/prowler_mcp_server + - ./mcp_server/pyproject.toml:/app/pyproject.toml + - ./mcp_server/entrypoint.sh:/app/entrypoint.sh + command: ["uvicorn", "--host", "0.0.0.0", "--port", "8000"] + healthcheck: + test: + [ + "CMD-SHELL", + "wget -q -O /dev/null http://127.0.0.1:8000/health || exit 1", + ] + interval: 10s + timeout: 5s + retries: 3 + volumes: outputs: driver: local diff --git a/docker-compose.yml b/docker-compose.yml index ee9c088c3f..8bd5f70ba3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,9 @@ +# Production Docker Compose configuration +# Uses pre-built images from Docker Hub (prowlercloud/*) +# +# For development with local builds and hot-reload, use docker-compose-dev.yml instead: +# docker compose -f docker-compose-dev.yml up +# services: api: hostname: "prowler-api" @@ -26,6 +32,9 @@ services: required: false ports: - ${UI_PORT:-3000}:${UI_PORT:-3000} + depends_on: + mcp-server: + condition: service_healthy postgres: image: postgres:16.3-alpine3.20 @@ -124,6 +133,22 @@ services: - "../docker-entrypoint.sh" - "beat" + mcp-server: + image: prowlercloud/prowler-mcp:${PROWLER_MCP_VERSION:-stable} + environment: + - PROWLER_MCP_TRANSPORT_MODE=http + env_file: + - path: .env + required: false + ports: + - "8000:8000" + command: ["uvicorn", "--host", "0.0.0.0", "--port", "8000"] + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8000/health || exit 1"] + interval: 10s + timeout: 5s + retries: 3 + volumes: output: driver: local diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 47e24d419d..2f9efd405d 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -479,6 +479,66 @@ Effective headers and section titles enhance document readability and structure, --- +## Version Badge for Feature Documentation + +The Version Badge component indicates when a specific feature or functionality was introduced in Prowler. This component is located at `docs/snippets/version-badge.mdx` and should be used consistently across the documentation. + +### When to Use the Version Badge + +Use the Version Badge when documenting: + +* New features added in a specific version. +* New CLI options or flags. +* New API endpoints or SDK methods. +* New compliance frameworks or security checks. +* Breaking changes or deprecated features (with appropriate context). + +### How to Use the Version Badge + +1. **Import the Component** + + At the top of the MDX file, import the snippet: + + ```mdx + import { VersionBadge } from "/snippets/version-badge.mdx" + ``` + +2. **Place the Badge** + + Insert the badge immediately after the section header or feature title: + + ```mdx + ## New Feature Name + + + + Description of the feature... + ``` + +3. **Version Format** + + Use semantic versioning format (e.g., `4.5.0`, `5.0.0`). Do not include the "v" prefix. + +### Placement Guidelines + +* Place the Version Badge on its own line, directly below the header. +* Leave a blank line after the badge before continuing with the content. +* For subsections, place the badge only if the subsection introduces something new independently from the parent section. + +**Example:** + +```mdx +## Tag-Based Scanning + +import { VersionBadge } from "/snippets/version-badge.mdx" + + + +Tag-Based Scanning allows filtering resources by AWS tags during security assessments... +``` + +--- + ## Avoid Assumptions Regarding Audience’s Expertise ### Understand Your Audience’s Expertise diff --git a/docs/developer-guide/ai-skills.mdx b/docs/developer-guide/ai-skills.mdx new file mode 100644 index 0000000000..6a0787dac8 --- /dev/null +++ b/docs/developer-guide/ai-skills.mdx @@ -0,0 +1,219 @@ +--- +title: 'AI Skills System' +--- + +This guide explains the AI Skills system that provides on-demand context and patterns to AI agents working with the Prowler codebase. + + +**What are AI Skills?** Skills are structured instructions that help AI agents (Claude Code, Cursor, Copilot, etc.) understand Prowler's conventions, patterns, and best practices. + + +## Architecture Overview + +```mermaid +graph LR + subgraph FLOW["AI Skills Architecture"] + A["AI Agent"] -->|"1. matches trigger"| B["AGENTS.md"] + B -->|"2. loads"| C["Skill"] + C -->|"3. provides"| D["Patterns
Templates
Commands"] + C -->|"4. references"| E["Local Docs"] + D --> F["Correct Output"] + E --> F + end + + style A fill:#1e3a5f,stroke:#4a9eff,color:#fff + style B fill:#5c4d1a,stroke:#ffd700,color:#fff + style C fill:#1a4d1a,stroke:#4caf50,color:#fff + style E fill:#4a1a4d,stroke:#ba68c8,color:#fff + style F fill:#1a4d2e,stroke:#66bb6a,color:#fff +``` + +## How It Works + +```mermaid +sequenceDiagram + participant U as User + participant A as AI Agent + participant R as AGENTS.md + participant S as Skill + participant AS as assets/ + participant RF as references/ + participant D as Local Docs + + U->>A: "Create an AWS security check" + + Note over A: Analyze request context + + A->>R: Find matching skill trigger + R-->>A: prowler-sdk-check matches + + A->>S: Load SKILL.md + S-->>A: Patterns, rules, templates, commands + + Note over A: Need code template? + + A->>AS: Read assets/aws_check.py + AS-->>A: Check implementation template + + Note over A: Need more details? + + A->>RF: Read references/metadata-docs.md + RF-->>A: Points to local docs + + A->>D: Read docs/developer-guide/checks.mdx + D-->>A: Full documentation + + Note over A: Execute with full context + + A->>U: Creates check with correct patterns +``` + +## Before vs After + +```mermaid +graph TD + subgraph COMPARISON["BEFORE vs AFTER"] + direction LR + + subgraph BEFORE["Without Skills"] + B1["AI guesses conventions"] + B2["Wrong structure"] + B3["Multiple iterations"] + B4["Web searches for docs"] + B5["Inconsistent patterns"] + end + + subgraph AFTER["With Skills"] + A1["AI loads exact patterns"] + A2["Correct structure"] + A3["First-time right"] + A4["Local docs referenced"] + A5["Consistent patterns"] + end + end + + style BEFORE fill:#5c1a1a,stroke:#ef5350,color:#fff + style AFTER fill:#1a4d1a,stroke:#66bb6a,color:#fff +``` + +## Complete Architecture + +```mermaid +flowchart TB + subgraph ENTRY["ENTRY POINT"] + AGENTS["AGENTS.md
━━━━━━━━━━━━━━━━━
• Available skills registry
• Skill → Trigger mapping
• Component navigation"] + end + + subgraph SKILLS["SKILLS LIBRARY"] + direction TB + + subgraph GENERIC["Generic Skills"] + G1["typescript"] + G2["react-19"] + G3["nextjs-15"] + G4["tailwind-4"] + G5["pytest"] + G6["playwright"] + G7["django-drf"] + G8["zod-4"] + G9["zustand-5"] + G10["ai-sdk-5"] + end + + subgraph PROWLER["Prowler Skills"] + P1["prowler"] + P2["prowler-sdk-check"] + P3["prowler-api"] + P4["prowler-ui"] + P5["prowler-mcp"] + P6["prowler-provider"] + P7["prowler-compliance"] + P8["prowler-compliance-review"] + P9["prowler-docs"] + P10["prowler-pr"] + P11["prowler-ci"] + end + + subgraph TESTING["Testing Skills"] + T1["prowler-test-sdk"] + T2["prowler-test-api"] + T3["prowler-test-ui"] + end + + subgraph META["Meta Skills"] + M1["skill-creator"] + M2["skill-sync"] + end + end + + subgraph STRUCTURE["SKILL STRUCTURE"] + direction LR + + SKILLMD["SKILL.md
━━━━━━━━━━━━━━
• Frontmatter
• Critical patterns
• Decision trees
• Code examples
• Commands
• Keywords"] + + ASSETS["assets/
━━━━━━━━━━━━━━
• Code templates
• JSON schemas
• Config examples"] + + REFS["references/
━━━━━━━━━━━━━━
• Local doc paths
• No web URLs
• Single source"] + end + + subgraph DOCS["DOCUMENTATION"] + direction TB + DD["docs/developer-guide/"] + D1["checks.mdx"] + D2["unit-testing.mdx"] + D3["provider.mdx"] + D4["mcp-server.mdx"] + D5["..."] + + DD --> D1 + DD --> D2 + DD --> D3 + DD --> D4 + DD --> D5 + end + + ENTRY --> SKILLS + SKILLS --> STRUCTURE + SKILLMD --> ASSETS + SKILLMD --> REFS + REFS -.->|"points to"| DOCS + + style ENTRY fill:#1e3a5f,stroke:#4a9eff,color:#fff + style GENERIC fill:#5c4d1a,stroke:#ffd700,color:#fff + style PROWLER fill:#1a4d1a,stroke:#66bb6a,color:#fff + style TESTING fill:#4d1a3d,stroke:#f06292,color:#fff + style META fill:#4a1a4d,stroke:#ba68c8,color:#fff + style STRUCTURE fill:#5c3d1a,stroke:#ffb74d,color:#fff + style DOCS fill:#1a3d4d,stroke:#4dd0e1,color:#fff +``` + +## Skills Included + +| Type | Skills | +|------|--------| +| **Generic** | typescript, react-19, nextjs-15, tailwind-4, pytest, playwright, django-drf, zod-4, zustand-5, ai-sdk-5 | +| **Prowler** | prowler, prowler-sdk-check, prowler-api, prowler-ui, prowler-mcp, prowler-provider, prowler-compliance, prowler-compliance-review, prowler-docs, prowler-pr, prowler-ci | +| **Testing** | prowler-test-sdk, prowler-test-api, prowler-test-ui | +| **Meta** | skill-creator, skill-sync | + +## Skill Structure + +Each skill follows the [Agent Skills spec](https://agentskills.io): + +``` +skills/{skill-name}/ +├── SKILL.md # Patterns, rules, decision trees +├── assets/ # Code templates, schemas +└── references/ # Links to local docs (single source of truth) +``` + +## Key Design Decisions + +1. **Self-contained skills** - Critical patterns inline for fast loading +2. **Local doc references** - No web URLs, points to `docs/developer-guide/*.mdx` +3. **Single source of truth** - Skills reference docs, no duplication +4. **On-demand loading** - AI loads only what's needed for the task + +## Creating New Skills + +Use the `skill-creator` meta-skill to create new skills that follow the Agent Skills spec. See `AGENTS.md` for the full list of available skills and their triggers. diff --git a/docs/developer-guide/alibabacloud-details.mdx b/docs/developer-guide/alibabacloud-details.mdx new file mode 100644 index 0000000000..4c21e17b29 --- /dev/null +++ b/docs/developer-guide/alibabacloud-details.mdx @@ -0,0 +1,212 @@ +--- +title: 'Alibaba Cloud Provider' +--- + +This page details the [Alibaba Cloud](https://www.alibabacloud.com/) provider implementation in Prowler. + +By default, Prowler will audit all the Alibaba Cloud regions that are available. To configure it, follow the [Alibaba Cloud getting started guide](/user-guide/providers/alibabacloud/getting-started-alibabacloud). + +## Alibaba Cloud Provider Classes Architecture + +The Alibaba Cloud provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the Alibaba Cloud-specific implementation, highlighting how the generic provider concepts are realized for Alibaba Cloud in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). + +### Main Class + +- **Location:** [`prowler/providers/alibabacloud/alibabacloud_provider.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/alibabacloud/alibabacloud_provider.py) +- **Base Class:** Inherits from `Provider` (see [base class details](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/common/provider.py)). +- **Purpose:** Central orchestrator for Alibaba Cloud-specific logic, session management, credential validation, and configuration. +- **Key Alibaba Cloud Responsibilities:** + - Initializes and manages Alibaba Cloud sessions (supports Access Keys, STS Temporary Credentials, RAM Role Assumption, ECS RAM Role, OIDC Authentication, and Credentials URI). + - Validates credentials using STS GetCallerIdentity. + - Loads and manages configuration, mutelist, and fixer settings. + - Discovers and manages Alibaba Cloud regions. + - Provides properties and methods for downstream Alibaba Cloud service classes to access session, identity, and configuration data. + +### Data Models + +- **Location:** [`prowler/providers/alibabacloud/models.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/alibabacloud/models.py) +- **Purpose:** Define structured data for Alibaba Cloud identity, session, credentials, and region info. +- **Key Alibaba Cloud Models:** + - `AlibabaCloudCallerIdentity`: Stores caller identity information from STS GetCallerIdentity (account_id, principal_id, arn, identity_type). + - `AlibabaCloudIdentityInfo`: Holds Alibaba Cloud identity metadata including account ID, user info, profile, and audited regions. + - `AlibabaCloudCredentials`: Stores credentials (access_key_id, access_key_secret, security_token). + - `AlibabaCloudRegion`: Represents an Alibaba Cloud region with region_id and region_name. + - `AlibabaCloudSession`: Manages the session and provides methods to create service clients. + +### `AlibabaCloudService` (Service Base Class) + +- **Location:** [`prowler/providers/alibabacloud/lib/service/service.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/alibabacloud/lib/service/service.py) +- **Purpose:** Abstract base class that all Alibaba Cloud service-specific classes inherit from. This implements the generic service pattern (described in [service page](/developer-guide/services#service-base-class)) specifically for Alibaba Cloud. +- **Key Alibaba Cloud Responsibilities:** + - Receives an `AlibabacloudProvider` instance to access session, identity, and configuration. + - Manages regional clients for services that are region-specific. + - Provides `__threading_call__` method to make API calls in parallel by region or resource. + - Exposes common audit context (`audited_account`, `audited_account_name`, `audit_resources`, `audit_config`) to subclasses. + +### Exception Handling + +- **Location:** [`prowler/providers/alibabacloud/exceptions/exceptions.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/alibabacloud/exceptions/exceptions.py) +- **Purpose:** Custom exception classes for Alibaba Cloud-specific error handling. +- **Key Alibaba Cloud Exceptions:** + - `AlibabaCloudClientError`: General client errors + - `AlibabaCloudNoCredentialsError`: No credentials found + - `AlibabaCloudInvalidCredentialsError`: Invalid credentials provided + - `AlibabaCloudSetUpSessionError`: Session setup failures + - `AlibabaCloudAssumeRoleError`: RAM role assumption failures + - `AlibabaCloudInvalidRegionError`: Invalid region specified + - `AlibabaCloudHTTPError`: HTTP/API errors + +### Session and Utility Helpers + +- **Location:** [`prowler/providers/alibabacloud/lib/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/alibabacloud/lib/) +- **Purpose:** Helpers for argument parsing, mutelist management, and other cross-cutting concerns. + +## Specific Patterns in Alibaba Cloud Services + +The generic service pattern is described in [service page](/developer-guide/services#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: + +- Directly in the code, in location [`prowler/providers/alibabacloud/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/alibabacloud/services) +- In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. + +The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used across all Alibaba Cloud services. + +### Alibaba Cloud Service Common Patterns + +- Services communicate with Alibaba Cloud using the official Alibaba Cloud Python SDKs. Documentation for individual services can be found in the [Alibaba Cloud SDK documentation](https://www.alibabacloud.com/help/en/sdk). +- Every Alibaba Cloud service class inherits from `AlibabaCloudService`, ensuring access to session, identity, configuration, and client utilities. +- The constructor (`__init__`) always calls `super().__init__` with the service name, provider, and optionally `global_service=True` for services that are not regional (e.g., RAM). +- Resource containers **must** be initialized in the constructor. For regional services, resources are typically stored in dictionaries keyed by region and resource ID. +- All Alibaba Cloud resources are represented as Pydantic `BaseModel` classes, providing type safety and structured access to resource attributes. +- Alibaba Cloud SDK functions are wrapped in try/except blocks, with specific handling for errors, always logging errors. +- Regional services use `self.regional_clients` to maintain clients for each audited region. +- The `__threading_call__` method is used for parallel execution across regions or resources. + +### Example Service Implementation + +```python +from prowler.lib.logger import logger +from prowler.providers.alibabacloud.lib.service.service import AlibabaCloudService + + +class MyService(AlibabaCloudService): + def __init__(self, provider): + # Initialize parent class with service name + super().__init__("myservice", provider) + + # Initialize resource containers + self.resources = {} + + # Discover resources using threading + self.__threading_call__(self._describe_resources) + + def _describe_resources(self, regional_client): + try: + region = regional_client.region + response = regional_client.describe_resources() + + for resource in response.body.resources: + self.resources[resource.id] = MyResource( + id=resource.id, + name=resource.name, + region=region, + # ... other attributes + ) + except Exception as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) +``` + +## Specific Patterns in Alibaba Cloud Checks + +The Alibaba Cloud checks pattern is described in [checks page](/developer-guide/checks). You can find all the currently implemented checks: + +- Directly in the code, within each service folder, each check has its own folder named after the name of the check. (e.g. [`prowler/providers/alibabacloud/services/ram/ram_no_root_access_key/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/alibabacloud/services/ram/ram_no_root_access_key)) +- In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. + +The best reference to understand how to implement a new check is following the [check implementation documentation](/developer-guide/checks#creating-a-check) and taking other similar checks as reference. + +### Check Report Class + +The `CheckReportAlibabaCloud` class models a single finding for an Alibaba Cloud resource in a check report. It is defined in [`prowler/lib/check/models.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/lib/check/models.py) and inherits from the generic `Check_Report` base class. + +#### Purpose + +`CheckReportAlibabaCloud` extends the base report structure with Alibaba Cloud-specific fields, enabling detailed tracking of the resource, resource ID, ARN, and region associated with each finding. + +#### Constructor and Attribute Population + +When you instantiate `CheckReportAlibabaCloud`, you must provide the check metadata and a resource object. The class will attempt to automatically populate its Alibaba Cloud-specific attributes from the resource, using the following logic: + +- **`resource_id`**: + - Uses `resource.id` if present. + - Otherwise, uses `resource.name` if present. + - Defaults to an empty string if not available. + +- **`resource_arn`**: + - Uses `resource.arn` if present. + - Defaults to an empty string if not available. + +- **`region`**: + - Uses `resource.region` if present. + - Defaults to an empty string if not available. + +If the resource object does not contain the required attributes, you must set them manually in the check logic. + +Other attributes are inherited from the `Check_Report` class, from which you **always** have to set the `status` and `status_extended` attributes in the check logic. + +#### Example Usage + +```python +from prowler.lib.check.models import Check, CheckReportAlibabaCloud +from prowler.providers.alibabacloud.services.myservice.myservice_client import myservice_client + + +class myservice_example_check(Check): + def execute(self) -> list[CheckReportAlibabaCloud]: + findings = [] + + for resource in myservice_client.resources.values(): + report = CheckReportAlibabaCloud( + metadata=self.metadata(), + resource=resource + ) + report.region = resource.region + report.resource_id = resource.id + report.resource_arn = f"acs:myservice::{myservice_client.audited_account}:resource/{resource.id}" + + if resource.is_compliant: + report.status = "PASS" + report.status_extended = f"Resource {resource.name} is compliant." + else: + report.status = "FAIL" + report.status_extended = f"Resource {resource.name} is not compliant." + + findings.append(report) + + return findings +``` + +## Authentication Methods + +The Alibaba Cloud provider supports multiple authentication methods, prioritized in the following order: + +1. **Credentials URI** - Retrieve credentials from an external URI endpoint +2. **OIDC Role Authentication** - For applications running in ACK with RRSA enabled +3. **ECS RAM Role** - For ECS instances with attached RAM roles +4. **RAM Role Assumption** - Cross-account access with role assumption +5. **STS Temporary Credentials** - Pre-obtained temporary credentials +6. **Permanent Access Keys** - Static access key credentials +7. **Default Credential Chain** - Automatic credential discovery + +For detailed authentication configuration, see the [Authentication documentation](/user-guide/providers/alibabacloud/authentication). + +## Regions + +Alibaba Cloud has multiple regions across the globe. By default, Prowler audits all available regions. You can specify specific regions using the `--regions` CLI argument: + +```bash +prowler alibabacloud --regions cn-hangzhou cn-shanghai +``` + +The list of supported regions is maintained in [`prowler/providers/alibabacloud/config.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/alibabacloud/config.py). diff --git a/docs/developer-guide/check-metadata-guidelines.mdx b/docs/developer-guide/check-metadata-guidelines.mdx index 2b6524bcc6..7bc335c3de 100644 --- a/docs/developer-guide/check-metadata-guidelines.mdx +++ b/docs/developer-guide/check-metadata-guidelines.mdx @@ -213,3 +213,5 @@ Also is important to keep all code examples as short as possible, including the | software-supply-chain | Detects or prevents tampering, unauthorized packages, or third-party risks in software supply chain | | e3 | M365-specific controls enabled by or dependent on an E3 license (e.g., baseline security policies, conditional access) | | e5 | M365-specific controls enabled by or dependent on an E5 license (e.g., advanced threat protection, audit, DLP, and eDiscovery) | +| privilege-escalation | Detects IAM policies or permissions that allow identities to elevate their privileges beyond their intended scope, potentially gaining administrator or higher-level access through specific action combinations | +| ec2-imdsv1 | Identifies EC2 instances using Instance Metadata Service version 1 (IMDSv1), which is vulnerable to SSRF attacks and should be replaced with IMDSv2 for enhanced security | \ No newline at end of file diff --git a/docs/developer-guide/checks.mdx b/docs/developer-guide/checks.mdx index 7a86686071..1b1f1cc4c9 100644 --- a/docs/developer-guide/checks.mdx +++ b/docs/developer-guide/checks.mdx @@ -237,6 +237,7 @@ Below is a generic example of a check metadata file. **Do not include comments i "ResourceIdTemplate": "", "Severity": "medium", "ResourceType": "Other", + "ResourceGroup": "security", "Description": "This check verifies that the service resource has the required **security setting** enabled to protect against potential vulnerabilities.\n\nIt ensures that the resource follows security best practices and maintains proper access controls. The check evaluates whether the security configuration is properly implemented and active.", "Risk": "Without proper security settings, the resource may be vulnerable to:\n\n- **Unauthorized access** - Malicious actors could gain entry\n- **Data breaches** - Sensitive information could be compromised\n- **Security threats** - Various attack vectors could be exploited\n\nThis could result in compliance violations and potential financial or reputational damage.", "RelatedUrl": "", @@ -312,7 +313,33 @@ The type of resource being audited. This field helps categorize and organize fin - **Azure**: Use types from [Azure Resource Graph](https://learn.microsoft.com/en-us/azure/governance/resource-graph/reference/supported-tables-resources), for example: `Microsoft.Storage/storageAccounts`. - **Google Cloud**: Use [Cloud Asset Inventory asset types](https://cloud.google.com/asset-inventory/docs/asset-types), for example: `compute.googleapis.com/Instance`. - **Kubernetes**: Use types shown under `KIND` from `kubectl api-resources`. -- **M365 / GitHub**: Leave empty due to lack of standardized types. +- **Oracle Cloud Infrastructure**: Use types from [Oracle Cloud Infrastructure documentation](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Search/Tasks/queryingresources_topic-Listing_Supported_Resource_Types.htm). +- **M365 / GitHub / MongoDB Atlas**: Leave empty due to lack of standardized types. + +#### ResourceGroup + +A high-level classification that groups checks by the type of cloud resource they audit. This field enables filtering and organizing findings by resource category across all providers. The value must be one of the following predefined groups: + +| Group | Description | +|-------|-------------| +| `compute` | Virtual machines, instances, auto-scaling groups, workspaces, streaming | +| `container` | Container orchestration, Kubernetes, registries, pods | +| `serverless` | Functions, step functions, event-driven compute | +| `database` | Relational, NoSQL, caches, search engines, data warehouses, graph databases | +| `storage` | Object storage, block storage, file systems, backups, archives | +| `network` | VPCs, subnets, load balancers, DNS, VPN, firewalls, CDN | +| `IAM` | IAM users, roles, policies, access keys, service accounts, directories | +| `messaging` | Queues, topics, event buses, streaming, email services | +| `security` | WAF, secrets, KMS, certificates, security tools, defenders, DDoS protection | +| `monitoring` | Logs, metrics, alerts, audit trails, observability, config tracking | +| `api_gateway` | API management, REST APIs, GraphQL endpoints | +| `ai_ml` | Machine learning, AI services, notebooks, training, LLM | +| `governance` | Accounts, organizations, projects, policies, settings, compliance tools | +| `collaboration` | Productivity SaaS apps (Exchange, Teams, SharePoint) | +| `devops` | CI/CD, infrastructure as code, automation, code repositories, version control | +| `analytics` | Data warehouses, query engines, ETL pipelines, BI tools, data lakes | + +The group is determined by the resource type being audited, not the service. For example, an EC2 security group check would use `network` (not `compute`), while an EC2 instance check would use `compute`. #### Description diff --git a/docs/developer-guide/end2end-testing.mdx b/docs/developer-guide/end2end-testing.mdx new file mode 100644 index 0000000000..0a62251531 --- /dev/null +++ b/docs/developer-guide/end2end-testing.mdx @@ -0,0 +1,327 @@ +--- +title: 'End-2-End Tests for Prowler App' +--- + +End-to-end (E2E) tests validate complete user flows in Prowler App (UI + API). These tests are implemented with [Playwright](https://playwright.dev/) under the `ui/tests` folder and are designed to run against a Prowler App environment. + +## General Recommendations + +When adding or maintaining E2E tests for Prowler App, follow these guidelines: + +1. **Test real user journeys** + Focus on full workflows (for example, sign-up → login → add provider → launch scan) instead of low-level UI details already covered by unit or integration tests. + +2. **Group tests by entity or feature area** + - Organize E2E tests by entity or feature area (for example, `providers.spec.ts`, `scans.spec.ts`, `invitations.spec.ts`, `sign-up.spec.ts`). + - Each entity should have its own test file and corresponding page model class (for example, `ProvidersPage`, `ScansPage`, `InvitationsPage`). + - Related tests for the same entity should be grouped together in the same test file to improve maintainability and make it easier to find and update tests for a specific feature. + +3. **Use a Page Model (Page Object Model)** + - Encapsulate selectors and common actions in page classes instead of repeating them in each test. + - Leverage and extend the existing Playwright page models in `ui/tests`—such as `ProvidersPage`, `ScansPage`, and others—which are all based on the shared `BasePage`. + - Page models for Prowler App pages should be placed in their respective entity folders (for example, `ui/tests/providers/providers-page.ts`). + - Page models for external pages (not part of Prowler App) should be grouped in the `external` folder (for example, `ui/tests/external/github-page.ts`). + - This approach improves readability, reduces duplication, and makes refactors safer. + +4. **Reuse authentication states (StorageState)** + - Multiple authentication setup projects are available that generate pre-authenticated state files stored in `playwright/.auth/`. Each project requires specific environment variables: + - `admin.auth.setup` – Admin users with full system permissions (requires `E2E_ADMIN_USER` / `E2E_ADMIN_PASSWORD`) + - `manage-scans.auth.setup` – Users with scan management permissions (requires `E2E_MANAGE_SCANS_USER` / `E2E_MANAGE_SCANS_PASSWORD`) + - `manage-integrations.auth.setup` – Users with integration management permissions (requires `E2E_MANAGE_INTEGRATIONS_USER` / `E2E_MANAGE_INTEGRATIONS_PASSWORD`) + - `manage-account.auth.setup` – Users with account management permissions (requires `E2E_MANAGE_ACCOUNT_USER` / `E2E_MANAGE_ACCOUNT_PASSWORD`) + - `manage-cloud-providers.auth.setup` – Users with cloud provider management permissions (requires `E2E_MANAGE_CLOUD_PROVIDERS_USER` / `E2E_MANAGE_CLOUD_PROVIDERS_PASSWORD`) + - `unlimited-visibility.auth.setup` – Users with unlimited visibility permissions (requires `E2E_UNLIMITED_VISIBILITY_USER` / `E2E_UNLIMITED_VISIBILITY_PASSWORD`) + - `invite-and-manage-users.auth.setup` – Users with user invitation and management permissions (requires `E2E_INVITE_AND_MANAGE_USERS_USER` / `E2E_INVITE_AND_MANAGE_USERS_PASSWORD`) + + If fixtures have been applied (fixtures are used to populate the database with initial development data), you can use the user `e2e@prowler.com` with password `Thisisapassword123@` to configure the Admin credentials by setting `E2E_ADMIN_USER=e2e@prowler.com` and `E2E_ADMIN_PASSWORD=Thisisapassword123@`. + + + - Within test files, use `test.use({ storageState: "playwright/.auth/admin_user.json" })` to load the pre-authenticated state, avoiding redundant authentication steps in each test. This must be placed at the test level (not inside the test function) to apply the authentication state to all tests in that scope. This approach is preferred over declaring dependencies in `playwright.config.ts` because it provides more control over which authentication states are used in specific tests. + + **Example:** + + ```typescript + // Use admin authentication state for all tests in this scope + test.use({ storageState: "playwright/.auth/admin_user.json" }); + + test("should perform admin action", async ({ page }) => { + // Test implementation + }); + ``` + +5. **Tag and document scenarios** + - Follow the existing naming convention for suites and test cases (for example, `SCANS-E2E-001`, `PROVIDER-E2E-003`) and use tags such as `@e2e`, `@serial` and feature tags (for example, `@providers`, `@scans`,`@aws`) to filter and organize tests. + + **Example:** + ```typescript + test( + "should add a new AWS provider with static credentials", + { + tag: [ + "@critical", + "@e2e", + "@providers", + "@aws", + "@serial", + "@PROVIDER-E2E-001", + ], + }, + async ({ page }) => { + // Test implementation + } + ); + ``` + - Document each one in the Markdown files under `ui/tests`, including **Priority**, **Tags**, **Description**, **Preconditions**, **Flow steps**, **Expected results**,**Key verification points** and **Notes**. + + **Example** + ```Markdown + ## Test Case: `SCANS-E2E-001` - Execute On-Demand Scan + + **Priority:** `critical` + + **Tags:** + + - type → @e2e, @serial + - feature → @scans + + **Description/Objective:** Validates the complete flow to execute an on-demand scan selecting a provider by UID and confirming success on the Scans page. + + **Preconditions:** + + - Admin user authentication required (admin.auth.setup setup) + - Environment variables configured for : E2E_AWS_PROVIDER_ACCOUNT_ID,E2E_AWS_PROVIDER_ACCESS_KEY and E2E_AWS_PROVIDER_SECRET_KEY + - Remove any existing AWS provider with the same Account ID before starting the test + - This test must be run serially and never in parallel with other tests, as it requires the Account ID Provider to be already registered. + + ### Flow Steps: + + 1. Navigate to Scans page + 2. Open provider selector and choose the entry whose text contains E2E_AWS_PROVIDER_ACCOUNT_ID + 3. Optionally fill scan label (alias) + 4. Click "Start now" to launch the scan + 5. Verify the success toast appears + 6. Verify a row in the Scans table contains the provided scan label (or shows the new scan entry) + + ### Expected Result: + + - Scan is launched successfully + - Success toast is displayed to the user + - Scans table displays the new scan entry (including the alias when provided) + + ### Key verification points: + + - Scans page loads correctly + - Provider select is available and lists the configured provider UID + - "Start now" button is rendered and enabled when form is valid + - Success toast message: "The scan was launched successfully." + - Table contains a row with the scan label or new scan state (queued/available/executing) + + ### Notes: + + - The table may take a short time to reflect the new scan; assertions look for a row containing the alias. + - Provider cleanup performed before each test to ensure clean state + - Tests should run serially to avoid state conflicts. + + ``` + +6. **Use environment variables for secrets and dynamic data** + Credentials, provider identifiers, secrets, tokens must come from environment variables (for example, `E2E_AWS_PROVIDER_ACCOUNT_ID`, `E2E_AWS_PROVIDER_ACCESS_KEY`, `E2E_AWS_PROVIDER_SECRET_KEY`, `E2E_GCP_PROJECT_ID`). + + + Never commit real secrets, tokens, or account IDs to the repository. + + +7. **Keep tests deterministic and isolated** + - Use Playwright's `test.beforeEach()` and `test.afterEach()` hooks to manage test state: + - **`test.beforeEach()`**: Execute cleanup or setup logic before each test runs (for example, delete existing providers with a specific account ID to ensure a clean state). + - **`test.afterEach()`**: Execute cleanup logic after each test completes (for example, remove test data created during the test execution to prevent interference with subsequent tests). + - Define tests as serial using `test.describe.serial()` when they share state or resources that could interfere with parallel execution (for example, tests that use the same provider account ID or create dependent resources). This ensures tests within the serial group run sequentially, preventing race conditions and data conflicts. + - Use unique identifiers (for example, random suffixes for emails or labels) to prevent data collisions. + +8. **Use explicit waiting strategies** + - Avoid using `waitForLoadState('networkidle')` as it is unreliable and can lead to flaky tests or unnecessary delays. + - Leverage Playwright's auto-waiting capabilities by waiting for specific elements to be actionable (for example, `locator.click()`, `locator.fill()`, `locator.waitFor()`). + - **Prioritize selector strategies**: Prefer `page.getByRole()` over other approaches like `page.getByText()`. `getByRole()` is more resilient to UI changes, aligns with accessibility best practices, and better reflects how users interact with the application (by role and accessible name rather than implementation details). + - For dynamic content, wait for specific UI elements that indicate the page is ready (for example, button becoming enabled, a specific text appearing, etc). + - This approach makes tests more reliable, faster, and aligned with how users actually interact with the application. + + **Common waiting patterns used in Prowler E2E tests:** + + - **Element visibility assertions**: Use `expect(locator).toBeVisible()` or `expect(locator).not.toBeVisible()` to wait for elements to appear or disappear (Playwright automatically waits for these conditions). + + - **URL changes**: Use `expect(page).toHaveURL(url)` or `page.waitForURL(url)` to wait for navigation to complete. + + - **Element states**: Use `locator.waitFor({ state: "visible" })` or `locator.waitFor({ state: "hidden" })` when you need explicit state control. + + - **Text content**: Use `expect(locator).toHaveText(text)` or `expect(locator).toContainText(text)` to wait for specific text to appear. + + - **Element attributes**: Use `expect(locator).toHaveAttribute(name, value)` to wait for attributes like `aria-disabled="false"` indicating a button is enabled. + + - **Custom conditions**: Use `page.waitForFunction(() => condition)` for complex conditions that cannot be expressed with locators (for example, checking DOM element dimensions or computed styles). + + - **Retryable assertions**: Use `expect(async () => { ... }).toPass({ timeout })` for conditions that may take time to stabilize (for example, waiting for table rows to filter after a server request). + + - **Scroll into view**: Use `locator.scrollIntoViewIfNeeded()` before interacting with elements that may be outside the viewport. + + **Example from Prowler tests:** + + ```typescript + // Wait for page to load by checking main content is visible + await expect(page.locator("main")).toBeVisible(); + + // Wait for URL change after form submission + await expect(page).toHaveURL("/providers"); + + // Wait for button to become enabled + await expect(submitButton).toHaveAttribute("aria-disabled", "false"); + + // Wait for loading spinner to disappear + await expect(page.getByText("Loading")).not.toBeVisible(); + + // Wait for custom condition + await page.waitForFunction(() => { + const main = document.querySelector("main"); + return main && main.offsetHeight > 0; + }); + + // Wait for retryable condition (e.g., table filtering) + await expect(async () => { + const rowCount = await tableRows.count(); + expect(rowCount).toBeLessThanOrEqual(1); + }).toPass({ timeout: 20000 }); + ``` + +## Running Prowler Tests + +E2E tests for Prowler App run from the `ui` project using Playwright. The Playwright configuration lives in `ui/playwright.config.ts` and defines: + +- `testDir: "./tests"` – location of E2E test files (relative to the `ui` project root, so `ui/tests`). +- `webServer` – how to start the Next.js development server and connect to Prowler API. +- `use.baseURL` – base URL for browser interactions (defaults to `http://localhost:3000` or `AUTH_URL` if set). +- `reporter: [["list"]]` – uses the list reporter to display test results in a concise format in the terminal. Other reporter options are available (for example, `html`, `json`, `junit`, `github`), and multiple reporters can be configured simultaneously. See the [Playwright reporter documentation](https://playwright.dev/docs/test-reporters) for all available options. +- `expect.timeout: 20000` – timeout for assertions (20 seconds). This is the maximum time Playwright will wait for an assertion to pass before considering it failed. +- **Test artifacts** (in `use` configuration): By default, `trace`, `screenshot`, and `video` are set to `"off"` to minimize resource usage. To review test failures or debug issues, these can be enabled in `playwright.config.ts` by changing them to `"on"`, `"on-first-retry"`, or `"retain-on-failure"` depending on your needs. +- `outputDir: "/tmp/playwright-tests"` – directory where Playwright stores test artifacts (screenshots, videos, traces) during test execution. +- **CI-specific configuration**: The configuration uses different settings when running in CI environments (detected via `process.env.CI`): + - **Retries**: `2` retries in CI (to handle flaky tests), `0` retries locally (for faster feedback during development). + - **Workers**: `1` worker in CI (sequential execution for stability), `undefined` locally (parallel execution by default for faster test runs). + +### Prerequisites + +Before running E2E tests: + +- **Install root and UI dependencies** + - Follow the [developer guide introduction](/developer-guide/introduction#getting-the-code-and-installing-all-dependencies) to clone the repository and install core dependencies. + - From the `ui` directory, install frontend dependencies: + + ```bash + cd ui + pnpm install + pnpm run test:e2e:install # Install Playwright browsers + ``` + +- **Ensure Prowler API is available** + - By default, Playwright uses `NEXT_PUBLIC_API_BASE_URL=http://localhost:8080/api/v1` (configured in `playwright.config.ts`). + - Start Prowler API so it is reachable on that URL (for example, via `docker-compose-dev.yml` or the development orchestration used locally). + - If a different API URL is required, set `NEXT_PUBLIC_API_BASE_URL` accordingly before running the tests. + +- **Ensure Prowler App UI is available** + - Playwright automatically starts the Next.js server through the `webServer` block in `playwright.config.ts` (`pnpm run dev` by default). + - If the UI is already running on `http://localhost:3000`, Playwright will reuse the existing server when `reuseExistingServer` is `true`. + +- **Configure E2E environment variables** + - Suite-specific variables (for example, provider account IDs, credentials, and E2E user data) must be provided before running tests. + - They can be defined either: + - As exported environment variables in the shell before executing the Playwright commands, or + - In a `.env.local` or `.env` file under `ui/`, and then loaded into the shell before running tests, for example: + + ```bash + cd ui + set -a + source .env.local # or .env + set +a + ``` + - Refer to the Markdown documentation files in `ui/tests` for each E2E suite (for example, the `*.md` files that describe sign-up, providers, scans, invitations, and other flows) to see the exact list of required variables and their meaning. + - Each E2E test suite explicitly checks that its required environment variables are defined at runtime and will fail with a clear error message if any mandatory variable is missing, making misconfiguration easy to detect. + +### Executing Tests + +To execute E2E tests for Prowler App: + +1. **Run the full E2E suite (headless)** + + From the `ui` directory: + + ```bash + pnpm run test:e2e + ``` + + This command runs Playwright with the configured projects + +2. **Run E2E tests with the Playwright UI runner** + + ```bash + pnpm run test:e2e:ui + ``` + + This opens the Playwright test runner UI to inspect, debug, and rerun specific tests or projects. + +3. **Debug E2E tests interactively** + + ```bash + pnpm run test:e2e:debug + ``` + + Use this mode to step through flows, inspect selectors, and adjust timings. It runs tests in headed mode with debugging tools enabled. + +4. **Run tests in headed mode without debugger** + + ```bash + pnpm run test:e2e:headed + ``` + + This is useful to visually confirm flows while still running the full suite. + +5. **View previous test reports** + + ```bash + pnpm run test:e2e:report + ``` + + This opens the latest Playwright HTML report, including traces and screenshots when enabled. + +6. **Run specific tests or subsets** + + In addition to the predefined scripts, Playwright allows filtering which tests run. These examples use the Playwright CLI directly through `pnpm`: + + - **By test ID (`@ID` in the test metadata or description)** + + To run a single test case identified by its ID (for example, `@PROVIDER-E2E-001` or `@SCANS-E2E-001`): + + ```bash + pnpm playwright test --grep @PROVIDER-E2E-001 + ``` + + - **By tags** + + To run all tests that share a common tag (for example, all provider E2E tests tagged with `@providers`): + + ```bash + pnpm playwright test --grep @providers + ``` + + This is useful to focus on a specific feature area such as providers, scans, invitations, or sign-up. + + - **By Playwright project** + + To run only the tests associated with a given project defined in `playwright.config.ts` (for example, `providers` or `scans`): + + ```bash + pnpm playwright test --project=providers + ``` + + Combining project and grep filters is also supported, enabling very narrow runs (for example, a single test ID within the `providers` project). For additional CLI options and combinations, see the [Playwright command line documentation](https://playwright.dev/docs/test-cli). + + +For detailed flows, preconditions, and environment variable requirements per feature, always refer to the Markdown files in `ui/tests`. Those documents are the single source of truth for business expectations and validation points in each E2E suite. + diff --git a/docs/developer-guide/introduction.mdx b/docs/developer-guide/introduction.mdx index 9d1ea742b1..3f77c53ab5 100644 --- a/docs/developer-guide/introduction.mdx +++ b/docs/developer-guide/introduction.mdx @@ -6,6 +6,10 @@ Thanks for your interest in contributing to Prowler! Prowler can be extended in various ways. This guide provides the different ways to contribute and how to get started. + +Maintainers will assess whether a change fits the project roadmap and scope before merging. + + ## Contributing to Prowler ### Review Current Issues @@ -32,6 +36,9 @@ Prowler is constantly evolving. Contributions to checks, services, or integratio If you would like to extend Prowler to work with a new cloud provider, this typically involves setting up new services and checks to ensure compatibility. + + Need to ensure Prowler supports a specific compliance framework? Add new security compliance frameworks to map checks against regulatory or industry standards. + Want to tailor how results are displayed or exported? You can add custom output formats. @@ -213,4 +220,4 @@ pipx install "git+https://github.com/prowler-cloud/prowler.git@branch-name" Replace `branch-name` with the name of the branch you want to test. This will install Prowler in an isolated environment, allowing you to try out the changes safely. -For more details on testing go to the [Testing section](/developer-guide/unit-testing) of this documentation. \ No newline at end of file +For more details on testing go to the [Testing section](/developer-guide/unit-testing) of this documentation. diff --git a/docs/developer-guide/lighthouse-architecture.mdx b/docs/developer-guide/lighthouse-architecture.mdx new file mode 100644 index 0000000000..38ee50c5b8 --- /dev/null +++ b/docs/developer-guide/lighthouse-architecture.mdx @@ -0,0 +1,407 @@ +--- +title: 'Lighthouse AI Architecture' +--- + +This document describes the internal architecture of Prowler Lighthouse AI, enabling developers to understand how components interact and where to add new functionality. + + +**Looking for user documentation?** See: +- [Lighthouse AI Overview](/getting-started/products/prowler-lighthouse-ai) - Capabilities and FAQs +- [How Lighthouse AI Works](/user-guide/tutorials/prowler-app-lighthouse) - Configuration and usage +- [Multi-LLM Provider Setup](/user-guide/tutorials/prowler-app-lighthouse-multi-llm) - Provider configuration + + +## Architecture Overview + +Lighthouse AI operates as a Langchain-based agent that connects Large Language Models (LLMs) with Prowler security data through the Model Context Protocol (MCP). + +Prowler Lighthouse Architecture +Prowler Lighthouse Architecture + +### Three-Tier Architecture + +The system follows a three-tier architecture: + +1. **Frontend (Next.js)**: Chat interface, message rendering, model selection +2. **API Route**: Request handling, authentication, stream transformation +3. **Langchain Agent**: LLM orchestration, tool calling through MCP + +### Request Flow + +When a user sends a message through the Lighthouse chat interface, the system processes it through several stages: + +1. **User Submits a Message**. + The chat component (`ui/components/lighthouse/chat.tsx`) captures the user's question (e.g., "What are my critical findings in AWS?") and sends it as an HTTP POST request to the backend API route. + +2. **Authentication and Context Assembly**. + The API route (`ui/app/api/lighthouse/analyst/route.ts`) validates the user's session, extracts the JWT token (stored via `auth-context.ts`), and gathers context including the tenant's business context and current security posture data (assembled in `data.ts`). + +3. **Agent Initialization**. + The workflow orchestrator (`ui/lib/lighthouse/workflow.ts`) creates a Langchain agent configured with: + - The selected LLM, instantiated through the factory (`llm-factory.ts`) + - A system prompt containing available tools and instructions (`system-prompt.ts`) + - Two meta-tools (`describe_tool` and `execute_tool`) for accessing Prowler data + +4. **LLM Reasoning and Tool Calling**. + The agent sends the conversation to the LLM, which decides whether to respond directly or call tools to fetch data. When tools are needed, the meta-tools in `ui/lib/lighthouse/tools/meta-tool.ts` interact with the MCP client (`mcp-client.ts`) to: + - First call `describe_tool` to understand the tool's parameters + - Then call `execute_tool` to retrieve data from the MCP Server + - Continue reasoning with the returned data + +5. **Streaming Response**. + As the LLM generates its response, the stream handler (`ui/lib/lighthouse/analyst-stream.ts`) transforms Langchain events into UI-compatible messages and streams tokens back to the browser in real-time using Server-Sent Events. The stream includes both text tokens and tool execution events (displayed as "chain of thought"). + +6. **Message Rendering**. + The frontend receives the stream and renders it through `message-item.tsx` with markdown formatting. Any tool calls that occurred during reasoning are displayed via `chain-of-thought-display.tsx`. + +## Frontend Components + +Frontend components reside in `ui/components/lighthouse/` and handle the chat interface and configuration workflows. + +### Core Components + +| Component | Location | Purpose | +|-----------|----------|---------| +| `chat.tsx` | `ui/components/lighthouse/` | Main chat interface managing message history and input handling | +| `message-item.tsx` | `ui/components/lighthouse/` | Individual message rendering with markdown support | +| `select-model.tsx` | `ui/components/lighthouse/` | Model and provider selection dropdown | +| `chain-of-thought-display.tsx` | `ui/components/lighthouse/` | Displays tool calls and reasoning steps during execution | + +### Configuration Components + +| Component | Location | Purpose | +|-----------|----------|---------| +| `lighthouse-settings.tsx` | `ui/components/lighthouse/` | Settings panel for business context and preferences | +| `connect-llm-provider.tsx` | `ui/components/lighthouse/` | Provider connection workflow | +| `llm-providers-table.tsx` | `ui/components/lighthouse/` | Provider management table | +| `forms/delete-llm-provider-form.tsx` | `ui/components/lighthouse/forms/` | Provider deletion confirmation dialog | + +### Supporting Components + +| Component | Location | Purpose | +|-----------|----------|---------| +| `banner.tsx` / `banner-client.tsx` | `ui/components/lighthouse/` | Status banners and notifications | +| `workflow/` | `ui/components/lighthouse/workflow/` | Multi-step configuration workflows | +| `ai-elements/` | `ui/components/lighthouse/ai-elements/` | Custom UI primitives for chat interface (input, select, dropdown, tooltip) | + +## Library Code + +Core library code resides in `ui/lib/lighthouse/` and handles agent orchestration, MCP communication, and stream processing. + +### Workflow Orchestrator + +**Location:** `ui/lib/lighthouse/workflow.ts` + +The workflow module serves as the core orchestrator, responsible for: + +- Initializing the Langchain agent with system prompt and tools +- Loading tenant configuration (default provider, model, business context) +- Creating the LLM instance through the factory +- Generating dynamic tool listings from available MCP tools + +```typescript +// Simplified workflow initialization +export async function initLighthouseWorkflow(runtimeConfig?: RuntimeConfig) { + await initializeMCPClient(); + + const toolListing = generateToolListing(); + const systemPrompt = LIGHTHOUSE_SYSTEM_PROMPT_TEMPLATE.replace( + "{{TOOL_LISTING}}", + toolListing, + ); + + const llm = createLLM({ + provider: providerType, + model: modelId, + credentials, + // ... + }); + + return createAgent({ + model: llm, + tools: [describeTool, executeTool], + systemPrompt, + }); +} +``` + +### MCP Client Manager + +**Location:** `ui/lib/lighthouse/mcp-client.ts` + +The MCP client manages connections to the Prowler MCP Server using a singleton pattern: + +- **Connection Management**: Retry logic with configurable attempts and delays +- **Tool Discovery**: Fetches available tools from MCP server on initialization +- **Authentication Injection**: Automatically adds JWT tokens to `prowler_app_*` tool calls +- **Reconnection**: Supports forced reconnection after server restarts + +Key constants: +- `MAX_RETRY_ATTEMPTS`: 3 connection attempts +- `RETRY_DELAY_MS`: 2000ms between retries +- `RECONNECT_INTERVAL_MS`: 5 minutes before retry after failure + +```typescript +// Authentication injection for Prowler App tools +private handleBeforeToolCall = ({ name, args }) => { + // Only inject auth for prowler_app_* tools (user-specific data) + if (!name.startsWith("prowler_app_")) { + return { args }; + } + + const accessToken = getAuthContext(); + return { + args, + headers: { Authorization: `Bearer ${accessToken}` }, + }; +}; +``` + +### Meta-Tools + +**Location:** `ui/lib/lighthouse/tools/meta-tool.ts` + +Instead of registering all MCP tools directly with the agent, Lighthouse uses two meta-tools for dynamic tool discovery and execution: + +| Tool | Purpose | +|------|---------| +| `describe_tool` | Retrieves full schema and parameter details for a specific tool | +| `execute_tool` | Executes a tool with provided parameters | + +This pattern reduces the number of tools the LLM must track while maintaining access to all MCP capabilities. + +### Additional Library Modules + +| Module | Location | Purpose | +|--------|----------|---------| +| `analyst-stream.ts` | `ui/lib/lighthouse/` | Transforms Langchain stream events to UI message format | +| `llm-factory.ts` | `ui/lib/lighthouse/` | Creates LLM instances for OpenAI, Bedrock, and OpenAI-compatible providers | +| `system-prompt.ts` | `ui/lib/lighthouse/` | System prompt template with dynamic tool listing injection | +| `auth-context.ts` | `ui/lib/lighthouse/` | AsyncLocalStorage for JWT token propagation across async boundaries | +| `types.ts` | `ui/lib/lighthouse/` | TypeScript type definitions | +| `constants.ts` | `ui/lib/lighthouse/` | Configuration constants and error messages | +| `utils.ts` | `ui/lib/lighthouse/` | Message conversion and model parameter extraction | +| `validation.ts` | `ui/lib/lighthouse/` | Input validation utilities | +| `data.ts` | `ui/lib/lighthouse/` | Current data section generation for context enrichment | + +## API Route + +**Location:** `ui/app/api/lighthouse/analyst/route.ts` + +The API route handles chat requests and manages the streaming response pipeline: + +1. **Request Parsing**: Extracts messages, model, and provider from request body +2. **Authentication**: Validates session and extracts access token +3. **Context Assembly**: Gathers business context and current data +4. **Agent Initialization**: Creates Langchain agent with runtime configuration +5. **Stream Processing**: Transforms agent events to UI-compatible format +6. **Error Handling**: Captures errors with Sentry integration + +```typescript +export async function POST(req: Request) { + const { messages, model, provider } = await req.json(); + + const session = await auth(); + if (!session?.accessToken) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + return await authContextStorage.run(accessToken, async () => { + const app = await initLighthouseWorkflow(runtimeConfig); + const agentStream = app.streamEvents({ messages }, { version: "v2" }); + + // Transform stream events to UI format + const stream = new ReadableStream({ + async start(controller) { + for await (const streamEvent of agentStream) { + // Handle on_chat_model_stream, on_tool_start, on_tool_end, etc. + } + }, + }); + + return createUIMessageStreamResponse({ stream }); + }); +} +``` + +## Backend Components + +Backend components handle LLM provider configuration, model management, and credential storage. + +### Database Models + +**Location:** `api/src/backend/api/models.py` + +| Model | Purpose | +|-------|---------| +| `LighthouseProviderConfiguration` | Per-tenant LLM provider credentials (encrypted with Fernet) | +| `LighthouseTenantConfiguration` | Tenant-level settings including business context and default provider/model | +| `LighthouseProviderModels` | Available models per provider configuration | + +All models implement Row-Level Security (RLS) for tenant isolation. + +#### LighthouseProviderConfiguration + +Stores provider-specific credentials for each tenant: + +- **provider_type**: `openai`, `bedrock`, or `openai_compatible` +- **credentials**: Encrypted JSON containing API keys or AWS credentials +- **base_url**: Custom endpoint for OpenAI-compatible providers +- **is_active**: Connection validation status + +#### LighthouseTenantConfiguration + +Stores tenant-wide Lighthouse settings: + +- **business_context**: Optional context for personalized responses +- **default_provider**: Default LLM provider type +- **default_models**: JSON mapping provider types to default model IDs + +#### LighthouseProviderModels + +Catalogs available models for each provider: + +- **model_id**: Provider-specific model identifier +- **model_name**: Human-readable display name +- **default_parameters**: Optional model-specific parameters + +### Background Jobs + +**Location:** `api/src/backend/tasks/jobs/lighthouse_providers.py` + +#### check_lighthouse_provider_connection + +Validates provider credentials by making a test API call: + +- OpenAI: Lists models via `client.models.list()` +- Bedrock: Lists foundation models via `bedrock_client.list_foundation_models()` +- OpenAI-compatible: Lists models via custom base URL + +Updates `is_active` status based on connection result. + +#### refresh_lighthouse_provider_models + +Synchronizes available models from provider APIs: + +- Fetches current model catalog from provider +- Filters out non-chat models (DALL-E, Whisper, TTS, embeddings) +- Upserts model records in `LighthouseProviderModels` +- Removes stale models no longer available + +**Excluded OpenAI model prefixes:** +```python +EXCLUDED_OPENAI_MODEL_PREFIXES = ( + "dall-e", "whisper", "tts-", "sora", + "text-embedding", "text-moderation", + # Legacy models + "text-davinci", "davinci", "curie", "babbage", "ada", +) +``` + +## MCP Server Integration + +Lighthouse AI communicates with the Prowler MCP Server to access security data. For detailed MCP Server architecture, see [Extending the MCP Server](/developer-guide/mcp-server). + +### Tool Namespacing + +MCP tools are organized into three namespaces based on authentication requirements: + +| Namespace | Auth Required | Description | +|-----------|---------------|-------------| +| `prowler_app_*` | Yes (JWT) | Prowler Cloud/App tools for findings, providers, scans, resources | +| `prowler_hub_*` | No | Security checks catalog, compliance frameworks | +| `prowler_docs_*` | No | Documentation search and retrieval | + +### Authentication Flow + +1. User authenticates with Prowler App, receiving a JWT token +2. Token is stored in session and propagated via `authContextStorage` +3. MCP client injects `Authorization: Bearer ` header for `prowler_app_*` calls +4. MCP Server validates token and applies RLS filtering + +### Tool Execution Pattern + +The agent uses meta-tools rather than direct tool registration: + +``` +Agent needs data → describe_tool("prowler_app_search_findings") + → Returns parameter schema → execute_tool with parameters + → MCP client adds auth header → MCP Server executes + → Results returned to agent → Agent continues reasoning +``` + +## Extension Points + +### Adding New LLM Providers + +To add a new LLM provider: + +1. **Frontend**: Update `ui/lib/lighthouse/llm-factory.ts` with provider-specific initialization +2. **Backend**: Add provider type to `LighthouseProviderConfiguration.LLMProviderChoices` +3. **Jobs**: Add credential extraction and model fetching in `lighthouse_providers.py` +4. **UI**: Add connection workflow in `ui/components/lighthouse/workflow/` + +### Modifying System Prompt + +The system prompt template lives in `ui/lib/lighthouse/system-prompt.ts`. The `{{TOOL_LISTING}}` placeholder is dynamically replaced with available MCP tools during agent initialization. + +### Adding Stream Events + +To handle new Langchain stream events, modify `ui/lib/lighthouse/analyst-stream.ts`. Current handlers include: + +- `on_chat_model_stream`: Token-by-token text streaming +- `on_chat_model_end`: Model completion with tool call detection +- `on_tool_start`: Tool execution started +- `on_tool_end`: Tool execution completed + +### Adding MCP Tools + +See [Extending the MCP Server](/developer-guide/mcp-server) for detailed instructions on adding new tools to the Prowler MCP Server. + +## Configuration + +### Environment Variables + +| Variable | Description | +|----------|-------------| +| `PROWLER_MCP_SERVER_URL` | MCP server endpoint (e.g., `https://mcp.prowler.com/mcp`) | + +### Database Configuration + +Provider credentials are stored encrypted in `LighthouseProviderConfiguration`: + +- **OpenAI**: `{"api_key": "sk-..."}` +- **Bedrock**: `{"access_key_id": "...", "secret_access_key": "...", "region": "us-east-1"}` or `{"api_key": "...", "region": "us-east-1"}` +- **OpenAI-compatible**: `{"api_key": "..."}` with `base_url` field + +### Tenant Configuration + +Business context and default settings are stored in `LighthouseTenantConfiguration`: + +```python +{ + "business_context": "Optional organization context for personalized responses", + "default_provider": "openai", + "default_models": { + "openai": "gpt-4o", + "bedrock": "anthropic.claude-3-5-sonnet-20240620-v1:0" + } +} +``` + +## Related Documentation + + + + Adding new tools to the Prowler MCP Server + + + Capabilities, FAQs, and limitations + + + Configuring multiple LLM providers + + + User-facing architecture and setup guide + + diff --git a/docs/developer-guide/lighthouse.mdx b/docs/developer-guide/lighthouse.mdx deleted file mode 100644 index 25afd51728..0000000000 --- a/docs/developer-guide/lighthouse.mdx +++ /dev/null @@ -1,140 +0,0 @@ ---- -title: 'Extending Prowler Lighthouse AI' ---- - -This guide helps developers customize and extend Prowler Lighthouse AI by adding or modifying AI agents. - -## Understanding AI Agents - -AI agents combine Large Language Models (LLMs) with specialized tools that provide environmental context. These tools can include API calls, system command execution, or any function-wrapped capability. - -### Types of AI Agents - -AI agents fall into two main categories: - -- **Autonomous Agents**: Freely chooses from available tools to complete tasks, adapting their approach based on context. They decide which tools to use and when. -- **Workflow Agents**: Follows structured paths with predefined logic. They execute specific tool sequences and can include conditional logic. - -Prowler Lighthouse AI is an autonomous agent - selecting the right tool(s) based on the users query. - - -To learn more about AI agents, read [Anthropic's blog post on building effective agents](https://www.anthropic.com/engineering/building-effective-agents). - - -### LLM Dependency - -The autonomous nature of agents depends on the underlying LLM. Autonomous agents using identical system prompts and tools but powered by different LLM providers might approach user queries differently. Agent with one LLM might solve a problem efficiently, while with another it might take a different route or fail entirely. - -After evaluating multiple LLM providers (OpenAI, Gemini, Claude, LLama) based on tool calling features and response accuracy, we recommend using the `gpt-4o` model. - -## Prowler Lighthouse AI Architecture - -Prowler Lighthouse AI uses a multi-agent architecture orchestrated by the [Langgraph-Supervisor](https://www.npmjs.com/package/@langchain/langgraph-supervisor) library. - -### Architecture Components - -Prowler Lighthouse architecture - -Prowler Lighthouse AI integrates with the NextJS application: - -- The [Langgraph-Supervisor](https://www.npmjs.com/package/@langchain/langgraph-supervisor) library integrates directly with NextJS -- The system uses the authenticated user session to interact with the Prowler API server -- Agents only access data the current user is authorized to view -- Session management operates automatically, ensuring Role-Based Access Control (RBAC) is maintained - -## Available Prowler AI Agents - -The following specialized AI agents are available in Prowler: - -### Agent Overview - -- **provider_agent**: Fetches information about cloud providers connected to Prowler -- **user_info_agent**: Retrieves information about Prowler users -- **scans_agent**: Fetches information about Prowler scans -- **compliance_agent**: Retrieves compliance overviews across scans -- **findings_agent**: Fetches information about individual findings across scans -- **overview_agent**: Retrieves overview information (providers, findings by status and severity, etc.) - -## How to Add New Capabilities - -### Updating the Supervisor Prompt - -The supervisor agent controls system behavior, tone, and capabilities. You can find the supervisor prompt at: [https://github.com/prowler-cloud/prowler/blob/master/ui/lib/lighthouse/prompts.ts](https://github.com/prowler-cloud/prowler/blob/master/ui/lib/lighthouse/prompts.ts) - -#### Supervisor Prompt Modifications - -Modifying the supervisor prompt allows you to: - -- Change personality or response style -- Add new high-level capabilities -- Modify task delegation to specialized agents -- Set up guardrails (query types to answer or decline) - - -The supervisor agent should not have its own tools. This design keeps the system modular and maintainable. - - -### How to Create New Specialized Agents - -The supervisor agent and all specialized agents are defined in the `route.ts` file. The supervisor agent uses [langgraph-supervisor](https://www.npmjs.com/package/@langchain/langgraph-supervisor), while other agents use the prebuilt [create-react-agent](https://langchain-ai.github.io/langgraphjs/how-tos/create-react-agent/). - -To add new capabilities or all Lighthouse AI to interact with other APIs, create additional specialized agents: - -1. First determine what the new agent would do. Create a detailed prompt defining the agent's purpose and capabilities. You can see an example from [here](https://github.com/prowler-cloud/prowler/blob/master/ui/lib/lighthouse/prompts.ts#L359-L385). - -Ensure that the new agent's capabilities don't collide with existing agents. For example, if there's already a *findings_agent* that talks to findings APIs don't create a new agent to do the same. - - -2. Create necessary tools for the agents to access specific data or perform actions. A tool is a specialized function that extends the capabilities of LLM by allowing it to access external data or APIs. A tool is triggered by LLM based on the description of the tool and the user's query. -For example, the description of `getScanTool` is "Fetches detailed information about a specific scan by its ID." If the description doesn't convey what the tool is capable of doing, LLM will not invoke the function. If the description of `getScanTool` was set to something random or not set at all, LLM will not answer queries like "Give me the critical issues from the scan ID xxxxxxxxxxxxxxx" - -Ensure that one tool is added to one agent only. Adding tools is optional. There can be agents with no tools at all. - - -3. Use the `createReactAgent` function to define a new agent. For example, the rolesAgent name is "roles_agent" and has access to call tools "*getRolesTool*" and "*getRoleTool*" -```js -const rolesAgent = createReactAgent({ - llm: llm, - tools: [getRolesTool, getRoleTool], - name: "roles_agent", - prompt: rolesAgentPrompt, -}); -``` - -4. Create a detailed prompt defining the agent's purpose and capabilities. - -5. Add the new agent to the available agents list: -```js -const agents = [ - userInfoAgent, - providerAgent, - overviewAgent, - scansAgent, - complianceAgent, - findingsAgent, - rolesAgent, // New agent added here -]; -// Create supervisor workflow -const workflow = createSupervisor({ - agents: agents, - llm: supervisorllm, - prompt: supervisorPrompt, - outputMode: "last_message", -}); -``` - -6. Update the supervisor's system prompt to summarize the new agent's capabilities. - -### Best Practices for Agent Development - -When developing new agents or capabilities: - -- **Clear Responsibility Boundaries**: Each agent should have a defined purpose with minimal overlap. No two agents should access the same tools or different tools accessing the same Prowler APIs. -- **Minimal Data Access**: Agents should only request the data they need, keeping requests specific to minimize context window usage, cost, and response time. -- **Thorough Prompting:** Ensure agent prompts include clear instructions about: - - The agent's purpose and limitations - - How to use its tools - - How to format responses for the supervisor - - Error handling procedures (Optional) -- **Security Considerations:** Agents should never modify data or access sensitive information like secrets or credentials. -- **Testing:** Thoroughly test new agents with various queries before deploying to production. diff --git a/docs/developer-guide/mcp-server.mdx b/docs/developer-guide/mcp-server.mdx new file mode 100644 index 0000000000..4174f0f730 --- /dev/null +++ b/docs/developer-guide/mcp-server.mdx @@ -0,0 +1,447 @@ +--- +title: 'Extending the MCP Server' +--- + +This guide explains how to extend the Prowler MCP Server with new tools and features. + + +**New to Prowler MCP Server?** Start with the user documentation: +- [Overview](/getting-started/products/prowler-mcp) - Key capabilities, use cases, and deployment options +- [Installation](/getting-started/installation/prowler-mcp) - Install locally or use the managed server +- [Configuration](/getting-started/basic-usage/prowler-mcp) - Configure Claude Desktop, Cursor, and other MCP hosts +- [Tools Reference](/getting-started/basic-usage/prowler-mcp-tools) - Complete list of all available tools + + +## Introduction + +The Prowler MCP Server brings the entire Prowler ecosystem to AI assistants through the [Model Context Protocol (MCP)](https://modelcontextprotocol.io). It enables seamless integration with AI tools like Claude Desktop, Cursor, and other MCP clients. + +The server follows a modular architecture with three independent sub-servers: + +| Sub-Server | Auth Required | Description | +|------------|---------------|-------------| +| Prowler App | Yes | Full access to Prowler Cloud and Self-Managed features | +| Prowler Hub | No | Security checks catalog with **over 1000 checks**, fixers, and **70+ compliance frameworks** | +| Prowler Documentation | No | Full-text search and retrieval of official documentation | + + +For a complete list of tools and their descriptions, see the [Tools Reference](/getting-started/basic-usage/prowler-mcp-tools). + + +## Architecture Overview + +The MCP Server architecture is illustrated in the [Overview documentation](/getting-started/products/prowler-mcp#mcp-server-architecture). AI assistants connect through the MCP protocol to access Prowler's three main components. + +### Server Structure + +The main server orchestrates three sub-servers with prefixed namespacing: + +``` +mcp_server/prowler_mcp_server/ +├── server.py # Main orchestrator +├── main.py # CLI entry point +├── prowler_hub/ +├── prowler_app/ +│ ├── tools/ # Tool implementations +│ ├── models/ # Pydantic models +│ └── utils/ # API client, auth, loader +└── prowler_documentation/ +``` + +### Tool Registration Patterns + +The MCP Server uses two patterns for tool registration: + +1. **Direct Decorators** (Prowler Hub/Docs): Tools are registered using `@mcp.tool()` decorators +2. **Auto-Discovery** (Prowler App): All public methods of `BaseTool` subclasses are auto-registered + +## Adding Tools to Prowler App + +### Step 1: Create the Tool Class + +Create a new file or add to an existing file in `prowler_app/tools/`: + +```python +# prowler_app/tools/new_feature.py +from typing import Any + +from pydantic import Field + +from prowler_mcp_server.prowler_app.models.new_feature import ( + FeatureListResponse, + DetailedFeature, +) +from prowler_mcp_server.prowler_app.tools.base import BaseTool + + +class NewFeatureTools(BaseTool): + """Tools for managing new features.""" + + async def list_features( + self, + status: str | None = Field( + default=None, + description="Filter by status (active, inactive, pending)" + ), + page_size: int = Field( + default=50, + description="Number of results per page (1-100)" + ), + ) -> dict[str, Any]: + """List all features with optional filtering. + + Returns a lightweight list of features optimized for LLM consumption. + Use get_feature for complete information about a specific feature. + """ + # Validate parameters + self.api_client.validate_page_size(page_size) + + # Build query parameters + params: dict[str, Any] = {"page[size]": page_size} + if status: + params["filter[status]"] = status + + # Make API request + clean_params = self.api_client.build_filter_params(params) + response = await self.api_client.get("/api/v1/features", params=clean_params) + + # Transform to LLM-friendly format + return FeatureListResponse.from_api_response(response).model_dump() + + async def get_feature( + self, + feature_id: str = Field(description="The UUID of the feature"), + ) -> dict[str, Any]: + """Get detailed information about a specific feature. + + Returns complete feature details including configuration and metadata. + """ + try: + response = await self.api_client.get(f"/api/v1/features/{feature_id}") + return DetailedFeature.from_api_response(response["data"]).model_dump() + except Exception as e: + self.logger.error(f"Failed to get feature {feature_id}: {e}") + return {"error": str(e), "status": "failed"} +``` + +### Step 2: Create the Models + +Create corresponding models in `prowler_app/models/`: + +```python +# prowler_app/models/new_feature.py +from typing import Any + +from pydantic import Field + +from prowler_mcp_server.prowler_app.models.base import MinimalSerializerMixin + + +class SimplifiedFeature(MinimalSerializerMixin): + """Lightweight feature for list operations.""" + + id: str = Field(description="Unique feature identifier") + name: str = Field(description="Feature name") + status: str = Field(description="Current status") + + @classmethod + def from_api_response(cls, data: dict[str, Any]) -> "SimplifiedFeature": + """Transform API response to simplified format.""" + attributes = data.get("attributes", {}) + return cls( + id=data["id"], + name=attributes["name"], + status=attributes["status"], + ) + + +class DetailedFeature(SimplifiedFeature): + """Extended feature with complete details.""" + + description: str | None = Field(default=None, description="Feature description") + configuration: dict[str, Any] | None = Field(default=None, description="Configuration") + created_at: str = Field(description="Creation timestamp") + updated_at: str = Field(description="Last update timestamp") + + @classmethod + def from_api_response(cls, data: dict[str, Any]) -> "DetailedFeature": + """Transform API response to detailed format.""" + attributes = data.get("attributes", {}) + return cls( + id=data["id"], + name=attributes["name"], + status=attributes["status"], + description=attributes.get("description"), + configuration=attributes.get("configuration"), + created_at=attributes["created_at"], + updated_at=attributes["updated_at"], + ) + + +class FeatureListResponse(MinimalSerializerMixin): + """Response wrapper for feature list operations.""" + + count: int = Field(description="Total number of features") + features: list[SimplifiedFeature] = Field(description="List of features") + + @classmethod + def from_api_response(cls, response: dict[str, Any]) -> "FeatureListResponse": + """Transform API response to list format.""" + data = response.get("data", []) + features = [SimplifiedFeature.from_api_response(item) for item in data] + return cls(count=len(features), features=features) +``` + +### Step 3: Verify Auto-Discovery + +No manual registration is needed. The `tool_loader.py` automatically discovers and registers all `BaseTool` subclasses. Verify your tool is loaded by checking the server logs: + +``` +INFO - Auto-registered 2 tools from NewFeatureTools +INFO - Loaded and registered: NewFeatureTools +``` + +## Adding Tools to Prowler Hub/Docs + +For Prowler Hub or Documentation tools, use the `@mcp.tool()` decorator directly: + +```python +# prowler_hub/server.py +from fastmcp import FastMCP + +hub_mcp_server = FastMCP("prowler-hub") + +@hub_mcp_server.tool() +async def get_new_artifact( + artifact_id: str, +) -> dict: + """Fetch a specific artifact from Prowler Hub. + + Args: + artifact_id: The unique identifier of the artifact + + Returns: + Dictionary containing artifact details + """ + response = prowler_hub_client.get(f"/artifact/{artifact_id}") + response.raise_for_status() + return response.json() +``` + +## Model Design Patterns + +### MinimalSerializerMixin + +All models should use `MinimalSerializerMixin` to optimize responses for LLM consumption: + +```python +from prowler_mcp_server.prowler_app.models.base import MinimalSerializerMixin + +class MyModel(MinimalSerializerMixin): + """Model that excludes empty values from serialization.""" + required_field: str + optional_field: str | None = None # Excluded if None + empty_list: list = [] # Excluded if empty +``` + +This mixin automatically excludes: +- `None` values +- Empty strings +- Empty lists +- Empty dictionaries + +### Two-Tier Model Pattern + +Use two-tier models for efficient responses: + +- **Simplified**: Lightweight models for list operations +- **Detailed**: Extended models for single-item retrieval + +```python +class SimplifiedItem(MinimalSerializerMixin): + """Use for list operations - minimal fields.""" + id: str + name: str + status: str + +class DetailedItem(SimplifiedItem): + """Use for get operations - extends simplified with details.""" + description: str | None = None + configuration: dict | None = None + created_at: str + updated_at: str +``` + +### Factory Method Pattern + +Always implement `from_api_response()` for API transformation: + +```python +@classmethod +def from_api_response(cls, data: dict[str, Any]) -> "MyModel": + """Transform API response to model. + + This method handles the JSON:API format used by Prowler API, + extracting attributes and relationships as needed. + """ + attributes = data.get("attributes", {}) + return cls( + id=data["id"], + name=attributes["name"], + # ... map other fields + ) +``` + +## API Client Usage + +The `ProwlerAPIClient` is a singleton that handles authentication and HTTP requests: + +```python +class MyTools(BaseTool): + async def my_tool(self) -> dict: + # GET request + response = await self.api_client.get("/api/v1/endpoint", params={"key": "value"}) + + # POST request + response = await self.api_client.post( + "/api/v1/endpoint", + json_data={"data": {"type": "items", "attributes": {...}}} + ) + + # PATCH request + response = await self.api_client.patch( + f"/api/v1/endpoint/{id}", + json_data={"data": {"attributes": {...}}} + ) + + # DELETE request + response = await self.api_client.delete(f"/api/v1/endpoint/{id}") +``` + +### Helper Methods + +The API client provides useful helper methods: + +```python +# Validate page size (1-1000) +self.api_client.validate_page_size(page_size) + +# Normalize date range with max days limit +date_range = self.api_client.normalize_date_range(date_from, date_to, max_days=2) + +# Build filter parameters (handles type conversion) +clean_params = self.api_client.build_filter_params({ + "filter[status]": "active", + "filter[severity__in]": ["high", "critical"], # Converts to comma-separated + "filter[muted]": True, # Converts to "true" +}) + +# Poll async task until completion +result = await self.api_client.poll_task_until_complete( + task_id=task_id, + timeout=60, + poll_interval=1.0 +) +``` + +## Best Practices + +### Tool Docstrings + +Tool docstrings become description that is going to be read by the LLM. Provide clear usage instructions and common workflows: + +```python +async def search_items(self, status: str = Field(...)) -> dict: + """Search items with advanced filtering. + + Returns a lightweight list optimized for LLM consumption. + Use get_item for complete details about a specific item. + + Common workflows: + - Find critical items: status="critical" + - Find recent items: Use date_from parameter + """ +``` + +### Error Handling + +Return structured error responses instead of raising exceptions: + +```python +async def get_item(self, item_id: str) -> dict: + try: + response = await self.api_client.get(f"/api/v1/items/{item_id}") + return DetailedItem.from_api_response(response["data"]).model_dump() + except Exception as e: + self.logger.error(f"Failed to get item {item_id}: {e}") + return {"error": str(e), "status": "failed"} +``` + +### Parameter Descriptions + +Use Pydantic `Field()` with clear descriptions. This also helps LLMs understand +the purpose of each parameter, so be as descriptive as possible: + +```python +async def list_items( + self, + severity: list[str] = Field( + default=[], + description="Filter by severity levels (critical, high, medium, low)" + ), + status: str | None = Field( + default=None, + description="Filter by status (PASS, FAIL, MANUAL)" + ), + page_size: int = Field( + default=50, + description="Results per page" + ), +) -> dict: +``` + +## Development Commands + +```bash +# Navigate to MCP server directory +cd mcp_server + +# Run in STDIO mode (default) +uv run prowler-mcp + +# Run in HTTP mode +uv run prowler-mcp --transport http --host 0.0.0.0 --port 8000 + +# Run with environment variables +PROWLER_APP_API_KEY="pk_xxx" uv run prowler-mcp +``` + +For complete installation and deployment options, see: +- [Installation Guide](/getting-started/installation/prowler-mcp#from-source-development) - Development setup instructions +- [Configuration Guide](/getting-started/basic-usage/prowler-mcp) - MCP client configuration + +For development I recommend to use the [Model Context Protocol Inspector](https://github.com/modelcontextprotocol/inspector) as MCP client to test and debug your tools. + +## Related Documentation + + + + Key capabilities, use cases, and deployment options + + + Complete reference of all available tools + + + Security checks and compliance frameworks catalog + + + AI-powered security analyst + + + +## Additional Resources + +- [MCP Protocol Specification](https://modelcontextprotocol.io) - Model Context Protocol details +- [Prowler API Documentation](https://api.prowler.com/api/v1/docs) - API reference +- [Prowler Hub API](https://hub.prowler.com/api/docs) - Hub API reference +- [GitHub Repository](https://github.com/prowler-cloud/prowler) - Source code diff --git a/docs/developer-guide/provider.mdx b/docs/developer-guide/provider.mdx index 85dafc6300..867dff14d8 100644 --- a/docs/developer-guide/provider.mdx +++ b/docs/developer-guide/provider.mdx @@ -220,6 +220,7 @@ The function returns a JSON file containing the list of regions for the provider "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2" ], "aws-cn": ["cn-north-1", "cn-northwest-1"], + "aws-eusc": ["eusc-de-east-1"], "aws-us-gov": ["us-gov-east-1", "us-gov-west-1"] } } diff --git a/docs/developer-guide/unit-testing.mdx b/docs/developer-guide/unit-testing.mdx index 8713988440..b08b98f33c 100644 --- a/docs/developer-guide/unit-testing.mdx +++ b/docs/developer-guide/unit-testing.mdx @@ -63,6 +63,82 @@ Other Commands for Running Tests Refer to the [pytest documentation](https://docs.pytest.org/en/7.1.x/getting-started.html) for more details. + +## AWS Service Dependency Table (CI Optimization) + +To optimize CI pipeline execution time, the GitHub Actions workflow for AWS tests uses a **service dependency table** that determines which tests to run based on changed files. This ensures that when a service is modified, all dependent services are also tested. + +### How It Works + +The dependency table is defined in `.github/workflows/sdk-tests.yml` within the "Resolve AWS services under test" step. When files in a specific AWS service are changed: + +1. Tests for the changed service are run +2. Tests for all services that **depend on** the changed service are also run + +For example, if you modify the `ec2` service, tests will also run for `dlm`, `dms`, `elbv2`, `emr`, `inspector2`, `rds`, `redshift`, `route53`, `shield`, `ssm`, and `workspaces` because these services use the EC2 client. + +### Current Dependency Table + +The table maps a service (key) to the list of services that depend on it (values): + +| Service | Dependent Services | +|---------|-------------------| +| `acm` | `elb` | +| `autoscaling` | `dynamodb` | +| `awslambda` | `ec2`, `inspector2` | +| `backup` | `dynamodb`, `ec2`, `rds` | +| `cloudfront` | `shield` | +| `cloudtrail` | `awslambda`, `cloudwatch` | +| `cloudwatch` | `bedrock` | +| `ec2` | `dlm`, `dms`, `elbv2`, `emr`, `inspector2`, `rds`, `redshift`, `route53`, `shield`, `ssm` | +| `ecr` | `inspector2` | +| `elb` | `shield` | +| `elbv2` | `shield` | +| `globalaccelerator` | `shield` | +| `iam` | `bedrock`, `cloudtrail`, `cloudwatch`, `codebuild` | +| `kafka` | `firehose` | +| `kinesis` | `firehose` | +| `kms` | `kafka` | +| `organizations` | `iam`, `servicecatalog` | +| `route53` | `shield` | +| `s3` | `bedrock`, `cloudfront`, `cloudtrail`, `macie` | +| `ssm` | `ec2` | +| `vpc` | `awslambda`, `ec2`, `efs`, `elasticache`, `neptune`, `networkfirewall`, `rds`, `redshift`, `workspaces` | +| `waf` | `elbv2` | +| `wafv2` | `cognito`, `elbv2` | + +### When to Update the Table + +You must update the dependency table when: + +1. **A new check or service uses another service's client**: If your check imports a client from another service (e.g., `from prowler.providers.aws.services.ec2.ec2_client import ec2_client` in a non-ec2 check), add your service to the dependent services list of that client's service. + +2. **A service relationship changes**: If you remove or add a service client dependency in an existing check, update the table accordingly. + +### How to Update the Table + +1. Open `.github/workflows/sdk-tests.yml` +2. Find the `dependents` dictionary in the "Resolve AWS services under test" step +3. Add or modify entries as needed +4. **Update this documentation page** (`docs/developer-guide/unit-testing.mdx`) to reflect the changes in the [Current Dependency Table](#current-dependency-table) section above + +```python +dependents = { + # ... existing entries ... + "service_being_used": ["service_that_uses_it"], +} +``` + +**Example**: If you create a new check in the `newservice` service that imports `ec2_client`, add `newservice` to the `ec2` entry: + +```python +"ec2": ["dlm", "dms", "elbv2", "emr", "inspector2", "newservice", "rds", "redshift", "route53", "shield", "ssm"], +``` + + +Failing to update this table when adding cross-service dependencies may result in CI tests passing even when related functionality is broken, as the dependent service tests won't be triggered. + + ## AWS Testing Approaches For AWS provider, different testing approaches apply based on API coverage based on several criteria. diff --git a/docs/docs.json b/docs/docs.json index 51b3960333..1aa7c96fa6 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -99,7 +99,14 @@ }, "user-guide/tutorials/prowler-app-rbac", "user-guide/tutorials/prowler-app-api-keys", - "user-guide/tutorials/prowler-app-mute-findings", + { + "group": "Mutelist", + "expanded": true, + "pages": [ + "user-guide/tutorials/prowler-app-simple-mutelist", + "user-guide/tutorials/prowler-app-mute-findings" + ] + }, { "group": "Integrations", "expanded": true, @@ -198,6 +205,13 @@ "user-guide/providers/gcp/retry-configuration" ] }, + { + "group": "Alibaba Cloud", + "pages": [ + "user-guide/providers/alibabacloud/getting-started-alibabacloud", + "user-guide/providers/alibabacloud/authentication" + ] + }, { "group": "Kubernetes", "pages": [ @@ -234,6 +248,13 @@ "user-guide/providers/mongodbatlas/authentication" ] }, + { + "group": "Cloudflare", + "pages": [ + "user-guide/providers/cloudflare/getting-started-cloudflare", + "user-guide/providers/cloudflare/authentication" + ] + }, { "group": "LLM", "pages": [ @@ -270,7 +291,9 @@ "developer-guide/outputs", "developer-guide/integrations", "developer-guide/security-compliance-framework", - "developer-guide/lighthouse" + "developer-guide/lighthouse-architecture", + "developer-guide/mcp-server", + "developer-guide/ai-skills" ] }, { @@ -279,6 +302,7 @@ "developer-guide/aws-details", "developer-guide/azure-details", "developer-guide/gcp-details", + "developer-guide/alibabacloud-details", "developer-guide/kubernetes-details", "developer-guide/m365-details", "developer-guide/github-details", @@ -293,7 +317,8 @@ "group": "Testing", "pages": [ "developer-guide/unit-testing", - "developer-guide/integration-testing" + "developer-guide/integration-testing", + "developer-guide/end2end-testing" ] }, "developer-guide/debugging", diff --git a/docs/getting-started/basic-usage/prowler-mcp-tools.mdx b/docs/getting-started/basic-usage/prowler-mcp-tools.mdx index dc984c9537..bf0092a069 100644 --- a/docs/getting-started/basic-usage/prowler-mcp-tools.mdx +++ b/docs/getting-started/basic-usage/prowler-mcp-tools.mdx @@ -10,7 +10,7 @@ Complete reference guide for all tools available in the Prowler MCP Server. Tool |----------|------------|------------------------| | Prowler Hub | 10 tools | No | | Prowler Documentation | 2 tools | No | -| Prowler Cloud/App | 28 tools | Yes | +| Prowler Cloud/App | 24 tools | Yes | ## Tool Naming Convention @@ -20,39 +20,6 @@ All tools follow a consistent naming pattern with prefixes: - `prowler_docs_*` - Prowler documentation search and retrieval - `prowler_app_*` - Prowler Cloud and App (Self-Managed) management tools -## Prowler Hub Tools - -Access Prowler's security check catalog and compliance frameworks. **No authentication required.** - -### Check Discovery - -- **`prowler_hub_get_checks`** - List security checks with advanced filtering options -- **`prowler_hub_get_check_filters`** - Return available filter values for checks (providers, services, severities, categories, compliances) -- **`prowler_hub_search_checks`** - Full-text search across check metadata -- **`prowler_hub_get_check_raw_metadata`** - Fetch raw check metadata in JSON format - -### Check Code - -- **`prowler_hub_get_check_code`** - Fetch the Python implementation code for a security check -- **`prowler_hub_get_check_fixer`** - Fetch the automated fixer code for a check (if available) - -### Compliance Frameworks - -- **`prowler_hub_get_compliance_frameworks`** - List and filter compliance frameworks -- **`prowler_hub_search_compliance_frameworks`** - Full-text search across compliance frameworks - -### Provider Information - -- **`prowler_hub_list_providers`** - List Prowler official providers and their services -- **`prowler_hub_get_artifacts_count`** - Get total count of checks and frameworks in Prowler Hub - -## Prowler Documentation Tools - -Search and access official Prowler documentation. **No authentication required.** - -- **`prowler_docs_search`** - Search the official Prowler documentation using full-text search -- **`prowler_docs_get_document`** - Retrieve the full markdown content of a specific documentation file - ## Prowler Cloud/App Tools Manage Prowler Cloud or Prowler App (Self-Managed) features. **Requires authentication.** @@ -63,49 +30,97 @@ These tools require a valid API key. See the [Configuration Guide](/getting-star ### Findings Management -- **`prowler_app_list_findings`** - List security findings with advanced filtering -- **`prowler_app_get_finding`** - Get detailed information about a specific finding -- **`prowler_app_get_latest_findings`** - Retrieve latest findings from the most recent scans -- **`prowler_app_get_findings_metadata`** - Get unique metadata values from filtered findings -- **`prowler_app_get_latest_findings_metadata`** - Get metadata from latest findings across all providers +Tools for searching, viewing, and analyzing security findings across all cloud providers. + +- **`prowler_app_search_security_findings`** - Search and filter security findings with advanced filtering options (severity, status, provider, region, service, check ID, date range, muted status) +- **`prowler_app_get_finding_details`** - Get comprehensive details about a specific finding including remediation guidance, check metadata, and resource relationships +- **`prowler_app_get_findings_overview`** - Get aggregate statistics and trends about security findings as a markdown report ### Provider Management -- **`prowler_app_list_providers`** - List all providers with filtering options -- **`prowler_app_create_provider`** - Create a new provider in the current tenant -- **`prowler_app_get_provider`** - Get detailed information about a specific provider -- **`prowler_app_update_provider`** - Update provider details (alias, etc.) -- **`prowler_app_delete_provider`** - Delete a specific provider -- **`prowler_app_test_provider_connection`** - Test provider connection status +Tools for managing cloud provider connections in Prowler. -### Provider Secrets Management - -- **`prowler_app_list_provider_secrets`** - List all provider secrets with filtering -- **`prowler_app_add_provider_secret`** - Add or update credentials for a provider -- **`prowler_app_get_provider_secret`** - Get detailed information about a provider secret -- **`prowler_app_update_provider_secret`** - Update provider secret details -- **`prowler_app_delete_provider_secret`** - Delete a provider secret +- **`prowler_app_search_providers`** - Search and view configured providers with their connection status +- **`prowler_app_connect_provider`** - Register and connect a provider with credentials for security scanning +- **`prowler_app_delete_provider`** - Permanently remove a provider from Prowler ### Scan Management -- **`prowler_app_list_scans`** - List all scans with filtering options -- **`prowler_app_create_scan`** - Trigger a manual scan for a specific provider -- **`prowler_app_get_scan`** - Get detailed information about a specific scan -- **`prowler_app_update_scan`** - Update scan details -- **`prowler_app_get_scan_compliance_report`** - Download compliance report as CSV -- **`prowler_app_get_scan_report`** - Download ZIP file containing complete scan report +Tools for managing and monitoring security scans. -### Schedule Management +- **`prowler_app_list_scans`** - List and filter security scans across all providers +- **`prowler_app_get_scan`** - Get comprehensive details about a specific scan (progress, duration, resource counts) +- **`prowler_app_trigger_scan`** - Trigger a manual security scan for a provider +- **`prowler_app_schedule_daily_scan`** - Schedule automated daily scans for continuous monitoring +- **`prowler_app_update_scan`** - Update scan name for better organization -- **`prowler_app_schedules_daily_scan`** - Create a daily scheduled scan for a provider +### Resources Management -### Processor Management +Tools for searching, viewing, and analyzing cloud resources discovered by Prowler. -- **`prowler_app_processors_list`** - List all processors with filtering -- **`prowler_app_processors_create`** - Create a new processor (currently only mute lists supported) -- **`prowler_app_processors_retrieve`** - Get processor details by ID -- **`prowler_app_processors_partial_update`** - Update processor configuration -- **`prowler_app_processors_destroy`** - Delete a processor +- **`prowler_app_list_resources`** - List and filter cloud resources with advanced filtering options (provider, region, service, resource type, tags) +- **`prowler_app_get_resource`** - Get comprehensive details about a specific resource including configuration, metadata, and finding relationships +- **`prowler_app_get_resources_overview`** - Get aggregate statistics about cloud resources as a markdown report + +### Muting Management + +Tools for managing finding muting, including pattern-based bulk muting (mutelist) and finding-specific mute rules. + +#### Mutelist (Pattern-Based Muting) + +- **`prowler_app_get_mutelist`** - Retrieve the current mutelist configuration for the tenant +- **`prowler_app_set_mutelist`** - Create or update the mutelist configuration for pattern-based bulk muting +- **`prowler_app_delete_mutelist`** - Remove the mutelist configuration from the tenant + +#### Mute Rules (Finding-Specific Muting) + +- **`prowler_app_list_mute_rules`** - Search and filter mute rules with pagination support +- **`prowler_app_get_mute_rule`** - Retrieve comprehensive details about a specific mute rule +- **`prowler_app_create_mute_rule`** - Create a new mute rule to mute specific findings with documentation and audit trail +- **`prowler_app_update_mute_rule`** - Update a mute rule's name, reason, or enabled status +- **`prowler_app_delete_mute_rule`** - Delete a mute rule from the system + +### Compliance Management + +Tools for viewing compliance status and framework details across all cloud providers. + +- **`prowler_app_get_compliance_overview`** - Get high-level compliance status across all frameworks for a specific scan or provider, including pass/fail statistics per framework +- **`prowler_app_get_compliance_framework_state_details`** - Get detailed requirement-level breakdown for a specific compliance framework, including failed requirements and associated finding IDs + +## Prowler Hub Tools + +Access Prowler's security check catalog and compliance frameworks. **No authentication required.** + +Tools follow a **two-tier pattern**: lightweight listing for browsing + detailed retrieval for complete information. + +### Check Discovery and Details + +- **`prowler_hub_list_checks`** - List security checks with lightweight data (id, title, severity, provider) and advanced filtering options +- **`prowler_hub_semantic_search_checks`** - Full-text search across check metadata with lightweight results +- **`prowler_hub_get_check_details`** - Get comprehensive details for a specific check including risk, remediation guidance, and compliance mappings + +### Check Code + +- **`prowler_hub_get_check_code`** - Fetch the Python implementation code for a security check +- **`prowler_hub_get_check_fixer`** - Fetch the automated fixer code for a check (if available) + +### Compliance Frameworks + +- **`prowler_hub_list_compliances`** - List compliance frameworks with lightweight data (id, name, provider) and filtering options +- **`prowler_hub_semantic_search_compliances`** - Full-text search across compliance frameworks with lightweight results +- **`prowler_hub_get_compliance_details`** - Get comprehensive compliance details including requirements and mapped checks + +### Providers Information + +- **`prowler_hub_list_providers`** - List Prowler official providers +- **`prowler_hub_get_provider_services`** - Get available services for a specific provider + +## Prowler Documentation Tools + +Search and access official Prowler documentation. **No authentication required.** + +- **`prowler_docs_search`** - Search the official Prowler documentation using full-text search with the `term` parameter +- **`prowler_docs_get_document`** - Retrieve the full markdown content of a specific documentation file using the path from search results ## Usage Tips diff --git a/docs/getting-started/basic-usage/prowler-mcp.mdx b/docs/getting-started/basic-usage/prowler-mcp.mdx index b6d59093e7..a9357dcdeb 100644 --- a/docs/getting-started/basic-usage/prowler-mcp.mdx +++ b/docs/getting-started/basic-usage/prowler-mcp.mdx @@ -139,7 +139,7 @@ STDIO mode is only available when running the MCP server locally. "args": ["/absolute/path/to/prowler/mcp_server/"], "env": { "PROWLER_APP_API_KEY": "", - "PROWLER_API_BASE_URL": "https://api.prowler.com" + "API_BASE_URL": "https://api.prowler.com/api/v1" } } } @@ -147,7 +147,7 @@ STDIO mode is only available when running the MCP server locally. ``` - Replace `/absolute/path/to/prowler/mcp_server/` with the actual path. The `PROWLER_API_BASE_URL` is optional and defaults to Prowler Cloud API. + Replace `/absolute/path/to/prowler/mcp_server/` with the actual path. The `API_BASE_URL` is optional and defaults to Prowler Cloud API. @@ -167,7 +167,7 @@ STDIO mode is only available when running the MCP server locally. "--env", "PROWLER_APP_API_KEY=", "--env", - "PROWLER_API_BASE_URL=https://api.prowler.com", + "API_BASE_URL=https://api.prowler.com/api/v1", "prowlercloud/prowler-mcp" ] } @@ -176,7 +176,7 @@ STDIO mode is only available when running the MCP server locally. ``` - The `PROWLER_API_BASE_URL` is optional and defaults to Prowler Cloud API. + The `API_BASE_URL` is optional and defaults to Prowler Cloud API. diff --git a/docs/getting-started/installation/prowler-app.mdx b/docs/getting-started/installation/prowler-app.mdx index 8cce9b0c50..cd3e18b3ea 100644 --- a/docs/getting-started/installation/prowler-app.mdx +++ b/docs/getting-started/installation/prowler-app.mdx @@ -115,10 +115,15 @@ To update the environment file: Edit the `.env` file and change version values: ```env -PROWLER_UI_VERSION="5.9.0" -PROWLER_API_VERSION="5.9.0" +PROWLER_UI_VERSION="5.16.0" +PROWLER_API_VERSION="5.16.0" ``` + + You can find the latest versions of Prowler App in the [Releases Github section](https://github.com/prowler-cloud/prowler/releases) or in the [Container Versions](#container-versions) section of this documentation. + + + #### Option 2: Using Docker Compose Pull ```bash diff --git a/docs/getting-started/installation/prowler-mcp.mdx b/docs/getting-started/installation/prowler-mcp.mdx index 8c3fd2f955..7bf766e143 100644 --- a/docs/getting-started/installation/prowler-mcp.mdx +++ b/docs/getting-started/installation/prowler-mcp.mdx @@ -52,7 +52,7 @@ Choose one of the following installation methods: ```bash docker run --rm -i \ -e PROWLER_APP_API_KEY="pk_your_api_key" \ - -e PROWLER_API_BASE_URL="https://api.prowler.com" \ + -e API_BASE_URL="https://api.prowler.com/api/v1" \ prowlercloud/prowler-mcp ``` @@ -181,19 +181,19 @@ Configure the server using environment variables: | Variable | Description | Required | Default | |----------|-------------|----------|---------| | `PROWLER_APP_API_KEY` | Prowler API key | Only for STDIO mode | - | -| `PROWLER_API_BASE_URL` | Custom Prowler API endpoint | No | `https://api.prowler.com` | +| `API_BASE_URL` | Custom Prowler API endpoint | No | `https://api.prowler.com/api/v1` | | `PROWLER_MCP_TRANSPORT_MODE` | Default transport mode (overwritten by `--transport` argument) | No | `stdio` | ```bash macOS/Linux export PROWLER_APP_API_KEY="pk_your_api_key_here" -export PROWLER_API_BASE_URL="https://api.prowler.com" +export API_BASE_URL="https://api.prowler.com/api/v1" export PROWLER_MCP_TRANSPORT_MODE="http" ``` ```bash Windows PowerShell $env:PROWLER_APP_API_KEY="pk_your_api_key_here" -$env:PROWLER_API_BASE_URL="https://api.prowler.com" +$env:API_BASE_URL="https://api.prowler.com/api/v1" $env:PROWLER_MCP_TRANSPORT_MODE="http" ``` @@ -208,7 +208,7 @@ For convenience, create a `.env` file in the `mcp_server` directory: ```bash .env PROWLER_APP_API_KEY=pk_your_api_key_here -PROWLER_API_BASE_URL=https://api.prowler.com +API_BASE_URL=https://api.prowler.com/api/v1 PROWLER_MCP_TRANSPORT_MODE=stdio ``` diff --git a/docs/getting-started/products/prowler-hub.mdx b/docs/getting-started/products/prowler-hub.mdx index a8dde7549f..b84d48b7d5 100644 --- a/docs/getting-started/products/prowler-hub.mdx +++ b/docs/getting-started/products/prowler-hub.mdx @@ -6,7 +6,7 @@ title: "Overview" **Why this matters**: Every engineer has asked, “What does this check actually do?” Prowler Hub answers that question in one place, lets you pin to a specific version, and pulls definitions into your own tools or dashboards. -![](/images/products/prowler-hub.webp) +![](/images/products/prowler-hub.png) @@ -14,4 +14,4 @@ Prowler Hub also provides a fully documented public API that you can integrate i 📚 Explore the API docs at: https://hub.prowler.com/api/docs -Whether you’re customizing policies, managing compliance, or enhancing visibility, Prowler Hub is built to support your security operations. \ No newline at end of file +Whether you’re customizing policies, managing compliance, or enhancing visibility, Prowler Hub is built to support your security operations. diff --git a/docs/getting-started/products/prowler-lighthouse-ai.mdx b/docs/getting-started/products/prowler-lighthouse-ai.mdx index 629699e41a..6f197ae546 100644 --- a/docs/getting-started/products/prowler-lighthouse-ai.mdx +++ b/docs/getting-started/products/prowler-lighthouse-ai.mdx @@ -59,6 +59,14 @@ Prowler Lighthouse AI is powerful, but there are limitations: - **NextJS session dependence**: If your Prowler application session expires or logs out, Lighthouse AI will error out. Refresh and log back in to continue. - **Response quality**: The response quality depends on the selected LLM provider and model. Choose models with strong tool-calling capabilities for best results. We recommend `gpt-5` model from OpenAI. +## Extending Lighthouse AI + +Lighthouse AI retrieves data through Prowler MCP. To add new capabilities, extend the Prowler MCP Server with additional tools and Lighthouse AI discovers them automatically. + +For development details, see: +- [Lighthouse AI Architecture](/developer-guide/lighthouse-architecture) - Internal architecture and extension points +- [Extending the MCP Server](/developer-guide/mcp-server) - Adding new tools to Prowler MCP + ### Getting Help If you encounter issues with Prowler Lighthouse AI or have suggestions for improvements, please [reach out through our Slack channel](https://goto.prowler.com/slack). @@ -67,94 +75,6 @@ If you encounter issues with Prowler Lighthouse AI or have suggestions for impro The following API endpoints are accessible to Prowler Lighthouse AI. Data from the following API endpoints could be shared with LLM provider depending on the scope of user's query: -#### Accessible API Endpoints - -**User Management:** - -- List all users - `/api/v1/users` -- Retrieve the current user's information - `/api/v1/users/me` - -**Provider Management:** - -- List all providers - `/api/v1/providers` -- Retrieve data from a provider - `/api/v1/providers/{id}` - -**Scan Management:** - -- List all scans - `/api/v1/scans` -- Retrieve data from a specific scan - `/api/v1/scans/{id}` - -**Resource Management:** - -- List all resources - `/api/v1/resources` -- Retrieve data for a resource - `/api/v1/resources/{id}` - -**Findings Management:** - -- List all findings - `/api/v1/findings` -- Retrieve data from a specific finding - `/api/v1/findings/{id}` -- Retrieve metadata values from findings - `/api/v1/findings/metadata` - -**Overview Data:** - -- Get aggregated findings data - `/api/v1/overviews/findings` -- Get findings data by severity - `/api/v1/overviews/findings_severity` -- Get aggregated provider data - `/api/v1/overviews/providers` -- Get findings data by service - `/api/v1/overviews/services` - -**Compliance Management:** - -- List compliance overviews (optionally filter by scan) - `/api/v1/compliance-overviews` -- Retrieve data from a specific compliance overview - `/api/v1/compliance-overviews/{id}` - -#### Excluded API Endpoints - -Not all Prowler API endpoints are integrated with Lighthouse AI. They are intentionally excluded for the following reasons: - -- OpenAI/other LLM providers shouldn't have access to sensitive data (like fetching provider secrets and other sensitive config) -- Users queries don't need responses from those API endpoints (ex: tasks, tenant details, downloading zip file, etc.) - -**Excluded Endpoints:** - -**User Management:** - -- List specific users information - `/api/v1/users/{id}` -- List user memberships - `/api/v1/users/{user_pk}/memberships` -- Retrieve membership data from the user - `/api/v1/users/{user_pk}/memberships/{id}` - -**Tenant Management:** - -- List all tenants - `/api/v1/tenants` -- Retrieve data from a tenant - `/api/v1/tenants/{id}` -- List tenant memberships - `/api/v1/tenants/{tenant_pk}/memberships` -- List all invitations - `/api/v1/tenants/invitations` -- Retrieve data from tenant invitation - `/api/v1/tenants/invitations/{id}` - -**Security and Configuration:** - -- List all secrets - `/api/v1/providers/secrets` -- Retrieve data from a secret - `/api/v1/providers/secrets/{id}` -- List all provider groups - `/api/v1/provider-groups` -- Retrieve data from a provider group - `/api/v1/provider-groups/{id}` - -**Reports and Tasks:** - -- Download zip report - `/api/v1/scans/{v1}/report` -- List all tasks - `/api/v1/tasks` -- Retrieve data from a specific task - `/api/v1/tasks/{id}` - -**Lighthouse AI Configuration:** - -- List LLM providers - `/api/v1/lighthouse/providers` -- Retrieve LLM provider - `/api/v1/lighthouse/providers/{id}` -- List available models - `/api/v1/lighthouse/models` -- Retrieve tenant configuration - `/api/v1/lighthouse/configuration` - - -Agents only have access to hit GET endpoints. They don't have access to other HTTP methods. - - - ## FAQs **1. Which LLM providers are supported?** @@ -167,13 +87,21 @@ Lighthouse AI supports three providers: For detailed configuration instructions, see [Using Multiple LLM Providers with Lighthouse](/user-guide/tutorials/prowler-app-lighthouse-multi-llm). -**2. Why a multi-agent supervisor model?** +**2. Why some models don't appear in Lighthouse AI?** -Context windows are limited. While demo data fits inside the context window, querying real-world data often exceeds it. A multi-agent architecture is used so different agents fetch different sizes of data and respond with the minimum required data to the supervisor. This spreads the context window usage across agents. +LLM providers offer different types of models. Not every model can be integrated with Lighthouse AI (for example, text-to-speech, vision, embedding, computer use, etc.). + +Lighthouse AI requires models that support: + +- Text input +- Text output +- Tool calling + +Lighthouse AI [automatically filters](https://github.com/prowler-cloud/prowler/blob/master/api/src/backend/tasks/jobs/lighthouse_providers.py#L341-L353) out models that do not support these capabilities, so some provider models may not appear in the Lighthouse AI model list. **3. Is my security data shared with LLM providers?** -Minimal data is shared to generate useful responses. Agents can access security findings and remediation details when needed. Provider secrets are protected by design and cannot be read. The LLM provider credentials configured with Lighthouse AI are only accessible to our NextJS server and are never sent to the LLM providers. Resource metadata (names, tags, account/project IDs, etc) may be shared with the configured LLM provider based on query requirements. +Minimal data is shared to generate useful responses. Agent can access security findings and remediation details when needed. Provider secrets are protected by design and cannot be read. The LLM provider credentials configured with Lighthouse AI are only accessible to the Next.js server and are never sent to the LLM providers. Resource metadata (names, tags, account/project IDs, etc.) may be shared with the configured LLM provider based on query requirements. **4. Can the Lighthouse AI change my cloud environment?** diff --git a/docs/getting-started/products/prowler-mcp.mdx b/docs/getting-started/products/prowler-mcp.mdx index 94a5466edb..df402f605b 100644 --- a/docs/getting-started/products/prowler-mcp.mdx +++ b/docs/getting-started/products/prowler-mcp.mdx @@ -19,12 +19,11 @@ The Prowler MCP Server provides three main integration points: ### 1. Prowler Cloud and Prowler App (Self-Managed) Full access to Prowler Cloud platform and self-managed Prowler App for: -- **Provider Management**: Create, configure, and manage cloud providers (AWS, Azure, GCP, etc.). -- **Scan Orchestration**: Trigger on-demand scans and schedule recurring security assessments. -- **Findings Analysis**: Query, filter, and analyze security findings across all your cloud environments. -- **Compliance Reporting**: Generate compliance reports for various frameworks (CIS, PCI-DSS, HIPAA, etc.). -- **Secrets Management**: Securely manage provider credentials and connection details. -- **Processor Configuration**: Set up the [Prowler Mutelist](/user-guide/tutorials/prowler-app-mute-findings) to mute findings. +- **Findings Analysis**: Query, filter, and analyze security findings across all your cloud environments +- **Provider Management**: Create, configure, and manage your configured Prowler providers (AWS, Azure, GCP, etc.) +- **Scan Orchestration**: Trigger on-demand scans and schedule recurring security assessments +- **Resource Inventory**: Search and view detailed information about your audited resources +- **Muting Management**: Create and manage muting lists/rules to suppress non-relevant findings ### 2. Prowler Hub @@ -49,7 +48,10 @@ The following diagram illustrates the Prowler MCP Server architecture and its in Prowler MCP Server Schema Prowler MCP Server Schema -The architecture shows how AI assistants connect through the MCP protocol to access Prowler's three main components: Prowler Cloud/App for security operations, Prowler Hub for security knowledge, and Prowler Documentation for guidance and reference. +The architecture shows how AI assistants connect through the MCP protocol to access Prowler's three main components: +- Prowler Cloud/App for security operations +- Prowler Hub for security knowledge +- Prowler Documentation for guidance and reference. ## Use Cases @@ -57,12 +59,12 @@ The Prowler MCP Server enables powerful workflows through AI assistants: **Security Operations** - "Show me all critical findings from my AWS production accounts" -- "What is my compliance status for the PCI standards accross all my AWS accounts according to the latest Prowler scan results?" -- "Register my new AWS account in Prowler and run an scheduled scan every day" +- "Register my new AWS account in Prowler and run a scheduled scan every day" +- "List all muted findings and detect what findgings are muted by a not enough good reason in relation to their severity" **Security Research** -- "Explain what the S3 bucket public access check does" -- "Find all checks related to encryption at rest" +- "Explain what the S3 bucket public access Prowler check does" +- "Find all Prowler checks related to encryption at rest" - "What is the latest version of the CIS that Prowler is covering per provider?" **Documentation & Learning** diff --git a/docs/images/cli/add-cloud-provider.png b/docs/images/cli/add-cloud-provider.png index dd42e73047..d8f19b2054 100644 Binary files a/docs/images/cli/add-cloud-provider.png and b/docs/images/cli/add-cloud-provider.png differ diff --git a/docs/images/cli/cloud-providers-page.png b/docs/images/cli/cloud-providers-page.png index 0581e0132f..dcbce73a10 100644 Binary files a/docs/images/cli/cloud-providers-page.png and b/docs/images/cli/cloud-providers-page.png differ diff --git a/docs/images/cli/lighthouse-architecture.png b/docs/images/cli/lighthouse-architecture.png deleted file mode 100644 index 63202ce7c7..0000000000 Binary files a/docs/images/cli/lighthouse-architecture.png and /dev/null differ diff --git a/docs/images/lighthouse-architecture-dark.png b/docs/images/lighthouse-architecture-dark.png new file mode 100644 index 0000000000..0ed77712c5 Binary files /dev/null and b/docs/images/lighthouse-architecture-dark.png differ diff --git a/docs/images/lighthouse-architecture-light.png b/docs/images/lighthouse-architecture-light.png new file mode 100644 index 0000000000..076240ccb2 Binary files /dev/null and b/docs/images/lighthouse-architecture-light.png differ diff --git a/docs/images/products/overview.png b/docs/images/products/overview.png index 724ffc9588..697e1f74dc 100644 Binary files a/docs/images/products/overview.png and b/docs/images/products/overview.png differ diff --git a/docs/images/products/prowler-hub.png b/docs/images/products/prowler-hub.png new file mode 100644 index 0000000000..149d0142ca Binary files /dev/null and b/docs/images/products/prowler-hub.png differ diff --git a/docs/images/products/prowler-hub.webp b/docs/images/products/prowler-hub.webp deleted file mode 100644 index b489ff9012..0000000000 Binary files a/docs/images/products/prowler-hub.webp and /dev/null differ diff --git a/docs/images/products/risk-pipeline.png b/docs/images/products/risk-pipeline.png new file mode 100644 index 0000000000..43dae07470 Binary files /dev/null and b/docs/images/products/risk-pipeline.png differ diff --git a/docs/images/products/threat-map.png b/docs/images/products/threat-map.png new file mode 100644 index 0000000000..7a4a276473 Binary files /dev/null and b/docs/images/products/threat-map.png differ diff --git a/docs/images/providers/add-alibaba-account-id.png b/docs/images/providers/add-alibaba-account-id.png new file mode 100644 index 0000000000..2e49f995cc Binary files /dev/null and b/docs/images/providers/add-alibaba-account-id.png differ diff --git a/docs/images/providers/alibaba-account-id.png b/docs/images/providers/alibaba-account-id.png new file mode 100644 index 0000000000..89e79b1f20 Binary files /dev/null and b/docs/images/providers/alibaba-account-id.png differ diff --git a/docs/images/providers/alibaba-connect-via-credentials-static.png b/docs/images/providers/alibaba-connect-via-credentials-static.png new file mode 100644 index 0000000000..f9b9a00ee6 Binary files /dev/null and b/docs/images/providers/alibaba-connect-via-credentials-static.png differ diff --git a/docs/images/providers/alibaba-connect-via-credentials.png b/docs/images/providers/alibaba-connect-via-credentials.png new file mode 100644 index 0000000000..f9b9a00ee6 Binary files /dev/null and b/docs/images/providers/alibaba-connect-via-credentials.png differ diff --git a/docs/images/providers/alibaba-credentials-form.png b/docs/images/providers/alibaba-credentials-form.png new file mode 100644 index 0000000000..3cefc0253c Binary files /dev/null and b/docs/images/providers/alibaba-credentials-form.png differ diff --git a/docs/images/providers/alibaba-get-role-arn.png b/docs/images/providers/alibaba-get-role-arn.png new file mode 100644 index 0000000000..d95822a18e Binary files /dev/null and b/docs/images/providers/alibaba-get-role-arn.png differ diff --git a/docs/images/providers/alibaba-ram-role-overview.png b/docs/images/providers/alibaba-ram-role-overview.png new file mode 100644 index 0000000000..a9420d0749 Binary files /dev/null and b/docs/images/providers/alibaba-ram-role-overview.png differ diff --git a/docs/images/providers/launch-scan-alibaba.png b/docs/images/providers/launch-scan-alibaba.png new file mode 100644 index 0000000000..324e1334c4 Binary files /dev/null and b/docs/images/providers/launch-scan-alibaba.png differ diff --git a/docs/images/providers/select-alibaba-cloud.png b/docs/images/providers/select-alibaba-cloud.png new file mode 100644 index 0000000000..8b65931472 Binary files /dev/null and b/docs/images/providers/select-alibaba-cloud.png differ diff --git a/docs/images/providers/select-auth-method-alibaba.png b/docs/images/providers/select-auth-method-alibaba.png new file mode 100644 index 0000000000..ffd0083bf0 Binary files /dev/null and b/docs/images/providers/select-auth-method-alibaba.png differ diff --git a/docs/images/prowler-app/add-cloud-provider.png b/docs/images/prowler-app/add-cloud-provider.png index dd42e73047..d8f19b2054 100644 Binary files a/docs/images/prowler-app/add-cloud-provider.png and b/docs/images/prowler-app/add-cloud-provider.png differ diff --git a/docs/images/prowler-app/cloud-providers-page.png b/docs/images/prowler-app/cloud-providers-page.png index 0581e0132f..dcbce73a10 100644 Binary files a/docs/images/prowler-app/cloud-providers-page.png and b/docs/images/prowler-app/cloud-providers-page.png differ diff --git a/docs/images/prowler-app/lighthouse-architecture.png b/docs/images/prowler-app/lighthouse-architecture.png deleted file mode 100644 index 63202ce7c7..0000000000 Binary files a/docs/images/prowler-app/lighthouse-architecture.png and /dev/null differ diff --git a/docs/img/prowler-cli-quick.gif b/docs/img/prowler-cli-quick.gif deleted file mode 100644 index 16197a5ba7..0000000000 Binary files a/docs/img/prowler-cli-quick.gif and /dev/null differ diff --git a/docs/img/prowler-cloud.gif b/docs/img/prowler-cloud.gif new file mode 100644 index 0000000000..b42043f1c1 Binary files /dev/null and b/docs/img/prowler-cloud.gif differ diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 44438f7278..928a527a88 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -23,19 +23,19 @@ The supported providers right now are: -| Provider | Support | Interface | -| -------------------------------------------------------------------------------- | ---------- | ------------ | -| [AWS](/user-guide/providers/aws/getting-started-aws) | Official | UI, API, CLI | -| [Azure](/user-guide/providers/azure/getting-started-azure) | Official | UI, API, CLI | -| [Google Cloud](/user-guide/providers/gcp/getting-started-gcp) | Official | UI, API, CLI | -| [Kubernetes](/user-guide/providers/kubernetes/getting-started-k8s) | Official | UI, API, CLI | -| [M365](/user-guide/providers/microsoft365/getting-started-m365) | Official | UI, API, CLI | -| [Github](/user-guide/providers/github/getting-started-github) | Official | UI, API, CLI | -| [Oracle Cloud](/user-guide/providers/oci/getting-started-oci) | Official | UI, API, CLI | -| [Infra as Code](/user-guide/providers/iac/getting-started-iac) | Official | UI, API, CLI | -| [MongoDB Atlas](/user-guide/providers/mongodbatlas/getting-started-mongodbatlas) | Official | UI, API, CLI | -| [LLM](/user-guide/providers/llm/getting-started-llm) | Official | CLI | -| **NHN** | Unofficial | CLI | +| Provider | Support | Audit Scope/Entities | Interface | +| -------------------------------------------------------------------------------- | ---------- | ---------------------------- | ------------ | +| [AWS](/user-guide/providers/aws/getting-started-aws) | Official | Accounts | UI, API, CLI | +| [Azure](/user-guide/providers/azure/getting-started-azure) | Official | Subscriptions | UI, API, CLI | +| [Google Cloud](/user-guide/providers/gcp/getting-started-gcp) | Official | Projects | UI, API, CLI | +| [Kubernetes](/user-guide/providers/kubernetes/getting-started-k8s) | Official | Clusters | UI, API, CLI | +| [M365](/user-guide/providers/microsoft365/getting-started-m365) | Official | Tenants | UI, API, CLI | +| [Github](/user-guide/providers/github/getting-started-github) | Official | Organizations / Repositories | UI, API, CLI | +| [Oracle Cloud](/user-guide/providers/oci/getting-started-oci) | Official | Tenancies / Compartments | UI, API, CLI | +| [Infra as Code](/user-guide/providers/iac/getting-started-iac) | Official | Repositories | UI, API, CLI | +| [MongoDB Atlas](/user-guide/providers/mongodbatlas/getting-started-mongodbatlas) | Official | Organizations | UI, API, CLI | +| [LLM](/user-guide/providers/llm/getting-started-llm) | Official | Models | CLI | +| **NHN** | Unofficial | Tenants | CLI | For more information about the checks and compliance of each provider visit [Prowler Hub](https://hub.prowler.com). diff --git a/docs/security.mdx b/docs/security.mdx index d8a3f63c73..fbf90e63ef 100644 --- a/docs/security.mdx +++ b/docs/security.mdx @@ -24,6 +24,80 @@ We enforce [pre-commit](https://github.com/prowler-cloud/prowler/blob/master/.pr Our container registries are continuously scanned for vulnerabilities, with findings automatically reported to our security team for assessment and remediation. This process evolves alongside our stack as we adopt new languages, frameworks, and technologies, ensuring our security practices remain comprehensive, proactive, and adaptable. +### Static Application Security Testing (SAST) + +We employ multiple SAST tools across our codebase to identify security vulnerabilities, code quality issues, and potential bugs during development: + +#### CodeQL Analysis +- **Scope**: UI (JavaScript/TypeScript), API (Python), and SDK (Python) +- **Frequency**: On every push and pull request, plus daily scheduled scans +- **Integration**: Results uploaded to GitHub Security tab via SARIF format +- **Purpose**: Identifies security vulnerabilities, coding errors, and potential exploits in source code + +#### Python Security Scanners +- **Bandit**: Detects common security issues in Python code (SQL injection, hardcoded passwords, etc.) + - Configured to ignore test files and report only high-severity issues + - Runs on both SDK and API codebases +- **Pylint**: Static code analysis with security-focused checks + - Integrated into pre-commit hooks and CI/CD pipelines + +#### Code Quality & Dead Code Detection +- **Vulture**: Identifies unused code that could indicate incomplete implementations or security gaps +- **Flake8**: Style guide enforcement with security-relevant checks +- **Shellcheck**: Security and correctness checks for shell scripts + +### Software Composition Analysis (SCA) + +We continuously monitor our dependencies for known vulnerabilities and ensure timely updates: + +#### Dependency Vulnerability Scanning +- **Safety**: Scans Python dependencies against known vulnerability databases + - Runs on every commit via pre-commit hooks + - Integrated into CI/CD for SDK and API + - Configured with selective ignores for tracked exceptions +- **Trivy**: Multi-purpose scanner for containers and dependencies + - Scans all container images (UI, API, SDK, MCP Server) + - Checks for vulnerabilities in OS packages and application dependencies + - Reports findings to GitHub Security tab + +#### Automated Dependency Updates +- **Dependabot**: Automated pull requests for dependency updates + - **Python (pip)**: Monthly updates for SDK + - **GitHub Actions**: Monthly updates for workflow dependencies + - **Docker**: Monthly updates for base images + - Temporarily paused for API and UI to maintain stability during active development + - **Security-first approach**: Even when paused, Dependabot automatically creates pull requests for security vulnerabilities, ensuring critical security patches are never delayed + +### Container Security + +All container images are scanned before deployment: + +- **Trivy Vulnerability Scanning**: + - Scans images for vulnerabilities and misconfigurations + - Generates SARIF reports uploaded to GitHub Security tab + - Creates PR comments with scan summaries + - Configurable to fail builds on critical findings + - Reports include CVE counts and remediation guidance +- **Hadolint**: Dockerfile linting to enforce best practices + - Validates Dockerfile syntax and structure + - Ensures secure image building practices + +### Secrets Detection + +We protect against accidental exposure of sensitive credentials: + +- **TruffleHog**: Scans entire codebase and Git history for secrets + - Runs on every push and pull request + - Pre-commit hook prevents committing secrets + - Detects high-entropy strings, API keys, tokens, and credentials + - Configured to report verified and unknown findings + +### Security Monitoring + +- **GitHub Security Tab**: Centralized view of all security findings from CodeQL, Trivy, and other SARIF-compatible tools +- **Artifact Retention**: Security scan reports retained for post-deployment analysis +- **PR Comments**: Automated security feedback on pull requests for rapid remediation + ## Reporting Vulnerabilities At Prowler, we consider the security of our open source software and systems a top priority. But no matter how much effort we put into system security, there can still be vulnerabilities present. diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index 80b6590669..1fbf715ea7 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -2,46 +2,87 @@ title: 'Troubleshooting' --- -- **Running `prowler` I get `[File: utils.py:15] [Module: utils] CRITICAL: path/redacted: OSError[13]`**: +## Running `prowler` I get `[File: utils.py:15] [Module: utils] CRITICAL: path/redacted: OSError[13]` - That is an error related to file descriptors or opened files allowed by your operating system. +That is an error related to file descriptors or opened files allowed by your operating system. - In macOS Ventura, the default value for the `file descriptors` is `256`. With the following command `ulimit -n 1000` you'll increase that value and solve the issue. +In macOS Ventura, the default value for the `file descriptors` is `256`. With the following command `ulimit -n 1000` you'll increase that value and solve the issue. - If you have a different OS and you are experiencing the same, please increase the value of your `file descriptors`. You can check it running `ulimit -a | grep "file descriptors"`. - - This error is also related with a lack of system requirements. To improve performance, Prowler stores information in memory so it may need to be run in a system with more than 1GB of memory. +If you have a different OS and you are experiencing the same, please increase the value of your `file descriptors`. You can check it running `ulimit -a | grep "file descriptors"`. +This error is also related with a lack of system requirements. To improve performance, Prowler stores information in memory so it may need to be run in a system with more than 1GB of memory. See section [Logging](/user-guide/cli/tutorials/logging) for further information or [contact us](/contact). ## Common Issues with Docker Compose Installation -- **Problem adding AWS Provider using "Connect assuming IAM Role" in Docker (see [GitHub Issue #7745](https://github.com/prowler-cloud/prowler/issues/7745))**: +### Problem adding AWS Provider using "Connect assuming IAM Role" in Docker - When running Prowler App via Docker, you may encounter errors such as `Provider not set`, `AWS assume role error - Unable to locate credentials`, or `Provider has no secret` when trying to add an AWS Provider using the "Connect assuming IAM Role" option. This typically happens because the container does not have access to the necessary AWS credentials or profiles. +See [GitHub Issue #7745](https://github.com/prowler-cloud/prowler/issues/7745) for more details. - **Workaround:** +When running Prowler App via Docker, you may encounter errors such as `Provider not set`, `AWS assume role error - Unable to locate credentials`, or `Provider has no secret` when trying to add an AWS Provider using the "Connect assuming IAM Role" option. This typically happens because the container does not have access to the necessary AWS credentials or profiles. - - Ensure your AWS credentials and configuration are available to the Docker container. You can do this by mounting your local `.aws` directory into the container. For example, in your `docker-compose.yaml`, add the following volume to the relevant services: +**Workaround:** - ```yaml - volumes: - - "${HOME}/.aws:/home/prowler/.aws:ro" - ``` - This should be added to the `api`, `worker`, and `worker-beat` services. +- Ensure your AWS credentials and configuration are available to the Docker container. You can do this by mounting your local `.aws` directory into the container. For example, in your `docker-compose.yaml`, add the following volume to the relevant services: - - Create or update your `~/.aws/config` and `~/.aws/credentials` files with the appropriate profiles and roles. For example: +```yaml +volumes: + - "${HOME}/.aws:/home/prowler/.aws:ro" +``` - ```ini - [profile prowler-profile] - role_arn = arn:aws:iam:::role/ProwlerScan - source_profile = default - ``` - And set the environment variable in your `.env` file: +This should be added to the `api`, `worker`, and `worker-beat` services. - ```env - AWS_PROFILE=prowler-profile - ``` +- Create or update your `~/.aws/config` and `~/.aws/credentials` files with the appropriate profiles and roles. For example: - - If you are scanning multiple AWS accounts, you may need to add multiple profiles to your AWS config. Note that this workaround is mainly for local testing; for production or multi-account setups, follow the [CloudFormation Template guide](https://github.com/prowler-cloud/prowler/issues/7745) and ensure the correct IAM roles and permissions are set up in each account. +```ini +[profile prowler-profile] +role_arn = arn:aws:iam:::role/ProwlerScan +source_profile = default +``` + +And set the environment variable in your `.env` file: + +```env +AWS_PROFILE=prowler-profile +``` + +- If you are scanning multiple AWS accounts, you may need to add multiple profiles to your AWS config. Note that this workaround is mainly for local testing; for production or multi-account setups, follow the [CloudFormation Template guide](https://github.com/prowler-cloud/prowler/issues/7745) and ensure the correct IAM roles and permissions are set up in each account. + +### Scans complete but reports are missing or compliance data is empty (`Too many open files` error) + +When running Prowler App via Docker Compose, you may encounter situations where scans complete successfully but reports are not available for download, compliance data shows as empty, or you see 404 errors when trying to access scan reports. Checking the `worker` container logs may reveal errors like `[Errno 24] Too many open files`. + +This issue occurs because the default file descriptor limits in Docker containers are too low for Prowler's operations. + +**Solution:** + +Add `ulimits` configuration to the `worker` and `worker-beat` services in your `docker-compose.yaml`: + +```yaml +services: + worker: + ulimits: + nofile: + soft: 65536 + hard: 65536 + # ... rest of service configuration + + worker-beat: + ulimits: + nofile: + soft: 65536 + hard: 65536 + # ... rest of service configuration +``` + +After making these changes, restart your Docker Compose stack: + +```bash +docker compose down +docker compose up -d +``` + + +We are evaluating adding these values to the default `docker-compose.yml` to avoid this issue in future releases. + diff --git a/docs/user-guide/cli/img/add-cloud-provider.png b/docs/user-guide/cli/img/add-cloud-provider.png index dd42e73047..d8f19b2054 100644 Binary files a/docs/user-guide/cli/img/add-cloud-provider.png and b/docs/user-guide/cli/img/add-cloud-provider.png differ diff --git a/docs/user-guide/cli/img/cloud-providers-page.png b/docs/user-guide/cli/img/cloud-providers-page.png index 0581e0132f..dcbce73a10 100644 Binary files a/docs/user-guide/cli/img/cloud-providers-page.png and b/docs/user-guide/cli/img/cloud-providers-page.png differ diff --git a/docs/user-guide/cli/img/lighthouse-architecture.png b/docs/user-guide/cli/img/lighthouse-architecture.png deleted file mode 100644 index 63202ce7c7..0000000000 Binary files a/docs/user-guide/cli/img/lighthouse-architecture.png and /dev/null differ diff --git a/docs/user-guide/cli/tutorials/configuration_file.mdx b/docs/user-guide/cli/tutorials/configuration_file.mdx index c2f886b3f3..73f9ff3999 100644 --- a/docs/user-guide/cli/tutorials/configuration_file.mdx +++ b/docs/user-guide/cli/tutorials/configuration_file.mdx @@ -93,6 +93,12 @@ The following list includes all the Azure checks with configurable variables tha ## GCP ### Configurable Checks +The following list includes all the GCP checks with configurable variables that can be changed in the configuration yaml file: + +| Check Name | Value | Type | +|---------------------------------------------------------------|--------------------------------------------------|-----------------| +| `compute_configuration_changes` | `compute_audit_log_lookback_days` | Integer | +| `compute_instance_group_multiple_zones` | `mig_min_zones` | Integer | ## Kubernetes @@ -548,6 +554,12 @@ gcp: # GCP Compute Configuration # gcp.compute_public_address_shodan shodan_api_key: null + # gcp.compute_configuration_changes + # Number of days to look back for Compute Engine configuration changes in audit logs + compute_audit_log_lookback_days: 1 + # gcp.compute_instance_group_multiple_zones + # Minimum number of zones a MIG should span for high availability + mig_min_zones: 2 # Kubernetes Configuration kubernetes: diff --git a/docs/user-guide/cli/tutorials/dashboard.mdx b/docs/user-guide/cli/tutorials/dashboard.mdx index 01c60b29e9..8c819743fe 100644 --- a/docs/user-guide/cli/tutorials/dashboard.mdx +++ b/docs/user-guide/cli/tutorials/dashboard.mdx @@ -1,5 +1,5 @@ --- -title: 'Dashboard' +title: "Dashboard" --- Prowler allows you to run your own local dashboards using the csv outputs provided by Prowler @@ -34,26 +34,30 @@ The overview page provides a full impression of your findings obtained from Prow This page allows for multiple functions: -* Apply filters: +- Apply filters: - * Assesment Date - * Account - * Region - * Severity - * Service - * Status + - Assesment Date + - Account + - Region + - Severity + - Service + - Provider + - Status + - Category -* See which files has been scanned to generate the dashboard by placing your mouse on the `?` icon: +- See which files has been scanned to generate the dashboard by placing your mouse on the `?` icon: - + {" "} + -* Download the `Top Findings by Severity` table using the button `DOWNLOAD THIS TABLE AS CSV` or `DOWNLOAD THIS TABLE AS XLSX` +- Download the `Top Findings by Severity` table using the button `DOWNLOAD THIS TABLE AS CSV` or `DOWNLOAD THIS TABLE AS XLSX` -* Click the provider cards to filter by provider. +- Click the provider cards to filter by provider. -* On the dropdowns under `Top Findings by Severity` you can apply multiple sorts to see the information, also you will get a detailed view of each finding using the dropdowns: +- On the dropdowns under `Top Findings by Severity` you can apply multiple sorts to see the information, also you will get a detailed view of each finding using the dropdowns: - + {" "} + ## Compliance Page @@ -110,17 +114,16 @@ To change the path, modify the values `folder_path_overview` or `folder_path_com If you have any issue related with dashboards, check that the output path where the dashboard is getting the outputs is correct. - ## Output Support Prowler dashboard supports the detailed outputs: -| Provider| V3| V4| COMPLIANCE-V3| COMPLIANCE-V4 -|----------|----------|----------|----------|---------- -| AWS| ✅| ✅| ✅| ✅ -| Azure| ❌| ✅| ❌| ✅ -| Kubernetes| ❌| ✅| ❌| ✅ -| GCP| ❌| ✅| ❌| ✅ -| M365| ❌| ✅| ❌| ✅ -| GitHub| ❌| ✅| ❌| ✅ +| Provider | V3 | V4 | COMPLIANCE-V3 | COMPLIANCE-V4 | +| ---------- | --- | --- | ------------- | ------------- | +| AWS | ✅ | ✅ | ✅ | ✅ | +| Azure | ❌ | ✅ | ❌ | ✅ | +| Kubernetes | ❌ | ✅ | ❌ | ✅ | +| GCP | ❌ | ✅ | ❌ | ✅ | +| M365 | ❌ | ✅ | ❌ | ✅ | +| GitHub | ❌ | ✅ | ❌ | ✅ | diff --git a/docs/user-guide/img/add-cloud-provider.png b/docs/user-guide/img/add-cloud-provider.png index dd42e73047..d8f19b2054 100644 Binary files a/docs/user-guide/img/add-cloud-provider.png and b/docs/user-guide/img/add-cloud-provider.png differ diff --git a/docs/user-guide/img/cloud-providers-page.png b/docs/user-guide/img/cloud-providers-page.png index 0581e0132f..dcbce73a10 100644 Binary files a/docs/user-guide/img/cloud-providers-page.png and b/docs/user-guide/img/cloud-providers-page.png differ diff --git a/docs/user-guide/img/lighthouse-architecture.png b/docs/user-guide/img/lighthouse-architecture.png deleted file mode 100644 index 63202ce7c7..0000000000 Binary files a/docs/user-guide/img/lighthouse-architecture.png and /dev/null differ diff --git a/docs/user-guide/providers/alibabacloud/authentication.mdx b/docs/user-guide/providers/alibabacloud/authentication.mdx new file mode 100644 index 0000000000..0baa160e69 --- /dev/null +++ b/docs/user-guide/providers/alibabacloud/authentication.mdx @@ -0,0 +1,112 @@ +--- +title: 'Alibaba Cloud Authentication in Prowler' +--- + +Prowler requires Alibaba Cloud credentials to perform security checks. Authentication is supported via multiple methods, prioritized as follows: + +1. **Credentials URI** +2. **OIDC Role Authentication** +3. **ECS RAM Role** +4. **RAM Role Assumption** +5. **STS Temporary Credentials** +6. **Permanent Access Keys** +7. **Default Credential Chain** + +## Authentication Methods + +### Credentials URI (Recommended for Centralized Services) + +If `--credentials-uri` is provided (or `ALIBABA_CLOUD_CREDENTIALS_URI` environment variable), Prowler will retrieve credentials from the specified external URI endpoint. The URI must return credentials in the standard JSON format. + +```bash +export ALIBABA_CLOUD_CREDENTIALS_URI="http://localhost:8080/credentials" +prowler alibabacloud +``` + +### OIDC Role Authentication (Recommended for ACK/Kubernetes) + +If OIDC environment variables are set, Prowler will use OIDC authentication to assume the specified role. This is the most secure method for containerized applications running in ACK (Alibaba Container Service for Kubernetes) with RRSA enabled. + +Required environment variables: +- `ALIBABA_CLOUD_ROLE_ARN` +- `ALIBABA_CLOUD_OIDC_PROVIDER_ARN` +- `ALIBABA_CLOUD_OIDC_TOKEN_FILE` + +```bash +export ALIBABA_CLOUD_ROLE_ARN="acs:ram::123456789012:role/YourRole" +export ALIBABA_CLOUD_OIDC_PROVIDER_ARN="acs:ram::123456789012:oidc-provider/ack-rrsa-provider" +export ALIBABA_CLOUD_OIDC_TOKEN_FILE="/var/run/secrets/tokens/oidc-token" +prowler alibabacloud +``` + +### ECS RAM Role (Recommended for ECS Instances) + +When running on an ECS instance with an attached RAM role, Prowler can obtain credentials from the ECS instance metadata service. + +```bash +# Using CLI argument +prowler alibabacloud --ecs-ram-role RoleName + +# Or using environment variable +export ALIBABA_CLOUD_ECS_METADATA="RoleName" +prowler alibabacloud +``` + +### RAM Role Assumption (Recommended for Cross-Account) + +For cross-account access, use RAM role assumption. You must provide the initial credentials (access keys) and the target role ARN. + +```bash +export ALIBABA_CLOUD_ACCESS_KEY_ID="your-access-key-id" +export ALIBABA_CLOUD_ACCESS_KEY_SECRET="your-access-key-secret" +export ALIBABA_CLOUD_ROLE_ARN="acs:ram::123456789012:role/ProwlerAuditRole" +prowler alibabacloud +``` + +### STS Temporary Credentials + +If you already have temporary STS credentials, you can provide them via environment variables. + +```bash +export ALIBABA_CLOUD_ACCESS_KEY_ID="your-sts-access-key-id" +export ALIBABA_CLOUD_ACCESS_KEY_SECRET="your-sts-access-key-secret" +export ALIBABA_CLOUD_SECURITY_TOKEN="your-sts-security-token" +prowler alibabacloud +``` + +### Permanent Access Keys + +You can use standard permanent access keys via environment variables. + +```bash +export ALIBABA_CLOUD_ACCESS_KEY_ID="your-access-key-id" +export ALIBABA_CLOUD_ACCESS_KEY_SECRET="your-access-key-secret" +prowler alibabacloud +``` + +## Required Permissions + +The credentials used by Prowler should have the minimum required permissions to audit the resources. At a minimum, the following permissions are recommended: + +- `ram:GetUser` +- `ram:ListUsers` +- `ram:GetPasswordPolicy` +- `ram:GetAccountSummary` +- `ram:ListVirtualMFADevices` +- `ram:ListGroups` +- `ram:ListPolicies` +- `ram:ListAccessKeys` +- `ram:GetLoginProfile` +- `ram:ListPoliciesForUser` +- `ram:ListGroupsForUser` +- `actiontrail:DescribeTrails` +- `oss:GetBucketLogging` +- `oss:GetBucketAcl` +- `rds:DescribeDBInstances` +- `rds:DescribeDBInstanceAttribute` +- `ecs:DescribeInstances` +- `vpc:DescribeVpcs` +- `sls:ListProject` +- `sls:ListAlerts` +- `sls:ListLogStores` +- `sls:GetLogStore` diff --git a/docs/user-guide/providers/alibabacloud/getting-started-alibabacloud.mdx b/docs/user-guide/providers/alibabacloud/getting-started-alibabacloud.mdx new file mode 100644 index 0000000000..c38cc42e37 --- /dev/null +++ b/docs/user-guide/providers/alibabacloud/getting-started-alibabacloud.mdx @@ -0,0 +1,162 @@ +--- +title: 'Getting Started With Alibaba Cloud on Prowler' +--- + +import { VersionBadge } from "/snippets/version-badge.mdx" + +Prowler supports Alibaba Cloud both from the CLI and from Prowler Cloud. This guide walks you through the requirements, how to connect the provider in the UI, and how to run scans from the command line. + +## Prerequisites + +Before you begin, make sure you have: + +1. An **Alibaba Cloud Account ID** (visible in the Alibaba Cloud Console under your profile). +2. **Credentials** with appropriate permissions: + - **RAM User with Access Keys**: For static credential authentication. + - **RAM Role**: For cross-account access using role assumption (recommended). +3. The required permissions for Prowler to audit your resources. See the [Alibaba Cloud Authentication](/user-guide/providers/alibabacloud/authentication) guide for the full list of required permissions. + + + + Onboard Alibaba Cloud using Prowler Cloud + + + Onboard Alibaba Cloud using Prowler CLI + + + +## Prowler Cloud + + + +### Step 1: Get Your Alibaba Cloud Account ID + +1. Log in to the [Alibaba Cloud Console](https://home.console.alibabacloud.com/) +2. Click on your profile avatar in the top-right corner +3. Locate and copy your Account ID + +![Get Account ID](/images/providers/alibaba-account-id.png) + +### Step 2: Access Prowler Cloud or Prowler App + +1. Navigate to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](/user-guide/tutorials/prowler-app) +2. Go to "Configuration" > "Cloud Providers" + + ![Cloud Providers Page](/images/prowler-app/cloud-providers-page.png) + +3. Click "Add Cloud Provider" + + ![Add a Cloud Provider](/images/prowler-app/add-cloud-provider.png) + +4. Select "Alibaba Cloud" + + ![Select Alibaba Cloud](/images/providers/select-alibaba-cloud.png) + +5. Enter your Alibaba Cloud Account ID and optionally provide a friendly alias + + ![Add Account ID](/images/providers/add-alibaba-account-id.png) + +### Step 3: Choose and Provide Authentication + +After the Account ID is in place, select the authentication method that matches your Alibaba Cloud setup: + +![Select Auth Method](/images/providers/select-auth-method-alibaba.png) + +#### RAM Role Assumption (Recommended) + +Use this method for secure cross-account access. For detailed instructions on how to create the RAM role, see the [Authentication guide](/user-guide/providers/alibabacloud/authentication#ram-role-assumption-recommended-for-cross-account). + +1. Enter the **Role ARN** (format: `acs:ram:::role/`) +2. Enter the **Access Key ID** and **Access Key Secret** of the RAM user that will assume the role + + ![Input the Role ARN](/images/providers/alibaba-get-role-arn.png) + + +The RAM user whose credentials you provide must have permission to assume the target role. For more details, see the [Alibaba Cloud AssumeRole API documentation](https://www.alibabacloud.com/help/en/ram/developer-reference/api-sts-2015-04-01-assumerole). + + +#### Credentials (Static Access Keys) + +Use static credentials for quick scans (not recommended for production). For detailed setup, see the [Authentication guide](/user-guide/providers/alibabacloud/authentication#permanent-access-keys). + +1. Enter the **Access Key ID** and **Access Key Secret** + + ![Filled Credentials Page](/images/providers/alibaba-credentials-form.png) + + +Static access keys are long-lived credentials. For production environments, consider using RAM Role Assumption instead. + + +### Step 4: Launch the Scan + +1. Click "Next" to review your configuration +2. Click "Launch Scan" to start auditing your Alibaba Cloud account + + ![Launch Scan](/images/providers/launch-scan-alibaba.png) + +--- + +## Prowler CLI + + + +You can also run Alibaba Cloud assessments directly from the CLI. Both command-line flags and environment variables are supported. + +### Step 1: Select an Authentication Method + +Choose one of the following authentication methods. For the complete list and detailed configuration, see the [Authentication guide](/user-guide/providers/alibabacloud/authentication). + +#### Environment Variables + +```bash +export ALIBABA_CLOUD_ACCESS_KEY_ID="your-access-key-id" +export ALIBABA_CLOUD_ACCESS_KEY_SECRET="your-access-key-secret" +prowler alibabacloud +``` + +#### RAM Role Assumption + +```bash +export ALIBABA_CLOUD_ACCESS_KEY_ID="your-access-key-id" +export ALIBABA_CLOUD_ACCESS_KEY_SECRET="your-access-key-secret" +export ALIBABA_CLOUD_ROLE_ARN="acs:ram::123456789012:role/ProwlerAuditRole" +prowler alibabacloud +``` + +#### ECS RAM Role (for ECS instances) + +```bash +prowler alibabacloud --ecs-ram-role RoleName +``` + +### Step 2: Run the First Scan + +#### Scan all regions + +```bash +prowler alibabacloud +``` + +#### Scan specific regions + +```bash +prowler alibabacloud --regions cn-hangzhou cn-shanghai +``` + +#### Run specific checks + +```bash +prowler alibabacloud --checks ram_no_root_access_key ram_user_mfa_enabled_console_access +``` + +#### Run a compliance framework + +```bash +prowler alibabacloud --compliance cis_2.0_alibabacloud +``` + +### Additional Tips + +- Combine flags (for example, `--checks` or `--services`) just like with other providers. +- Use `--output-modes` to export findings in JSON, CSV, ASFF, etc. +- For more authentication options (OIDC, Credentials URI, STS), see the [Authentication guide](/user-guide/providers/alibabacloud/authentication). diff --git a/docs/user-guide/providers/aws/regions-and-partitions.mdx b/docs/user-guide/providers/aws/regions-and-partitions.mdx index 2676a02ef2..02ae8fe6ab 100644 --- a/docs/user-guide/providers/aws/regions-and-partitions.mdx +++ b/docs/user-guide/providers/aws/regions-and-partitions.mdx @@ -6,15 +6,16 @@ By default Prowler is able to scan the following AWS partitions: - Commercial: `aws` - China: `aws-cn` +- European Sovereign Cloud: `aws-eusc` - GovCloud (US): `aws-us-gov` To check the available regions for each partition and service, refer to: [aws\_regions\_by\_service.json](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/aws/aws_regions_by_service.json) -## Scanning AWS China and GovCloud Partitions in Prowler +## Scanning AWS China, European Sovereign Cloud and GovCloud Partitions in Prowler -When scanning the China (`aws-cn`) or GovCloud (`aws-us-gov`), ensure one of the following: +When scanning the China (`aws-cn`), European Sovereign Cloud (`aws-eusc`) or GovCloud (`aws-us-gov`) partitions, ensure one of the following: - Your AWS credentials include a valid region within the desired partition. @@ -83,6 +84,29 @@ To scan an account in the AWS GovCloud (US) partition (`aws-us-gov`): With this configuration, all partition regions will be scanned without needing the `-f/--region` flag + +### AWS European Sovereign Cloud + +To scan an account in the AWS European Sovereign Cloud partition (`aws-eusc`): + +- By using the `-f/--region` flag: + + ``` + prowler aws --region eusc-de-east-1 + ``` + +- By using the region configured in your AWS profile at `~/.aws/credentials` or `~/.aws/config`: + + ``` + [default] + aws_access_key_id = XXXXXXXXXXXXXXXXXXX + aws_secret_access_key = XXXXXXXXXXXXXXXXXXX + region = eusc-de-east-1 + ``` + + +With this configuration, all partition regions will be scanned without needing the `-f/--region` flag + ### AWS ISO (US \& Europe) @@ -99,6 +123,9 @@ The AWS ISO partitions—commonly referred to as "secret partitions"—are air-g "cn-north-1", "cn-northwest-1" ], + "aws-eusc": [ + "eusc-de-east-1" + ], "aws-us-gov": [ "us-gov-east-1", "us-gov-west-1" diff --git a/docs/user-guide/providers/aws/role-assumption.mdx b/docs/user-guide/providers/aws/role-assumption.mdx index 3a2461b50b..b714beafbd 100644 --- a/docs/user-guide/providers/aws/role-assumption.mdx +++ b/docs/user-guide/providers/aws/role-assumption.mdx @@ -69,7 +69,19 @@ If your IAM Role is configured with Multi-Factor Authentication (MFA), use `--mf ## Creating a Role for One or Multiple Accounts -To create an IAM role that can be assumed in one or multiple AWS accounts, use either a CloudFormation Stack or StackSet and adapt the provided [template](https://github.com/prowler-cloud/prowler/blob/master/permissions/create_role_to_assume_cfn.yaml). +To create an IAM role that can be assumed in one or multiple AWS accounts, use either a CloudFormation Stack or StackSet with the provided [template](https://github.com/prowler-cloud/prowler/blob/master/permissions/templates/cloudformation/prowler-scan-role.yml). + +The template requires the following parameters: + +- **ExternalId:** A unique identifier to prevent the [confused deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html) +- **AccountId:** *(Optional)* AWS Account ID that will assume the role (default: Prowler Cloud account) +- **IAMPrincipal:** *(Optional)* The IAM principal allowed to assume the role (default: `role/prowler*`) + +When running Prowler CLI, include the External ID using the `-I/--external-id` flag: + +```sh +prowler aws -R arn:aws:iam:::role/ProwlerScan -I +``` **Session Duration Considerations**: Depending on the number of checks performed and the size of your infrastructure, Prowler may require more than 1 hour to complete. Use the `-T ` option to allow up to 12 hours (43,200 seconds). If you need more than 1 hour, modify the _“Maximum CLI/API session duration”_ setting for the role. Learn more [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session). diff --git a/docs/user-guide/providers/azure/authentication.mdx b/docs/user-guide/providers/azure/authentication.mdx index 4e33da0775..d1abb023f7 100644 --- a/docs/user-guide/providers/azure/authentication.mdx +++ b/docs/user-guide/providers/azure/authentication.mdx @@ -30,6 +30,7 @@ Assign the following Microsoft Graph permissions: - `Directory.Read.All` - `Policy.Read.All` - `UserAuthenticationMethod.Read.All` (optional, for multifactor authentication (MFA) checks) +- `AuditLog.Read.All` (optional, for multifactor authentication (MFA) checks) Replace `Directory.Read.All` with `Domain.Read.All` for more restrictive permissions. Note that Entra checks related to DirectoryRoles and GetUsers will not run with this permission. @@ -51,6 +52,7 @@ Replace `Directory.Read.All` with `Domain.Read.All` for more restrictive permiss - `Directory.Read.All` - `Policy.Read.All` - `UserAuthenticationMethod.Read.All` + - `AuditLog.Read.All` ![Permission Screenshots](/images/providers/domain-permission.png) @@ -62,7 +64,7 @@ Replace `Directory.Read.All` with `Domain.Read.All` for more restrictive permiss 1. To grant permissions to a Service Principal, execute the following command in a terminal: ```console - az ad app permission add --id {appId} --api 00000003-0000-0000-c000-000000000000 --api-permissions 7ab1d382-f21e-4acd-a863-ba3e13f7da61=Role 246dd0d5-5bd0-4def-940b-0421030a5b68=Role 38d9df27-64da-44fd-b7c5-a6fbac20248f=Role + az ad app permission add --id {appId} --api 00000003-0000-0000-c000-000000000000 --api-permissions 7ab1d382-f21e-4acd-a863-ba3e13f7da61=Role 246dd0d5-5bd0-4def-940b-0421030a5b68=Role 38d9df27-64da-44fd-b7c5-a6fbac20248f=Role b0afded3-3588-46d8-8b3d-9842eff778da=Role ``` @@ -246,12 +248,223 @@ prowler azure --az-cli-auth *Available only for Prowler CLI* -Authenticate via Azure Managed Identity (when running on Azure resources): +Authenticate via Azure Managed Identity when running Prowler on Azure resources (VMs, Container Instances, Azure Functions, etc.): ```console prowler azure --managed-identity-auth ``` +### Prerequisites + +Before using Managed Identity authentication, the following steps are required: + +1. **Enable Managed Identity** on the Azure resource (e.g., VM, Container Instance) +2. **Assign the required permissions** to the Managed Identity on the target subscription(s) to scan + + +A common misconception is that enabling a Managed Identity on a resource automatically grants it permissions. **This is not the case.** Without explicit role assignments, Prowler will be unable to scan subscriptions and will return authorization errors, resulting in incomplete security assessments. The Managed Identity itself is a service principal that must be explicitly granted Reader and ProwlerRole permissions on each subscription to scan. + + +### Step-by-Step Setup Guide + +#### Step 1: Enable Managed Identity on the Azure Resource + + + + **Via Azure Portal:** + 1. Navigate to the VM in Azure Portal + 2. Select "Identity" from the left menu under "Security" + 3. Under "System assigned" tab, set Status to "On" + 4. Click "Save" + 5. Note the "Object (principal) ID" - this value is required for permission assignment + + **Via Azure CLI:** + ```console + # Enable system-assigned managed identity + az vm identity assign --name --resource-group + + # Get the principal ID + az vm identity show --name --resource-group --query principalId -o tsv + ``` + + + **Via Azure CLI:** + ```console + # Enable system-assigned managed identity + az container create \ + --resource-group \ + --name \ + --image \ + --assign-identity + + # Get the principal ID + az container show --resource-group --name --query identity.principalId -o tsv + ``` + + + +#### Step 2: Assign Reader Role to the Managed Identity + +The Managed Identity needs the **Reader** role on each subscription to scan. This role must be assigned to the **Managed Identity's principal ID**, not the VM or resource itself. + + + + 1. Navigate to the **target subscription** to scan (not the VM's resource group) + 2. Select "Access control (IAM)" from the left menu + 3. Click "+ Add" > "Add role assignment" + 4. Select "Reader" role, click "Next" + 5. Click "+ Select members" + 6. Search for the VM name or paste the Managed Identity's Object/Principal ID + 7. Select it and click "Select" + 8. Click "Review + assign" + + + When scanning a subscription different from where the VM is located, ensure the role is assigned on the **target subscription**, not the VM's subscription. + + + + ```console + # Get the principal ID of the resource's managed identity + PRINCIPAL_ID=$(az vm identity show --name --resource-group --query principalId -o tsv) + + # Assign Reader role on the target subscription + az role assignment create \ + --role "Reader" \ + --assignee-object-id $PRINCIPAL_ID \ + --assignee-principal-type ServicePrincipal \ + --scope /subscriptions/ + ``` + + + +#### Step 3: Create and Assign ProwlerRole to the Managed Identity + +The ProwlerRole is a custom role required for specific security checks. First, create the role if it does not exist, then assign it to the Managed Identity. + + + + **Create the ProwlerRole:** + ```console + az role definition create --role-definition '{ + "Name": "ProwlerRole", + "IsCustom": true, + "Description": "Role used for checks that require read-only access to Azure resources and are not covered by the Reader role.", + "AssignableScopes": ["/subscriptions/"], + "Actions": [ + "Microsoft.Web/sites/host/listkeys/action", + "Microsoft.Web/sites/config/list/Action" + ] + }' + ``` + + **Assign ProwlerRole to the Managed Identity:** + ```console + # Get the principal ID if not already available + PRINCIPAL_ID=$(az vm identity show --name --resource-group --query principalId -o tsv) + + # Assign ProwlerRole on the target subscription + az role assignment create \ + --role "ProwlerRole" \ + --assignee-object-id $PRINCIPAL_ID \ + --assignee-principal-type ServicePrincipal \ + --scope /subscriptions/ + ``` + + + Follow the same process as creating the ProwlerRole in the [Assigning ProwlerRole Permissions](/user-guide/providers/azure/authentication#assigning-prowlerrole-permissions-at-the-subscription-level) section, then assign it to the Managed Identity using the same steps as the Reader role assignment. + + + +#### Step 4: (Optional) Assign Microsoft Graph Permissions + +For Entra ID (Azure AD) checks, the Managed Identity needs Microsoft Graph API permissions: `Directory.Read.All`, `Policy.Read.All`, and optionally `UserAuthenticationMethod.Read.All` and `AuditLog.Read.All`. + + +Assigning Microsoft Graph API permissions to a Managed Identity requires Azure CLI or PowerShell - it cannot be done through the Azure Portal's standard role assignment interface. + + +```console +# Get the Managed Identity's principal ID +PRINCIPAL_ID=$(az vm identity show --name --resource-group --query principalId -o tsv) + +# Get Microsoft Graph's service principal ID +GRAPH_SP_ID=$(az ad sp list --display-name "Microsoft Graph" --query [0].id -o tsv) + +# Assign Directory.Read.All permission (App Role ID: 7ab1d382-f21e-4acd-a863-ba3e13f7da61) +az rest --method POST \ + --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$PRINCIPAL_ID/appRoleAssignments" \ + --headers "Content-Type=application/json" \ + --body "{\"principalId\": \"$PRINCIPAL_ID\", \"resourceId\": \"$GRAPH_SP_ID\", \"appRoleId\": \"7ab1d382-f21e-4acd-a863-ba3e13f7da61\"}" + +# Assign Policy.Read.All permission (App Role ID: 246dd0d5-5bd0-4def-940b-0421030a5b68) +az rest --method POST \ + --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$PRINCIPAL_ID/appRoleAssignments" \ + --headers "Content-Type=application/json" \ + --body "{\"principalId\": \"$PRINCIPAL_ID\", \"resourceId\": \"$GRAPH_SP_ID\", \"appRoleId\": \"246dd0d5-5bd0-4def-940b-0421030a5b68\"}" +``` + +#### Step 5: Run Prowler + +SSH or connect to the Azure resource and run Prowler: + +```console +# Scan all accessible subscriptions +prowler azure --managed-identity-auth + +# Scan specific subscription(s) +prowler azure --managed-identity-auth --subscription-ids +``` + + +Wait a few minutes after assigning roles for Azure to propagate permissions. Role assignments are not always immediately effective. + + +### Troubleshooting + +#### Error: "No subscriptions were found, please check your permission assignments" + +**Cause:** The Managed Identity does not have the Reader role assigned on any subscription. + +**Solution:** +- Verify the Managed Identity has the Reader role assigned on at least one subscription. +- Wait a few minutes after role assignment for Azure to propagate permissions. +- Verify role assignments: + ```console + az role assignment list --assignee --all + ``` + +#### Error: "does not have authorization to perform action 'Microsoft.Resources/subscriptions/read'" + +**Cause:** The Managed Identity lacks the Reader role on the target subscription. + +**Solution:** +- Ensure the Reader role is assigned to the **Managed Identity's principal ID**, not the VM resource. +- Verify the role is assigned on the **target subscription** to scan, not just the VM's resource group. +- Check role assignments: + ```console + az role assignment list --assignee --scope /subscriptions/ + ``` + +#### Error: "CredentialUnavailableError: ManagedIdentityCredential authentication unavailable" + +**Cause:** Managed Identity is not enabled on the resource, or Prowler is running outside of Azure. + +**Solution:** +- Verify Managed Identity is enabled on the Azure resource. +- Ensure Prowler is running from within the Azure resource (not a local machine). +- Check Managed Identity status: + ```console + az vm identity show --name --resource-group + ``` + +#### Error: Access token validation failure for Entra ID checks + +**Cause:** The Managed Identity lacks Microsoft Graph API permissions. + +**Solution:** +- Assign the required Graph API permissions as shown in Step 4. +- These permissions are optional for basic resource scanning but required for Entra ID security checks. + ## Browser Authentication *Available only for Prowler CLI* diff --git a/docs/user-guide/providers/cloudflare/authentication.mdx b/docs/user-guide/providers/cloudflare/authentication.mdx new file mode 100644 index 0000000000..0f4bbb8de6 --- /dev/null +++ b/docs/user-guide/providers/cloudflare/authentication.mdx @@ -0,0 +1,146 @@ +--- +title: 'Cloudflare Authentication' +--- + +Prowler for Cloudflare supports the following authentication methods: + +- [**API Token**](#api-token-recommended) (**Recommended**) +- [**API Key and Email (Legacy)**](#api-key-and-email-legacy) + +## Required Permissions + +Prowler requires read-only access to your Cloudflare zones and their settings. The following permissions are needed: + +| Permission | Description | +|------------|-------------| +| `Zone:Read` | Read access to zone settings and configurations | +| `Zone Settings:Read` | Read access to zone security settings (SSL/TLS, HSTS, etc.) | +| `DNS:Read` | Read access to DNS records (for DNSSEC checks) | + + +Ensure your API Token or API Key has access to all zones you want to scan. If permissions are missing, some checks may fail or return incomplete results. + + +## API Token (Recommended) + +API Tokens are the recommended authentication method because they: +- Can be scoped to specific permissions and zones +- Are more secure than global API keys +- Can be easily rotated without affecting other integrations + +### Step 1: Create an API Token + +1. **Log into Cloudflare Dashboard** + - Go to [https://dash.cloudflare.com](https://dash.cloudflare.com) and sign in + +2. **Navigate to API Tokens** + - Click on your profile icon in the top right corner + - Select **My Profile** + - Click on the **API Tokens** tab + +3. **Create a Custom Token** + - Click **Create Token** + - Select **Create Custom Token** (at the bottom) + +4. **Configure Token Permissions** + + Give your token a descriptive name (e.g., "Prowler Security Scanner") and add the [required permissions](#required-permissions) listed above. + +5. **Set Zone Resources** + - Under **Zone Resources**, select either: + - **Include → All zones** (to scan all zones in your account) + - **Include → Specific zone** (to limit access to specific zones) + +6. **Create and Copy Token** + - Click **Continue to summary** + - Review the permissions and click **Create Token** + - **Copy the token immediately** - Cloudflare will only show it once + +### Step 2: Store the Token Securely + +Store your API token as an environment variable: + +```bash +export CLOUDFLARE_API_TOKEN="your-api-token-here" +``` + + +Never commit API tokens to version control or share them in plain text. Use environment variables or a secrets manager. + + +## API Key and Email (Legacy) + +API Keys provide full access to your Cloudflare account. While supported, this method is less secure than API Tokens because it grants broader permissions. + +### Step 1: Get Your API Key + +1. **Log into Cloudflare Dashboard** + - Go to [https://dash.cloudflare.com](https://dash.cloudflare.com) and sign in + +2. **Navigate to API Tokens** + - Click on your profile icon in the top right corner + - Select **My Profile** + - Click on the **API Tokens** tab + +3. **View Global API Key** + - Scroll down to the **API Keys** section + - Click **View** next to **Global API Key** + - Enter your password to reveal the key + - Copy the API key + +### Step 2: Store Credentials Securely + +Store both your API key and email as environment variables: + +```bash +export CLOUDFLARE_API_KEY="your-api-key-here" +export CLOUDFLARE_API_EMAIL="your-email@example.com" +``` + + +The email must be the same email address used to log into your Cloudflare account. + + +## Best Practices + +### Security Recommendations + +- **Use API Tokens instead of API Keys** - Tokens can be scoped to specific permissions +- **Use environment variables** - Never hardcode credentials in scripts or commands +- **Rotate credentials regularly** - Create new tokens periodically and revoke old ones +- **Use least privilege** - Only grant the minimum permissions needed +- **Monitor token usage** - Review the Cloudflare audit log for suspicious activity + + +**Use only one authentication method at a time.** If both API Token and API Key + Email are set, Prowler will use the API Token and log an error message. + + +## Troubleshooting + +### "Missing X-Auth-Email header" Error + +This error occurs when using API Key authentication without providing the email address. Ensure both `CLOUDFLARE_API_KEY` and `CLOUDFLARE_API_EMAIL` are set. + +### "Authentication error" or "Permission denied" + +- Verify your API Token or API Key is correct and not expired +- Check that your token has the [required permissions](#required-permissions) +- Ensure your token has access to the zones you're trying to scan + +### "Both API Token and API Key and Email credentials are set" + +This warning appears when all three environment variables are set: +- `CLOUDFLARE_API_TOKEN` +- `CLOUDFLARE_API_KEY` +- `CLOUDFLARE_API_EMAIL` + +To resolve, unset the credentials you don't want to use: + +```bash +# To use API Token only (recommended) +unset CLOUDFLARE_API_KEY +unset CLOUDFLARE_API_EMAIL + +# Or to use API Key and Email only +unset CLOUDFLARE_API_TOKEN +``` diff --git a/docs/user-guide/providers/cloudflare/getting-started-cloudflare.mdx b/docs/user-guide/providers/cloudflare/getting-started-cloudflare.mdx new file mode 100644 index 0000000000..e32258eda3 --- /dev/null +++ b/docs/user-guide/providers/cloudflare/getting-started-cloudflare.mdx @@ -0,0 +1,104 @@ +--- +title: 'Getting Started with Cloudflare' +--- + +Prowler for Cloudflare allows you to scan your Cloudflare zones for security misconfigurations, including SSL/TLS settings, DNSSEC, HSTS, and more. + +## Prerequisites + +Before running Prowler with the Cloudflare provider, ensure you have: + +1. A Cloudflare account with at least one zone +2. One of the following authentication methods configured (see [Authentication](/user-guide/providers/cloudflare/authentication)): + - An **API Token** (recommended) + - An **API Key + Email** (legacy) + +## Quick Start + +### Step 1: Set Up Authentication + +The recommended method is using an API Token via environment variable: + +```bash +export CLOUDFLARE_API_TOKEN="your-api-token-here" +``` + +Alternatively, use API Key + Email: + +```bash +export CLOUDFLARE_API_KEY="your-api-key-here" +export CLOUDFLARE_API_EMAIL="your-email@example.com" +``` + +### Step 2: Run Prowler + +Run a scan across all your Cloudflare zones: + +```bash +prowler cloudflare +``` + +That's it! Prowler will automatically discover all zones in your account and run security checks against them. + +## Authentication + +Prowler reads Cloudflare credentials from environment variables. Set your credentials before running Prowler: + +**API Token (Recommended):** +```bash +export CLOUDFLARE_API_TOKEN="your-api-token-here" +prowler cloudflare +``` + +**API Key + Email (Legacy):** +```bash +export CLOUDFLARE_API_KEY="your-api-key-here" +export CLOUDFLARE_API_EMAIL="your-email@example.com" +prowler cloudflare +``` + +## Filtering Zones + +By default, Prowler scans all zones accessible with your credentials: + +```bash +prowler cloudflare +``` + +To scan only specific zones, use the `-f`, `--region`, or `--filter-region` argument: + +```bash +prowler cloudflare -f example.com +``` + +You can specify multiple zones: + +```bash +prowler cloudflare -f example.com example.org +``` + +You can also use zone IDs instead of domain names: + +```bash +prowler cloudflare -f 023e105f4ecef8ad9ca31a8372d0c353 +``` + +## Configuration + +Prowler uses a configuration file to customize provider behavior. The Cloudflare configuration includes: + +```yaml +cloudflare: + # Maximum number of retries for API requests (default is 2) + max_retries: 2 +``` + +To use a custom configuration: + +```bash +prowler cloudflare --config-file /path/to/config.yaml +``` + +## Next Steps + +- [Authentication](/user-guide/providers/cloudflare/authentication) - Detailed guide on creating API tokens and keys diff --git a/docs/user-guide/providers/iac/getting-started-iac.mdx b/docs/user-guide/providers/iac/getting-started-iac.mdx index 315afafed4..1a3bb20371 100644 --- a/docs/user-guide/providers/iac/getting-started-iac.mdx +++ b/docs/user-guide/providers/iac/getting-started-iac.mdx @@ -5,18 +5,26 @@ import { VersionBadge } from "/snippets/version-badge.mdx" Prowler's Infrastructure as Code (IaC) provider enables scanning of local or remote infrastructure code for security and compliance issues using [Trivy](https://trivy.dev/). This provider supports a wide range of IaC frameworks, allowing assessment of code before deployment. -## Supported Scanners +## Supported IaC Formats -The IaC provider leverages [Trivy](https://trivy.dev/latest/docs/scanner/vulnerability/) to support multiple scanners, including: +Prowler IaC provider scans the following Infrastructure as Code configurations for misconfigurations and secrets: -- Vulnerability -- Misconfiguration -- Secret -- License +| Configuration Type | File Patterns | +|--------------------|----------------------------------------------| +| Kubernetes | `*.yml`, `*.yaml`, `*.json` | +| Docker | `Dockerfile`, `Containerfile` | +| Terraform | `*.tf`, `*.tf.json`, `*.tfvars` | +| Terraform Plan | `tfplan`, `*.tfplan`, `*.json` | +| CloudFormation | `*.yml`, `*.yaml`, `*.json` | +| Azure ARM Template | `*.json` | +| Helm | `*.yml`, `*.yaml`, `*.tpl`, `*.tar.gz`, etc. | +| YAML | `*.yaml`, `*.yml` | +| JSON | `*.json` | +| Ansible | `*.yml`, `*.yaml`, `*.json`, `*.ini`, without extension | ## How It Works -- The IaC provider scans local directories (or specified paths) for supported IaC files, or scans remote repositories. +- Prowler App leverages [Trivy](https://trivy.dev/docs/latest/guide/coverage/iac/#scanner) to scan local directories (or specified paths) for supported IaC files, or scans remote repositories. - No cloud credentials or authentication are required for local scans. - For remote repository scans, authentication can be provided via [git URL](https://git-scm.com/docs/git-clone#_git_urls), CLI flags or environment variables. - Check the [IaC Authentication](/user-guide/providers/iac/authentication) page for more details. @@ -27,6 +35,10 @@ The IaC provider leverages [Trivy](https://trivy.dev/latest/docs/scanner/vulnera +### Supported Scanners + +Scanner selection is not configurable in Prowler App. Default scanners, misconfig and secret, run automatically during each scan. + ### Step 1: Access Prowler Cloud/App 1. Navigate to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](/user-guide/tutorials/prowler-app) @@ -63,6 +75,17 @@ The IaC provider leverages [Trivy](https://trivy.dev/latest/docs/scanner/vulnera +### Supported Scanners + +Prowler CLI supports the following scanners: + +- [Vulnerability](https://trivy.dev/docs/latest/guide/scanner/vulnerability/) +- [Misconfiguration](https://trivy.dev/docs/latest/guide/scanner/misconfiguration/) +- [Secret](https://trivy.dev/docs/latest/guide/scanner/secret/) +- [License](https://trivy.dev/docs/latest/guide/scanner/license/) + +By default, only misconfiguration and secret scanners run during a scan. To specify which scanners to use, refer to the [Specify Scanners](#specify-scanners) section below. + ### Usage Use the `iac` argument to run Prowler with the IaC provider. Specify the directory or repository to scan, frameworks to include, and paths to exclude. @@ -103,7 +126,7 @@ Authentication for private repositories can be provided using one of the followi #### Specify Scanners -Scan only vulnerability and misconfiguration scanners: +To run only specific scanners, use the `--scanners` flag. For example, to scan only for vulnerabilities and misconfigurations: ```sh prowler iac --scan-path ./my-iac-directory --scanners vuln misconfig diff --git a/docs/user-guide/providers/kubernetes/getting-started-k8s.mdx b/docs/user-guide/providers/kubernetes/getting-started-k8s.mdx index ae70c06263..0ec8215776 100644 --- a/docs/user-guide/providers/kubernetes/getting-started-k8s.mdx +++ b/docs/user-guide/providers/kubernetes/getting-started-k8s.mdx @@ -47,8 +47,43 @@ If you are adding an **EKS**, **GKE**, **AKS** or external cluster, follow these kubectl create token prowler-sa -n prowler-ns --duration=0 ``` - - **Security Note:** The `--duration=0` option generates a non-expiring token, which may pose a security risk if not managed properly. Users should decide on an appropriate expiration time based on their security policies. If a limited-time token is preferred, set `--duration=