diff --git a/.config/wt.toml b/.config/wt.toml new file mode 100644 index 0000000000..4690e8d918 --- /dev/null +++ b/.config/wt.toml @@ -0,0 +1,23 @@ +# Prowler worktree automation for worktrunk (wt CLI). +# Runs automatically on `wt switch --create`. + +# Block 1: setup + copy gitignored env files (.envrc, ui/.env.local) +# from the primary worktree — patterns selected via .worktreeinclude. +[[pre-start]] +skills = "./skills/setup.sh --claude" +python = "poetry env use python3.12" +envs = "wt step copy-ignored" + +# Block 2: install Python deps (requires `poetry env use` from block 1). +[[pre-start]] +deps = "poetry install --with dev" + +# Block 3: reminder — last visible output before `wt switch` returns. +# Hooks can't mutate the parent shell, so venv activation is manual. +[[pre-start]] +reminder = "echo '>> Reminder: activate the venv in this shell with: eval $(poetry env activate)'" + +# Background: pnpm install runs while you start working. +# Tail logs via `wt config state logs`. +[post-start] +ui = "cd ui && pnpm install" diff --git a/.env b/.env index f405293de7..dd67da71ed 100644 --- a/.env +++ b/.env @@ -78,6 +78,9 @@ TASK_RETRY_ATTEMPTS=5 # Valkey settings # If running Valkey and celery on host, use localhost, else use 'valkey' +VALKEY_SCHEME=redis +VALKEY_USERNAME= +VALKEY_PASSWORD= VALKEY_HOST=valkey VALKEY_PORT=6379 VALKEY_DB=0 @@ -142,7 +145,7 @@ SENTRY_RELEASE=local NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT} #### Prowler release version #### -NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.16.0 +NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.26.0 # Social login credentials SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b953610fa1..3300394d83 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,14 +1,15 @@ # SDK -/* @prowler-cloud/sdk -/prowler/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation -/tests/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation -/dashboard/ @prowler-cloud/sdk -/docs/ @prowler-cloud/sdk -/examples/ @prowler-cloud/sdk -/util/ @prowler-cloud/sdk -/contrib/ @prowler-cloud/sdk -/permissions/ @prowler-cloud/sdk -/codecov.yml @prowler-cloud/sdk @prowler-cloud/api +/* @prowler-cloud/detection-remediation +/prowler/ @prowler-cloud/detection-remediation +/prowler/compliance/ @prowler-cloud/compliance +/tests/ @prowler-cloud/detection-remediation +/dashboard/ @prowler-cloud/detection-remediation +/docs/ @prowler-cloud/detection-remediation +/examples/ @prowler-cloud/detection-remediation +/util/ @prowler-cloud/detection-remediation +/contrib/ @prowler-cloud/detection-remediation +/permissions/ @prowler-cloud/detection-remediation +/codecov.yml @prowler-cloud/detection-remediation @prowler-cloud/api # API /api/ @prowler-cloud/api @@ -17,7 +18,7 @@ /ui/ @prowler-cloud/ui # AI -/mcp_server/ @prowler-cloud/ai +/mcp_server/ @prowler-cloud/detection-remediation # Platform /.github/ @prowler-cloud/platform diff --git a/.github/actions/setup-python-poetry/action.yml b/.github/actions/setup-python-poetry/action.yml index 62d01091d8..fd6c082c7b 100644 --- a/.github/actions/setup-python-poetry/action.yml +++ b/.github/actions/setup-python-poetry/action.yml @@ -13,11 +13,19 @@ inputs: poetry-version: description: 'Poetry version to install' required: false - default: '2.1.1' + default: '2.3.4' install-dependencies: description: 'Install Python dependencies with Poetry' required: false default: 'true' + update-lock: + description: 'Run `poetry lock` during setup. Only enable when a prior step mutates pyproject.toml (e.g. API `@master` VCS rewrite). Default: false.' + required: false + default: 'false' + enable-cache: + description: 'Whether to enable Poetry dependency caching via actions/setup-python' + required: false + default: 'true' runs: using: 'composite' @@ -26,10 +34,18 @@ runs: if: github.event_name == 'pull_request' && github.base_ref == 'master' && github.repository == 'prowler-cloud/prowler' shell: bash working-directory: ${{ inputs.working-directory }} + env: + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} run: | BRANCH_NAME="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - echo "Using branch: $BRANCH_NAME" - sed -i "s|\(git+https://github.com/prowler-cloud/prowler[^@]*\)@master|\1@$BRANCH_NAME|g" pyproject.toml + UPSTREAM="prowler-cloud/prowler" + if [ "$HEAD_REPO" != "$UPSTREAM" ]; then + echo "Fork PR detected (${HEAD_REPO}), rewriting VCS URL to fork" + sed -i "s|git+https://github.com/prowler-cloud/prowler\([^@]*\)@master|git+https://github.com/${HEAD_REPO}\1@$BRANCH_NAME|g" pyproject.toml + else + echo "Same-repo PR, using branch: $BRANCH_NAME" + sed -i "s|\(git+https://github.com/prowler-cloud/prowler[^@]*\)@master|\1@$BRANCH_NAME|g" pyproject.toml + fi - name: Install poetry shell: bash @@ -52,21 +68,8 @@ runs: echo "Updated resolved_reference:" grep -A2 -B2 "resolved_reference" poetry.lock - - name: Update SDK resolved_reference to latest commit (prowler repo on push) - if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler' - shell: bash - working-directory: ${{ inputs.working-directory }} - run: | - LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha') - echo "Latest commit hash: $LATEST_COMMIT" - sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / { - s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/ - }' poetry.lock - echo "Updated resolved_reference:" - grep -A2 -B2 "resolved_reference" poetry.lock - - name: Update poetry.lock (prowler repo only) - if: github.repository == 'prowler-cloud/prowler' + if: github.repository == 'prowler-cloud/prowler' && inputs.update-lock == 'true' shell: bash working-directory: ${{ inputs.working-directory }} run: poetry lock @@ -75,8 +78,10 @@ runs: uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: ${{ inputs.python-version }} - cache: 'poetry' - cache-dependency-path: ${{ inputs.working-directory }}/poetry.lock + # Disable cache when callers skip dependency install: Poetry 2.3.4 creates + # the venv in a path setup-python can't hash, breaking the post-step save-cache. + cache: ${{ inputs.enable-cache == 'true' && 'poetry' || '' }} + cache-dependency-path: ${{ inputs.enable-cache == 'true' && format('{0}/poetry.lock', inputs.working-directory) || '' }} - name: Install Python dependencies if: inputs.install-dependencies == 'true' diff --git a/.github/actions/trivy-scan/action.yml b/.github/actions/trivy-scan/action.yml index 9360dd0953..9073e2fbcb 100644 --- a/.github/actions/trivy-scan/action.yml +++ b/.github/actions/trivy-scan/action.yml @@ -117,7 +117,10 @@ runs: INPUTS_IMAGE_TAG: ${{ inputs.image-tag }} - name: Comment scan results on PR - if: inputs.create-pr-comment == 'true' && github.event_name == 'pull_request' + if: >- + inputs.create-pr-comment == 'true' + && github.event_name == 'pull_request' + && github.event.pull_request.head.repo.full_name == github.repository uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: IMAGE_NAME: ${{ inputs.image-name }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 28eff02ff6..953c0742f6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -66,6 +66,18 @@ updates: cooldown: default-days: 7 + - package-ecosystem: "pre-commit" + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 25 + target-branch: master + labels: + - "dependencies" + - "pre-commit" + cooldown: + default-days: 7 + # Dependabot Updates are temporary disabled - 2025/04/15 # v4.6 # - package-ecosystem: "pip" diff --git a/.github/labeler.yml b/.github/labeler.yml index 9a6b7262d0..49ec92e0d5 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -67,6 +67,11 @@ provider/googleworkspace: - any-glob-to-any-file: "prowler/providers/googleworkspace/**" - any-glob-to-any-file: "tests/providers/googleworkspace/**" +provider/vercel: + - changed-files: + - any-glob-to-any-file: "prowler/providers/vercel/**" + - any-glob-to-any-file: "tests/providers/vercel/**" + github_actions: - changed-files: - any-glob-to-any-file: ".github/workflows/*" @@ -102,6 +107,8 @@ mutelist: - any-glob-to-any-file: "tests/providers/openstack/lib/mutelist/**" - any-glob-to-any-file: "prowler/providers/googleworkspace/lib/mutelist/**" - any-glob-to-any-file: "tests/providers/googleworkspace/lib/mutelist/**" + - any-glob-to-any-file: "prowler/providers/vercel/lib/mutelist/**" + - any-glob-to-any-file: "tests/providers/vercel/lib/mutelist/**" integration/s3: - changed-files: diff --git a/.github/test-impact.yml b/.github/test-impact.yml index 3fab82c80b..7c290eeaa9 100644 --- a/.github/test-impact.yml +++ b/.github/test-impact.yml @@ -177,6 +177,14 @@ modules: - tests/providers/llm/** e2e: [] + - name: sdk-vercel + match: + - prowler/providers/vercel/** + - prowler/compliance/vercel/** + tests: + - tests/providers/vercel/** + e2e: [] + # ============================================ # SDK - Lib modules # ============================================ diff --git a/.github/workflows/api-bump-version.yml b/.github/workflows/api-bump-version.yml index ca6a7f7c24..6abdc8c84c 100644 --- a/.github/workflows/api-bump-version.yml +++ b/.github/workflows/api-bump-version.yml @@ -13,6 +13,8 @@ env: PROWLER_VERSION: ${{ github.event.release.tag_name }} BASE_BRANCH: master +permissions: {} + jobs: detect-release-type: runs-on: ubuntu-latest @@ -27,6 +29,11 @@ jobs: patch_version: ${{ steps.detect.outputs.patch_version }} current_api_version: ${{ steps.get_api_version.outputs.current_api_version }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -79,6 +86,11 @@ jobs: contents: read pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -204,6 +216,11 @@ jobs: contents: read pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/api-code-quality.yml b/.github/workflows/api-code-quality.yml index 73bebefd81..1724e51d70 100644 --- a/.github/workflows/api-code-quality.yml +++ b/.github/workflows/api-code-quality.yml @@ -17,6 +17,8 @@ concurrency: env: API_WORKING_DIR: ./api +permissions: {} + jobs: api-code-quality: runs-on: ubuntu-latest @@ -32,6 +34,16 @@ jobs: working-directory: ./api steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + pypi.org:443 + files.pythonhosted.org:443 + api.github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -40,7 +52,7 @@ jobs: - name: Check for API changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | api/** @@ -57,6 +69,7 @@ jobs: with: python-version: ${{ matrix.python-version }} working-directory: ./api + update-lock: 'true' - name: Poetry check if: steps.check-changes.outputs.any_changed == 'true' diff --git a/.github/workflows/api-codeql.yml b/.github/workflows/api-codeql.yml index 5e244979eb..915a5b2722 100644 --- a/.github/workflows/api-codeql.yml +++ b/.github/workflows/api-codeql.yml @@ -24,6 +24,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: api-analyze: name: CodeQL Security Analysis @@ -41,6 +43,18 @@ jobs: - 'python' steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + release-assets.githubusercontent.com:443 + uploads.github.com:443 + release-assets.githubusercontent.com:443 + objects.githubusercontent.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/api-container-build-push.yml b/.github/workflows/api-container-build-push.yml index 69edd64e64..e6403ed689 100644 --- a/.github/workflows/api-container-build-push.yml +++ b/.github/workflows/api-container-build-push.yml @@ -18,9 +18,6 @@ on: required: true type: string -permissions: - contents: read - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false @@ -36,6 +33,8 @@ env: PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-api +permissions: {} + jobs: setup: if: github.repository == 'prowler-cloud/prowler' @@ -43,7 +42,14 @@ jobs: timeout-minutes: 5 outputs: short-sha: ${{ steps.set-short-sha.outputs.short-sha }} + permissions: + contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + - name: Calculate short SHA id: set-short-sha run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT @@ -55,7 +61,14 @@ jobs: timeout-minutes: 5 outputs: message-ts: ${{ steps.slack-notification.outputs.ts }} + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -94,24 +107,50 @@ jobs: packages: write steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + _http._tcp.deb.debian.org:443 + aka.ms:443 + auth.docker.io:443 + cdn.powershellgallery.com:443 + dc.services.visualstudio.com:443 + debian.map.fastlydns.net:80 + files.pythonhosted.org:443 + github.com:443 + powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443 + production.cloudflare.docker.com:443 + pypi.org:443 + registry-1.docker.io:443 + release-assets.githubusercontent.com:443 + www.powershellgallery.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Pin prowler SDK to latest master commit + if: github.event_name == 'push' + run: | + LATEST_SHA=$(git ls-remote https://github.com/prowler-cloud/prowler.git refs/heads/master | cut -f1) + sed -i "s|prowler-cloud/prowler.git@master|prowler-cloud/prowler.git@${LATEST_SHA}|" api/pyproject.toml + - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build and push API container for ${{ matrix.arch }} id: container-push if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: ${{ env.WORKING_DIRECTORY }} push: true @@ -126,17 +165,26 @@ jobs: needs: [setup, container-build-push] if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest + permissions: + contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + release-assets.githubusercontent.com:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Create and push manifests for push event if: github.event_name == 'push' run: | @@ -178,7 +226,14 @@ jobs: needs: [setup, notify-release-started, container-build-push, create-manifest] runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -221,6 +276,13 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + - name: Trigger API deployment uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: diff --git a/.github/workflows/api-container-checks.yml b/.github/workflows/api-container-checks.yml index cdc472aaa9..5b59939db9 100644 --- a/.github/workflows/api-container-checks.yml +++ b/.github/workflows/api-container-checks.yml @@ -18,6 +18,8 @@ env: API_WORKING_DIR: ./api IMAGE_NAME: prowler-api +permissions: {} + jobs: api-dockerfile-lint: if: github.repository == 'prowler-cloud/prowler' @@ -27,6 +29,13 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -35,7 +44,7 @@ jobs: - name: Check if Dockerfile changed id: dockerfile-changed - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: api/Dockerfile @@ -65,6 +74,30 @@ jobs: pull-requests: write steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + mirror.gcr.io:443 + check.trivy.dev:443 + github.com:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + debian.map.fastlydns.net:80 + release-assets.githubusercontent.com:443 + objects.githubusercontent.com:443 + pypi.org:443 + files.pythonhosted.org:443 + www.powershellgallery.com:443 + aka.ms:443 + cdn.powershellgallery.com:443 + _http._tcp.deb.debian.org:443 + powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443 + get.trivy.dev:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -73,7 +106,7 @@ jobs: - name: Check for API changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: api/** files_ignore: | @@ -84,11 +117,11 @@ jobs: - name: Set up Docker Buildx if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build container for ${{ matrix.arch }} if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: ${{ env.API_WORKING_DIR }} push: false diff --git a/.github/workflows/api-security.yml b/.github/workflows/api-security.yml index 99cc4fc132..7b8dc72cb1 100644 --- a/.github/workflows/api-security.yml +++ b/.github/workflows/api-security.yml @@ -17,6 +17,8 @@ concurrency: env: API_WORKING_DIR: ./api +permissions: {} + jobs: api-security-scans: runs-on: ubuntu-latest @@ -32,6 +34,19 @@ jobs: working-directory: ./api steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + pypi.org:443 + files.pythonhosted.org:443 + github.com:443 + auth.safetycli.com:443 + pyup.io:443 + data.safetycli.com:443 + api.github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -40,11 +55,12 @@ jobs: - name: Check for API changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | api/** .github/workflows/api-security.yml + .safety-policy.yml files_ignore: | api/docs/** api/README.md @@ -57,6 +73,7 @@ jobs: with: python-version: ${{ matrix.python-version }} working-directory: ./api + update-lock: 'true' - name: Bandit if: steps.check-changes.outputs.any_changed == 'true' @@ -64,8 +81,8 @@ jobs: - name: Safety if: steps.check-changes.outputs.any_changed == 'true' - run: poetry run safety check --ignore 79023,79027 - # TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0 + # Accepted CVEs, severity threshold, and ignore expirations live in ../.safety-policy.yml + run: poetry run safety check --policy-file ../.safety-policy.yml - name: Vulture if: steps.check-changes.outputs.any_changed == 'true' diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index eedfc99de4..3d4e7e1799 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -22,11 +22,16 @@ env: POSTGRES_USER: prowler_user POSTGRES_PASSWORD: prowler POSTGRES_DB: postgres-db + VALKEY_SCHEME: redis + VALKEY_USERNAME: "" + VALKEY_PASSWORD: "" VALKEY_HOST: localhost VALKEY_PORT: 6379 VALKEY_DB: 0 API_WORKING_DIR: ./api +permissions: {} + jobs: api-tests: runs-on: ubuntu-latest @@ -72,6 +77,22 @@ jobs: --health-retries 5 steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + pypi.org:443 + files.pythonhosted.org:443 + cli.codecov.io:443 + keybase.io:443 + ingest.codecov.io:443 + storage.googleapis.com:443 + o26192.ingest.us.sentry.io:443 + api.github.com:443 + + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -80,7 +101,7 @@ jobs: - name: Check for API changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | api/** @@ -97,6 +118,7 @@ jobs: with: python-version: ${{ matrix.python-version }} working-directory: ./api + update-lock: 'true' - name: Run tests with pytest if: steps.check-changes.outputs.any_changed == 'true' diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 79e578ffae..ff46749641 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -17,6 +17,8 @@ env: BACKPORT_LABEL_PREFIX: backport-to- BACKPORT_LABEL_IGNORE: was-backported +permissions: {} + jobs: backport: if: github.event.pull_request.merged == true && !(contains(github.event.pull_request.labels.*.name, 'backport')) && !(contains(github.event.pull_request.labels.*.name, 'was-backported')) @@ -27,6 +29,14 @@ jobs: pull-requests: write steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - name: Check labels id: label_check uses: agilepathway/label-checker@c3d16ad512e7cea5961df85ff2486bb774caf3c5 # v1.6.65 @@ -39,7 +49,7 @@ jobs: - name: Backport PR if: steps.label_check.outputs.label_check == 'success' - uses: sorenlouv/backport-github-action@516854e7c9f962b9939085c9a92ea28411d1ae90 # v10.2.0 + uses: sorenlouv/backport-github-action@9460b7102fea25466026ce806c9ebf873ac48721 # v11.0.0 with: github_token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} auto_backport_label_prefix: ${{ env.BACKPORT_LABEL_PREFIX }} diff --git a/.github/workflows/ci-zizmor.yml b/.github/workflows/ci-zizmor.yml index 9084b41b63..a66af9684c 100644 --- a/.github/workflows/ci-zizmor.yml +++ b/.github/workflows/ci-zizmor.yml @@ -21,6 +21,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: zizmor: if: github.repository == 'prowler-cloud/prowler' @@ -33,12 +35,22 @@ jobs: actions: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + ghcr.io:443 + pkg-containers.githubusercontent.com:443 + api.github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@0dce2577a4760a2749d8cfb7a84b7d5585ebcb7d # v0.5.0 + uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 with: token: ${{ github.token }} diff --git a/.github/workflows/comment-label-update.yml b/.github/workflows/comment-label-update.yml index 9bddd6a9dd..9649d18cfc 100644 --- a/.github/workflows/comment-label-update.yml +++ b/.github/workflows/comment-label-update.yml @@ -9,6 +9,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.issue.number }} cancel-in-progress: false +permissions: {} + jobs: update-labels: if: contains(github.event.issue.labels.*.name, 'status/awaiting-response') @@ -19,6 +21,11 @@ jobs: pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Remove 'status/awaiting-response' label env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/conventional-commit.yml b/.github/workflows/conventional-commit.yml index 58e1653b74..5fc31003a4 100644 --- a/.github/workflows/conventional-commit.yml +++ b/.github/workflows/conventional-commit.yml @@ -16,6 +16,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true +permissions: {} + jobs: conventional-commit-check: runs-on: ubuntu-latest @@ -25,6 +27,11 @@ jobs: pull-requests: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Check PR title format uses: agenthunt/conventional-commit-checker-action@f1823f632e95a64547566dcd2c7da920e67117ad # v2.0.1 with: diff --git a/.github/workflows/create-backport-label.yml b/.github/workflows/create-backport-label.yml index 32bc3759fd..d7aa5709ae 100644 --- a/.github/workflows/create-backport-label.yml +++ b/.github/workflows/create-backport-label.yml @@ -13,6 +13,8 @@ env: BACKPORT_LABEL_PREFIX: backport-to- BACKPORT_LABEL_COLOR: B60205 +permissions: {} + jobs: create-label: runs-on: ubuntu-latest @@ -22,6 +24,11 @@ jobs: issues: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Create backport label for minor releases env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs-bump-version.yml b/.github/workflows/docs-bump-version.yml index ca473dc3d7..80c482b6c7 100644 --- a/.github/workflows/docs-bump-version.yml +++ b/.github/workflows/docs-bump-version.yml @@ -12,247 +12,77 @@ concurrency: env: PROWLER_VERSION: ${{ github.event.release.tag_name }} BASE_BRANCH: master + DOCS_FILE: docs/getting-started/installation/prowler-app.mdx + +permissions: {} jobs: - detect-release-type: - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - outputs: - is_minor: ${{ steps.detect.outputs.is_minor }} - is_patch: ${{ steps.detect.outputs.is_patch }} - major_version: ${{ steps.detect.outputs.major_version }} - minor_version: ${{ steps.detect.outputs.minor_version }} - patch_version: ${{ steps.detect.outputs.patch_version }} - current_docs_version: ${{ steps.get_docs_version.outputs.current_docs_version }} - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Get current documentation version - id: get_docs_version - run: | - CURRENT_DOCS_VERSION=$(grep -oP 'PROWLER_UI_VERSION="\K[^"]+' docs/getting-started/installation/prowler-app.mdx) - echo "current_docs_version=${CURRENT_DOCS_VERSION}" >> "${GITHUB_OUTPUT}" - echo "Current documentation version: $CURRENT_DOCS_VERSION" - - - name: Detect release type and parse version - id: detect - run: | - if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then - MAJOR_VERSION=${BASH_REMATCH[1]} - MINOR_VERSION=${BASH_REMATCH[2]} - PATCH_VERSION=${BASH_REMATCH[3]} - - echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}" - echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}" - echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}" - - if (( MAJOR_VERSION != 5 )); then - echo "::error::Releasing another Prowler major version, aborting..." - exit 1 - fi - - if (( PATCH_VERSION == 0 )); then - echo "is_minor=true" >> "${GITHUB_OUTPUT}" - echo "is_patch=false" >> "${GITHUB_OUTPUT}" - echo "✓ Minor release detected: $PROWLER_VERSION" - else - echo "is_minor=false" >> "${GITHUB_OUTPUT}" - echo "is_patch=true" >> "${GITHUB_OUTPUT}" - echo "✓ Patch release detected: $PROWLER_VERSION" - fi - else - echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)" - exit 1 - fi - - bump-minor-version: - needs: detect-release-type - if: needs.detect-release-type.outputs.is_minor == 'true' + bump-version: runs-on: ubuntu-latest timeout-minutes: 15 permissions: contents: read pull-requests: write steps: - - name: Checkout repository + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + + - name: Validate release version + run: | + if [[ ! $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)" + exit 1 + fi + if (( ${BASH_REMATCH[1]} != 5 )); then + echo "::error::Releasing another Prowler major version, aborting..." + exit 1 + fi + + - name: Checkout master branch uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + ref: ${{ env.BASE_BRANCH }} persist-credentials: false - - name: Calculate next minor version + - name: Read current docs version on master + id: docs_version run: | - MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION} - MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION} - CURRENT_DOCS_VERSION="${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION}" - - NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0 + CURRENT_DOCS_VERSION=$(grep -oP 'PROWLER_UI_VERSION="\K[^"]+' "${DOCS_FILE}") echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}" - echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}" + echo "Current docs version on master: $CURRENT_DOCS_VERSION" + echo "Target release version: $PROWLER_VERSION" - echo "Current documentation version: $CURRENT_DOCS_VERSION" - echo "Current release version: $PROWLER_VERSION" - echo "Next minor version: $NEXT_MINOR_VERSION" - env: - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }} - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }} - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION: ${{ needs.detect-release-type.outputs.current_docs_version }} + # Skip if master is already at or ahead of the release version + # (re-run, or patch shipped against an older minor line) + HIGHEST=$(printf '%s\n%s\n' "${CURRENT_DOCS_VERSION}" "${PROWLER_VERSION}" | sort -V | tail -n1) + if [[ "${CURRENT_DOCS_VERSION}" == "${PROWLER_VERSION}" || "${HIGHEST}" != "${PROWLER_VERSION}" ]]; then + echo "skip=true" >> "${GITHUB_OUTPUT}" + echo "Skipping bump: current ($CURRENT_DOCS_VERSION) >= release ($PROWLER_VERSION)" + else + echo "skip=false" >> "${GITHUB_OUTPUT}" + fi - - name: Bump versions in documentation for master + - name: Bump versions in documentation + if: steps.docs_version.outputs.skip == 'false' run: | set -e - - # Update prowler-app.mdx with current release version - sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx - sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx - + sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" "${DOCS_FILE}" + sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" "${DOCS_FILE}" echo "Files modified:" git --no-pager diff - name: Create PR for documentation update to master + if: steps.docs_version.outputs.skip == 'false' uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0 with: author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - base: master - commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}' - branch: docs-version-update-to-v${{ env.PROWLER_VERSION }} - title: 'docs: Update version to v${{ env.PROWLER_VERSION }}' - labels: no-changelog,skip-sync - body: | - ### Description - - Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}. - - ### Files Updated - - `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION` - - All `*.mdx` files with `` components - - ### License - - By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. - - - name: Checkout version branch - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }} - persist-credentials: false - - - name: Calculate first patch version - run: | - MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION} - MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION} - CURRENT_DOCS_VERSION="${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION}" - - FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1 - VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} - - echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}" - echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}" - echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" - - echo "First patch version: $FIRST_PATCH_VERSION" - echo "Version branch: $VERSION_BRANCH" - env: - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }} - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }} - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION: ${{ needs.detect-release-type.outputs.current_docs_version }} - - - name: Bump versions in documentation for version branch - run: | - set -e - - # Update prowler-app.mdx with current release version - sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx - sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx - - echo "Files modified:" - git --no-pager diff - - - name: Create PR for documentation update to version branch - uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0 - with: - author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - base: ${{ env.VERSION_BRANCH }} - commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}' - branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}-branch - title: 'docs: Update version to v${{ env.PROWLER_VERSION }}' - labels: no-changelog,skip-sync - body: | - ### Description - - Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}. - - ### Files Updated - - `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION` - - ### License - - By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. - - bump-patch-version: - needs: detect-release-type - if: needs.detect-release-type.outputs.is_patch == 'true' - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Calculate next patch version - run: | - MAJOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION} - MINOR_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION} - PATCH_VERSION=${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION} - CURRENT_DOCS_VERSION="${NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION}" - - NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1)) - VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} - - echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}" - echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}" - echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" - - echo "Current documentation version: $CURRENT_DOCS_VERSION" - echo "Current release version: $PROWLER_VERSION" - echo "Next patch version: $NEXT_PATCH_VERSION" - echo "Target branch: $VERSION_BRANCH" - env: - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MAJOR_VERSION: ${{ needs.detect-release-type.outputs.major_version }} - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_MINOR_VERSION: ${{ needs.detect-release-type.outputs.minor_version }} - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_PATCH_VERSION: ${{ needs.detect-release-type.outputs.patch_version }} - NEEDS_DETECT_RELEASE_TYPE_OUTPUTS_CURRENT_DOCS_VERSION: ${{ needs.detect-release-type.outputs.current_docs_version }} - - - name: Bump versions in documentation for patch version - run: | - set -e - - # Update prowler-app.mdx with current release version - sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx - sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx - - echo "Files modified:" - git --no-pager diff - - - name: Create PR for documentation update to version branch - uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0 - with: - author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - base: ${{ env.VERSION_BRANCH }} - commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}' - branch: docs-version-update-to-v${{ env.PROWLER_VERSION }} - title: 'docs: Update version to v${{ env.PROWLER_VERSION }}' + base: ${{ env.BASE_BRANCH }} + commit-message: 'chore(docs): Bump version to v${{ env.PROWLER_VERSION }}' + branch: docs-version-bump-to-v${{ env.PROWLER_VERSION }} + title: 'chore(docs): Bump version to v${{ env.PROWLER_VERSION }}' labels: no-changelog,skip-sync body: | ### Description diff --git a/.github/workflows/find-secrets.yml b/.github/workflows/find-secrets.yml index 9e1036825e..88f84d6729 100644 --- a/.github/workflows/find-secrets.yml +++ b/.github/workflows/find-secrets.yml @@ -14,6 +14,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: scan-secrets: runs-on: ubuntu-latest @@ -22,6 +24,16 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + # We can't block as Trufflehog needs to verify secrets against vendors + egress-policy: audit + # allowed-endpoints: > + # github.com:443 + # ghcr.io:443 + # pkg-containers.githubusercontent.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/helm-chart-checks.yml b/.github/workflows/helm-chart-checks.yml index b1eec36fb5..66f5954f69 100644 --- a/.github/workflows/helm-chart-checks.yml +++ b/.github/workflows/helm-chart-checks.yml @@ -21,6 +21,8 @@ concurrency: env: CHART_PATH: contrib/k8s/helm/prowler-app +permissions: {} + jobs: helm-lint: if: github.repository == 'prowler-cloud/prowler' @@ -30,13 +32,18 @@ jobs: contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Set up Helm - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 + uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 - name: Update chart dependencies run: helm dependency update ${{ env.CHART_PATH }} diff --git a/.github/workflows/helm-chart-release.yml b/.github/workflows/helm-chart-release.yml index 960e611f43..43dcf4d355 100644 --- a/.github/workflows/helm-chart-release.yml +++ b/.github/workflows/helm-chart-release.yml @@ -13,6 +13,8 @@ concurrency: env: CHART_PATH: contrib/k8s/helm/prowler-app +permissions: {} + jobs: release-helm-chart: if: github.repository == 'prowler-cloud/prowler' @@ -23,13 +25,18 @@ jobs: packages: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Set up Helm - uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 + uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 - name: Set appVersion from release tag run: | diff --git a/.github/workflows/issue-lock-on-close.yml b/.github/workflows/issue-lock-on-close.yml new file mode 100644 index 0000000000..1848585745 --- /dev/null +++ b/.github/workflows/issue-lock-on-close.yml @@ -0,0 +1,53 @@ +name: 'Tools: Lock Issue on Close' + +on: + issues: + types: + - closed + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.number }} + cancel-in-progress: false + +permissions: {} + +jobs: + lock: + if: | + github.repository == 'prowler-cloud/prowler' && + github.event.issue.locked == false + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + + - name: Comment and lock issue + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { owner, repo } = context.repo; + const issue_number = context.payload.issue.number; + + try { + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body: 'This issue is now locked as it has been closed. If you are still hitting a related problem, please open a new issue and link back to this one for context. Thanks!' + }); + } catch (error) { + core.warning(`Failed to post lock comment on issue #${issue_number}: ${error.message}`); + } + + const lockParams = { owner, repo, issue_number }; + if (context.payload.issue.state_reason === 'completed') { + lockParams.lock_reason = 'resolved'; + } + await github.rest.issues.lock(lockParams); diff --git a/.github/workflows/issue-triage.lock.yml b/.github/workflows/issue-triage.lock.yml index ae824478ca..b694e8ecef 100644 --- a/.github/workflows/issue-triage.lock.yml +++ b/.github/workflows/issue-triage.lock.yml @@ -65,6 +65,11 @@ jobs: text: ${{ steps.compute-text.outputs.text }} title: ${{ steps.compute-text.outputs.title }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Setup Scripts uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23 with: @@ -129,6 +134,11 @@ jobs: output_types: ${{ steps.collect_output.outputs.output_types }} secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Setup Scripts uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23 with: @@ -762,7 +772,7 @@ jobs: SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload Safe Outputs if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: safe-output path: ${{ env.GH_AW_SAFE_OUTPUTS }} @@ -783,13 +793,13 @@ jobs: await main(); - name: Upload sanitized agent output if: always() && env.GH_AW_AGENT_OUTPUT - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: agent-output path: ${{ env.GH_AW_AGENT_OUTPUT }} if-no-files-found: warn - name: Upload engine output files - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: agent_outputs path: | @@ -829,7 +839,7 @@ jobs: - name: Upload agent artifacts if: always() continue-on-error: true - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: agent-artifacts path: | @@ -859,13 +869,18 @@ jobs: tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} total_count: ${{ steps.missing_tool.outputs.total_count }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Setup Scripts uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23 with: destination: /opt/gh-aw/actions - name: Download agent output artifact continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent-output path: /tmp/gh-aw/safeoutputs/ @@ -966,19 +981,24 @@ jobs: outputs: success: ${{ steps.parse_results.outputs.success }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Setup Scripts uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23 with: destination: /opt/gh-aw/actions - name: Download agent artifacts continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent-artifacts path: /tmp/gh-aw/threat-detection/ - name: Download agent output artifact continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent-output path: /tmp/gh-aw/threat-detection/ @@ -1051,7 +1071,7 @@ jobs: await main(); - name: Upload threat detection log if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: threat-detection.log path: /tmp/gh-aw/threat-detection/detection.log @@ -1070,6 +1090,11 @@ jobs: outputs: activated: ${{ (steps.check_membership.outputs.is_team_member == 'true') && (steps.check_rate_limit.outputs.rate_limit_ok == 'true') }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Setup Scripts uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23 with: @@ -1138,13 +1163,18 @@ jobs: process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Setup Scripts uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23 with: destination: /opt/gh-aw/actions - name: Download agent output artifact continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent-output path: /tmp/gh-aw/safeoutputs/ diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index ac181ce350..d55e60ae88 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -15,6 +15,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true +permissions: {} + jobs: labeler: runs-on: ubuntu-latest @@ -24,6 +26,11 @@ jobs: pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Apply labels to PR uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 with: @@ -38,6 +45,11 @@ jobs: pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Check if author is org member id: check_membership env: @@ -50,7 +62,7 @@ jobs: "Alan-TheGentleman" "alejandrobailo" "amitsharm" - "andoniaf" + # "andoniaf" "cesararroba" "danibarranqueroo" "HugoPBrito" @@ -65,7 +77,8 @@ jobs: "RosaRivasProwler" "StylusFrost" "toniblyx" - "vicferpoy" + "davidm4r" + "pfe-nazaries" ) echo "Checking if $AUTHOR is a member of prowler-cloud organization" diff --git a/.github/workflows/mcp-container-build-push.yml b/.github/workflows/mcp-container-build-push.yml index bdb3f3e73f..90157db9ab 100644 --- a/.github/workflows/mcp-container-build-push.yml +++ b/.github/workflows/mcp-container-build-push.yml @@ -17,9 +17,6 @@ on: required: true type: string -permissions: - contents: read - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false @@ -35,6 +32,8 @@ env: PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-mcp +permissions: {} + jobs: setup: if: github.repository == 'prowler-cloud/prowler' @@ -42,7 +41,14 @@ jobs: timeout-minutes: 5 outputs: short-sha: ${{ steps.set-short-sha.outputs.short-sha }} + permissions: + contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + - name: Calculate short SHA id: set-short-sha run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT @@ -54,7 +60,14 @@ jobs: timeout-minutes: 5 outputs: message-ts: ${{ steps.slack-notification.outputs.ts }} + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -92,24 +105,38 @@ jobs: contents: read packages: write steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + ghcr.io:443 + pkg-containers.githubusercontent.com:443 + files.pythonhosted.org:443 + pypi.org:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build and push MCP container for ${{ matrix.arch }} id: container-push if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: ${{ env.WORKING_DIRECTORY }} push: true @@ -132,17 +159,27 @@ jobs: needs: [setup, container-build-push] if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest + permissions: + contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + github.com:443 + release-assets.githubusercontent.com:443 + - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Create and push manifests for push event if: github.event_name == 'push' run: | @@ -184,7 +221,14 @@ jobs: needs: [setup, notify-release-started, container-build-push, create-manifest] runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -227,6 +271,13 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + - name: Trigger MCP deployment uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: diff --git a/.github/workflows/mcp-container-checks.yml b/.github/workflows/mcp-container-checks.yml index a8606d2d68..b205232b0c 100644 --- a/.github/workflows/mcp-container-checks.yml +++ b/.github/workflows/mcp-container-checks.yml @@ -18,6 +18,8 @@ env: MCP_WORKING_DIR: ./mcp_server IMAGE_NAME: prowler-mcp +permissions: {} + jobs: mcp-dockerfile-lint: if: github.repository == 'prowler-cloud/prowler' @@ -27,6 +29,13 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -35,7 +44,7 @@ jobs: - name: Check if Dockerfile changed id: dockerfile-changed - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: mcp_server/Dockerfile @@ -64,6 +73,26 @@ jobs: pull-requests: write steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + ghcr.io:443 + pkg-containers.githubusercontent.com:443 + files.pythonhosted.org:443 + pypi.org:443 + api.github.com:443 + mirror.gcr.io:443 + check.trivy.dev:443 + get.trivy.dev:443 + release-assets.githubusercontent.com:443 + objects.githubusercontent.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -72,7 +101,7 @@ jobs: - name: Check for MCP changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: mcp_server/** files_ignore: | @@ -81,11 +110,11 @@ jobs: - name: Set up Docker Buildx if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build MCP container for ${{ matrix.arch }} if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: ${{ env.MCP_WORKING_DIR }} push: false diff --git a/.github/workflows/mcp-pypi-release.yml b/.github/workflows/mcp-pypi-release.yml index 04fec33a67..cccda1f664 100644 --- a/.github/workflows/mcp-pypi-release.yml +++ b/.github/workflows/mcp-pypi-release.yml @@ -14,6 +14,8 @@ env: PYTHON_VERSION: "3.12" WORKING_DIRECTORY: ./mcp_server +permissions: {} + jobs: validate-release: if: github.repository == 'prowler-cloud/prowler' @@ -26,6 +28,11 @@ jobs: major_version: ${{ steps.parse-version.outputs.major }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Parse and validate version id: parse-version run: | @@ -59,13 +66,18 @@ jobs: url: https://pypi.org/project/prowler-mcp/ steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7 + uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 with: enable-cache: false diff --git a/.github/workflows/pr-check-changelog.yml b/.github/workflows/pr-check-changelog.yml index 45027ad494..fa01ba4cbe 100644 --- a/.github/workflows/pr-check-changelog.yml +++ b/.github/workflows/pr-check-changelog.yml @@ -16,6 +16,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true +permissions: {} + jobs: check-changelog: if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false @@ -28,6 +30,14 @@ jobs: MONITORED_FOLDERS: 'api ui prowler mcp_server' steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -37,7 +47,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | api/** diff --git a/.github/workflows/pr-check-compliance-mapping.yml b/.github/workflows/pr-check-compliance-mapping.yml new file mode 100644 index 0000000000..be934d5983 --- /dev/null +++ b/.github/workflows/pr-check-compliance-mapping.yml @@ -0,0 +1,188 @@ +name: 'Tools: Check Compliance Mapping' + +on: + pull_request: + types: + - 'opened' + - 'synchronize' + - 'reopened' + - 'labeled' + - 'unlabeled' + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: {} + +jobs: + check-compliance-mapping: + if: >- + github.event.pull_request.state == 'open' && + contains(github.event.pull_request.labels.*.name, 'no-compliance-check') == false && + ( + (github.event.action != 'labeled' && github.event.action != 'unlabeled') + || github.event.label.name == 'no-compliance-check' + ) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + + steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + # zizmor: ignore[artipacked] + persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 + with: + files: | + prowler/providers/**/services/**/*.metadata.json + prowler/compliance/**/*.json + + - name: Check if new checks are mapped in compliance + id: compliance-check + run: | + ADDED_METADATA="${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" + ALL_CHANGED="${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" + + # Filter only new metadata files (new checks) + new_checks="" + for f in $ADDED_METADATA; do + case "$f" in *.metadata.json) new_checks="$new_checks $f" ;; esac + done + + if [ -z "$(echo "$new_checks" | tr -d ' ')" ]; then + echo "No new checks detected." + echo "has_new_checks=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Collect compliance files changed in this PR + changed_compliance="" + for f in $ALL_CHANGED; do + case "$f" in prowler/compliance/*.json) changed_compliance="$changed_compliance $f" ;; esac + done + + UNMAPPED="" + MAPPED="" + + for metadata_file in $new_checks; do + check_dir=$(dirname "$metadata_file") + check_id=$(basename "$check_dir") + provider=$(echo "$metadata_file" | cut -d'/' -f3) + + # Read CheckID from the metadata JSON for accuracy + if [ -f "$metadata_file" ]; then + json_check_id=$(python3 -c "import json; print(json.load(open('$metadata_file')).get('CheckID', ''))" 2>/dev/null || echo "") + if [ -n "$json_check_id" ]; then + check_id="$json_check_id" + fi + fi + + # Search for the check ID in compliance files changed in this PR + found_in="" + for comp_file in $changed_compliance; do + if grep -q "\"${check_id}\"" "$comp_file" 2>/dev/null; then + found_in="${found_in}$(basename "$comp_file" .json), " + fi + done + + if [ -n "$found_in" ]; then + found_in=$(echo "$found_in" | sed 's/, $//') + MAPPED="${MAPPED}- \`${check_id}\` (\`${provider}\`): ${found_in}"$'\n' + else + UNMAPPED="${UNMAPPED}- \`${check_id}\` (\`${provider}\`)"$'\n' + fi + done + + echo "has_new_checks=true" >> "$GITHUB_OUTPUT" + + if [ -n "$UNMAPPED" ]; then + echo "has_unmapped=true" >> "$GITHUB_OUTPUT" + else + echo "has_unmapped=false" >> "$GITHUB_OUTPUT" + fi + + { + echo "unmapped<> "$GITHUB_OUTPUT" + + { + echo "mapped<> "$GITHUB_OUTPUT" + env: + STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES: ${{ steps.changed-files.outputs.added_files }} + STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + + - name: Manage compliance review label + if: steps.compliance-check.outputs.has_new_checks == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HAS_UNMAPPED: ${{ steps.compliance-check.outputs.has_unmapped }} + run: | + LABEL_NAME="needs-compliance-review" + + if [ "$HAS_UNMAPPED" = "true" ]; then + echo "Adding compliance review label to PR #${PR_NUMBER}..." + gh pr edit "$PR_NUMBER" --add-label "$LABEL_NAME" --repo "${{ github.repository }}" || true + else + echo "Removing compliance review label from PR #${PR_NUMBER}..." + gh pr edit "$PR_NUMBER" --remove-label "$LABEL_NAME" --repo "${{ github.repository }}" || true + fi + + - name: Find existing compliance comment + if: steps.compliance-check.outputs.has_new_checks == 'true' && github.event.pull_request.head.repo.full_name == github.repository + id: find-comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: '' + + - name: Create or update compliance comment + if: steps.compliance-check.outputs.has_new_checks == 'true' && github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-id: ${{ steps.find-comment.outputs.comment-id }} + edit-mode: replace + body: | + + ## Compliance Mapping Review + + This PR adds new checks. Please verify that they have been mapped to the relevant compliance framework requirements. + + ${{ steps.compliance-check.outputs.unmapped != '' && format('### New checks not mapped to any compliance framework in this PR + + {0} + + > Please review whether these checks should be added to compliance framework requirements in `prowler/compliance//`. Each compliance JSON has a `Checks` array inside each requirement — add the check ID there if it satisfies that requirement.', steps.compliance-check.outputs.unmapped) || '' }} + + ${{ steps.compliance-check.outputs.mapped != '' && format('### New checks already mapped in this PR + + {0}', steps.compliance-check.outputs.mapped) || '' }} + + Use the `no-compliance-check` label to skip this check. diff --git a/.github/workflows/pr-conflict-checker.yml b/.github/workflows/pr-conflict-checker.yml index 068ebd286d..667f807be8 100644 --- a/.github/workflows/pr-conflict-checker.yml +++ b/.github/workflows/pr-conflict-checker.yml @@ -15,6 +15,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true +permissions: {} + jobs: check-conflicts: runs-on: ubuntu-latest @@ -25,6 +27,11 @@ jobs: issues: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout PR head uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -34,7 +41,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: '**' diff --git a/.github/workflows/pr-merged.yml b/.github/workflows/pr-merged.yml index 41856b0ede..4a98ac70c4 100644 --- a/.github/workflows/pr-merged.yml +++ b/.github/workflows/pr-merged.yml @@ -12,6 +12,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: false +permissions: {} + jobs: trigger-cloud-pull-request: if: | @@ -23,6 +25,13 @@ jobs: permissions: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + - name: Calculate short commit SHA id: vars run: | diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index e1d658b7be..07caea7255 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -17,6 +17,8 @@ concurrency: env: PROWLER_VERSION: ${{ inputs.prowler_version }} +permissions: {} + jobs: prepare-release: if: github.event_name == 'workflow_dispatch' && github.repository == 'prowler-cloud/prowler' @@ -26,6 +28,11 @@ jobs: contents: write pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -33,15 +40,12 @@ jobs: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - name: Setup Python with Poetry + uses: ./.github/actions/setup-python-poetry with: python-version: '3.12' - - - name: Install Poetry - run: | - python3 -m pip install --user poetry==2.1.1 - echo "$HOME/.local/bin" >> $GITHUB_PATH + install-dependencies: 'false' + enable-cache: 'false' - name: Configure Git run: | @@ -375,7 +379,7 @@ jobs: no-changelog - name: Create draft release - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 + uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 with: tag_name: ${{ env.PROWLER_VERSION }} name: Prowler ${{ env.PROWLER_VERSION }} diff --git a/.github/workflows/sdk-bump-version.yml b/.github/workflows/sdk-bump-version.yml index 0e4c5e33fc..c3f018c0d3 100644 --- a/.github/workflows/sdk-bump-version.yml +++ b/.github/workflows/sdk-bump-version.yml @@ -13,6 +13,8 @@ env: PROWLER_VERSION: ${{ github.event.release.tag_name }} BASE_BRANCH: master +permissions: {} + jobs: detect-release-type: runs-on: ubuntu-latest @@ -26,6 +28,11 @@ jobs: minor_version: ${{ steps.detect.outputs.minor_version }} patch_version: ${{ steps.detect.outputs.patch_version }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Detect release type and parse version id: detect run: | @@ -66,6 +73,11 @@ jobs: contents: read pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -101,9 +113,9 @@ jobs: author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} base: master - commit-message: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}' - branch: version-bump-to-v${{ env.NEXT_MINOR_VERSION }} - title: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}' + commit-message: 'chore(sdk): Bump version to v${{ env.NEXT_MINOR_VERSION }}' + branch: sdk-version-bump-to-v${{ env.NEXT_MINOR_VERSION }} + title: 'chore(sdk): Bump version to v${{ env.NEXT_MINOR_VERSION }}' labels: no-changelog,skip-sync body: | ### Description @@ -153,9 +165,9 @@ jobs: author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} base: ${{ env.VERSION_BRANCH }} - commit-message: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}' - branch: version-bump-to-v${{ env.FIRST_PATCH_VERSION }} - title: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}' + commit-message: 'chore(sdk): Bump version to v${{ env.FIRST_PATCH_VERSION }}' + branch: sdk-version-bump-to-v${{ env.FIRST_PATCH_VERSION }} + title: 'chore(sdk): Bump version to v${{ env.FIRST_PATCH_VERSION }}' labels: no-changelog,skip-sync body: | ### Description @@ -175,6 +187,11 @@ jobs: contents: read pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -216,9 +233,9 @@ jobs: author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} base: ${{ env.VERSION_BRANCH }} - commit-message: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}' - branch: version-bump-to-v${{ env.NEXT_PATCH_VERSION }} - title: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}' + commit-message: 'chore(sdk): Bump version to v${{ env.NEXT_PATCH_VERSION }}' + branch: sdk-version-bump-to-v${{ env.NEXT_PATCH_VERSION }} + title: 'chore(sdk): Bump version to v${{ env.NEXT_PATCH_VERSION }}' labels: no-changelog,skip-sync body: | ### Description diff --git a/.github/workflows/sdk-check-duplicate-test-names.yml b/.github/workflows/sdk-check-duplicate-test-names.yml index 9fbe3fd917..17c595ca11 100644 --- a/.github/workflows/sdk-check-duplicate-test-names.yml +++ b/.github/workflows/sdk-check-duplicate-test-names.yml @@ -10,6 +10,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: check-duplicate-test-names: if: github.repository == 'prowler-cloud/prowler' @@ -19,6 +21,13 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/sdk-code-quality.yml b/.github/workflows/sdk-code-quality.yml index 2d785c6094..b777ee1657 100644 --- a/.github/workflows/sdk-code-quality.yml +++ b/.github/workflows/sdk-code-quality.yml @@ -14,6 +14,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: sdk-code-quality: if: github.repository == 'prowler-cloud/prowler' @@ -24,12 +26,20 @@ jobs: strategy: matrix: python-version: - - '3.9' - '3.10' - '3.11' - '3.12' steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + pypi.org:443 + files.pythonhosted.org:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -38,7 +48,7 @@ jobs: - name: Check for SDK changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: ./** files_ignore: | @@ -61,22 +71,11 @@ jobs: contrib/** **/AGENTS.md - - name: Install Poetry + - name: Setup Python with Poetry if: steps.check-changes.outputs.any_changed == 'true' - run: pipx install poetry==2.1.1 - - - name: Set up Python ${{ matrix.python-version }} - if: steps.check-changes.outputs.any_changed == 'true' - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: ./.github/actions/setup-python-poetry with: python-version: ${{ matrix.python-version }} - cache: 'poetry' - - - name: Install dependencies - if: steps.check-changes.outputs.any_changed == 'true' - run: | - poetry install --no-root - poetry run pip list - name: Check Poetry lock file if: steps.check-changes.outputs.any_changed == 'true' diff --git a/.github/workflows/sdk-codeql.yml b/.github/workflows/sdk-codeql.yml index e35776d9b6..0985ec1228 100644 --- a/.github/workflows/sdk-codeql.yml +++ b/.github/workflows/sdk-codeql.yml @@ -30,6 +30,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: sdk-analyze: if: github.repository == 'prowler-cloud/prowler' @@ -48,6 +50,16 @@ jobs: - 'python' steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + release-assets.githubusercontent.com:443 + uploads.github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/sdk-container-build-push.yml b/.github/workflows/sdk-container-build-push.yml index 8f21b00479..8a2bba691c 100644 --- a/.github/workflows/sdk-container-build-push.yml +++ b/.github/workflows/sdk-container-build-push.yml @@ -23,9 +23,6 @@ on: required: true type: string -permissions: - contents: read - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false @@ -45,10 +42,13 @@ env: # Container registries PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud PROWLERCLOUD_DOCKERHUB_IMAGE: prowler + TONIBLYX_DOCKERHUB_REPOSITORY: toniblyx # AWS configuration (for ECR) AWS_REGION: us-east-1 +permissions: {} + jobs: setup: if: github.repository == 'prowler-cloud/prowler' @@ -59,21 +59,32 @@ jobs: prowler_version_major: ${{ steps.get-prowler-version.outputs.prowler_version_major }} latest_tag: ${{ steps.get-prowler-version.outputs.latest_tag }} stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }} + permissions: + contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + pypi.org:443 + files.pythonhosted.org:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - name: Setup Python with Poetry + uses: ./.github/actions/setup-python-poetry with: python-version: ${{ env.PYTHON_VERSION }} + install-dependencies: 'false' + enable-cache: 'false' - - name: Install Poetry - run: | - pipx install poetry==2.1.1 - pipx inject poetry poetry-bumpversion + - name: Inject poetry-bumpversion plugin + run: pipx inject poetry poetry-bumpversion - name: Get Prowler version and set tags id: get-prowler-version @@ -115,7 +126,14 @@ jobs: timeout-minutes: 5 outputs: message-ts: ${{ steps.slack-notification.outputs.ts }} + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -154,19 +172,40 @@ jobs: packages: write steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.ecr-public.us-east-1.amazonaws.com:443 + public.ecr.aws:443 + registry-1.docker.io:443 + production.cloudflare.docker.com:443 + auth.docker.io:443 + debian.map.fastlydns.net:80 + github.com:443 + release-assets.githubusercontent.com:443 + pypi.org:443 + files.pythonhosted.org:443 + www.powershellgallery.com:443 + aka.ms:443 + cdn.powershellgallery.com:443 + _http._tcp.deb.debian.org:443 + powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Public ECR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: public.ecr.aws username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }} @@ -175,12 +214,12 @@ jobs: AWS_REGION: ${{ env.AWS_REGION }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build and push SDK container for ${{ matrix.arch }} id: container-push if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: . file: ${{ env.DOCKERFILE_PATH }} @@ -196,16 +235,32 @@ jobs: needs: [setup, container-build-push] if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest + permissions: + contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + registry-1.docker.io:443 + auth.docker.io:443 + public.ecr.aws:443 + production.cloudflare.docker.com:443 + github.com:443 + release-assets.githubusercontent.com:443 + api.ecr-public.us-east-1.amazonaws.com:443 + + - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Public ECR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: public.ecr.aws username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }} @@ -213,15 +268,11 @@ jobs: env: AWS_REGION: ${{ env.AWS_REGION }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Create and push manifests for push event if: github.event_name == 'push' run: | docker buildx imagetools create \ -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_LATEST_TAG} \ - -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_LATEST_TAG} \ -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_LATEST_TAG} \ ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_LATEST_TAG}-amd64 \ ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_LATEST_TAG}-arm64 @@ -232,12 +283,10 @@ jobs: if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' run: | docker buildx imagetools create \ - -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${NEEDS_SETUP_OUTPUTS_PROWLER_VERSION} \ - -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${NEEDS_SETUP_OUTPUTS_STABLE_TAG} \ - -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${NEEDS_SETUP_OUTPUTS_PROWLER_VERSION} \ - -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${NEEDS_SETUP_OUTPUTS_STABLE_TAG} \ -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_PROWLER_VERSION} \ -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_STABLE_TAG} \ + -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${NEEDS_SETUP_OUTPUTS_PROWLER_VERSION} \ + -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${NEEDS_SETUP_OUTPUTS_STABLE_TAG} \ ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_LATEST_TAG}-amd64 \ ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_LATEST_TAG}-arm64 env: @@ -245,6 +294,39 @@ jobs: NEEDS_SETUP_OUTPUTS_STABLE_TAG: ${{ needs.setup.outputs.stable_tag }} NEEDS_SETUP_OUTPUTS_LATEST_TAG: ${{ needs.setup.outputs.latest_tag }} + # Push to toniblyx/prowler only for current version (latest/stable/release tags) + - name: Login to DockerHub (toniblyx) + if: needs.setup.outputs.latest_tag == 'latest' + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + with: + username: ${{ secrets.TONIBLYX_DOCKERHUB_USERNAME }} + password: ${{ secrets.TONIBLYX_DOCKERHUB_PASSWORD }} + + - name: Push manifests to toniblyx for push event + if: needs.setup.outputs.latest_tag == 'latest' && github.event_name == 'push' + run: | + docker buildx imagetools create \ + -t ${{ env.TONIBLYX_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:latest \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:latest + + - name: Push manifests to toniblyx for release event + if: needs.setup.outputs.latest_tag == 'latest' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') + run: | + docker buildx imagetools create \ + -t ${{ env.TONIBLYX_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${NEEDS_SETUP_OUTPUTS_PROWLER_VERSION} \ + -t ${{ env.TONIBLYX_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:stable \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:stable + env: + NEEDS_SETUP_OUTPUTS_PROWLER_VERSION: ${{ needs.setup.outputs.prowler_version }} + + # Re-login as prowlercloud for cleanup of intermediate tags + - name: Login to DockerHub (prowlercloud) + if: always() + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Install regctl if: always() uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main @@ -264,7 +346,14 @@ jobs: needs: [setup, notify-release-started, container-build-push, create-manifest] runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -307,6 +396,11 @@ jobs: contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Calculate short SHA id: short-sha run: echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/sdk-container-checks.yml b/.github/workflows/sdk-container-checks.yml index 3200ff89c5..3474df676e 100644 --- a/.github/workflows/sdk-container-checks.yml +++ b/.github/workflows/sdk-container-checks.yml @@ -17,6 +17,8 @@ concurrency: env: IMAGE_NAME: prowler +permissions: {} + jobs: sdk-dockerfile-lint: if: github.repository == 'prowler-cloud/prowler' @@ -26,6 +28,13 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -34,7 +43,7 @@ jobs: - name: Check if Dockerfile changed id: dockerfile-changed - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: Dockerfile @@ -64,6 +73,30 @@ jobs: pull-requests: write steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + api.github.com:443 + mirror.gcr.io:443 + check.trivy.dev:443 + debian.map.fastlydns.net:80 + release-assets.githubusercontent.com:443 + objects.githubusercontent.com:443 + pypi.org:443 + files.pythonhosted.org:443 + www.powershellgallery.com:443 + aka.ms:443 + cdn.powershellgallery.com:443 + _http._tcp.deb.debian.org:443 + powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443 + get.trivy.dev:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -72,7 +105,7 @@ jobs: - name: Check for SDK changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: ./** files_ignore: | @@ -97,11 +130,11 @@ jobs: - name: Set up Docker Buildx if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build SDK container for ${{ matrix.arch }} if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: . push: false diff --git a/.github/workflows/sdk-pypi-release.yml b/.github/workflows/sdk-pypi-release.yml index 27196b147a..7916431dec 100644 --- a/.github/workflows/sdk-pypi-release.yml +++ b/.github/workflows/sdk-pypi-release.yml @@ -13,6 +13,8 @@ env: RELEASE_TAG: ${{ github.event.release.tag_name }} PYTHON_VERSION: '3.12' +permissions: {} + jobs: validate-release: if: github.repository == 'prowler-cloud/prowler' @@ -25,6 +27,11 @@ jobs: major_version: ${{ steps.parse-version.outputs.major }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Parse and validate version id: parse-version run: | @@ -58,18 +65,22 @@ jobs: url: https://pypi.org/project/prowler/${{ needs.validate-release.outputs.prowler_version }}/ steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: Install Poetry - run: pipx install poetry==2.1.1 - - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - name: Setup Python with Poetry + uses: ./.github/actions/setup-python-poetry with: python-version: ${{ env.PYTHON_VERSION }} + install-dependencies: 'false' + enable-cache: 'false' - name: Build Prowler package run: poetry build @@ -91,18 +102,22 @@ jobs: url: https://pypi.org/project/prowler-cloud/${{ needs.validate-release.outputs.prowler_version }}/ steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: Install Poetry - run: pipx install poetry==2.1.1 - - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - name: Setup Python with Poetry + uses: ./.github/actions/setup-python-poetry with: python-version: ${{ env.PYTHON_VERSION }} + install-dependencies: 'false' + enable-cache: 'false' - name: Install toml package run: pip install toml diff --git a/.github/workflows/sdk-refresh-aws-services-regions.yml b/.github/workflows/sdk-refresh-aws-services-regions.yml index 0ad26b0e0f..3ece09e8a5 100644 --- a/.github/workflows/sdk-refresh-aws-services-regions.yml +++ b/.github/workflows/sdk-refresh-aws-services-regions.yml @@ -13,6 +13,8 @@ env: PYTHON_VERSION: '3.12' AWS_REGION: 'us-east-1' +permissions: {} + jobs: refresh-aws-regions: if: github.repository == 'prowler-cloud/prowler' @@ -24,6 +26,11 @@ jobs: contents: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/sdk-refresh-oci-regions.yml b/.github/workflows/sdk-refresh-oci-regions.yml index e9a3665911..958d5beb80 100644 --- a/.github/workflows/sdk-refresh-oci-regions.yml +++ b/.github/workflows/sdk-refresh-oci-regions.yml @@ -12,6 +12,8 @@ concurrency: env: PYTHON_VERSION: '3.12' +permissions: {} + jobs: refresh-oci-regions: if: github.repository == 'prowler-cloud/prowler' @@ -22,6 +24,11 @@ jobs: contents: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -72,12 +79,13 @@ jobs: This PR updates the `OCI_COMMERCIAL_REGIONS` dictionary in `prowler/providers/oraclecloud/config.py` with the latest regions fetched from the OCI Identity API (`list_regions()`). - Government regions (`OCI_GOVERNMENT_REGIONS`) are preserved unchanged + - DOD regions (`OCI_US_DOD_REGIONS`) are preserved unchanged - Region display names are mapped from Oracle's official documentation ### Checklist - [x] This is an automated update from OCI official sources - - [x] Government regions (us-langley-1, us-luke-1) preserved + - [x] Government regions (us-langley-1, us-luke-1) and DOD regions (us-gov-ashburn-1, us-gov-phoenix-1, us-gov-chicago-1) are preserved - [x] No manual review of region data required ### License diff --git a/.github/workflows/sdk-security.yml b/.github/workflows/sdk-security.yml index d9488b0e4c..ceb6b1db1c 100644 --- a/.github/workflows/sdk-security.yml +++ b/.github/workflows/sdk-security.yml @@ -14,6 +14,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: sdk-security-scans: if: github.repository == 'prowler-cloud/prowler' @@ -23,6 +25,19 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + pypi.org:443 + files.pythonhosted.org:443 + github.com:443 + auth.safetycli.com:443 + pyup.io:443 + data.safetycli.com:443 + api.github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -31,7 +46,7 @@ jobs: - name: Check for SDK changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: ./** @@ -56,20 +71,11 @@ jobs: contrib/** **/AGENTS.md - - name: Install Poetry + - name: Setup Python with Poetry if: steps.check-changes.outputs.any_changed == 'true' - run: pipx install poetry==2.1.1 - - - name: Set up Python 3.12 - if: steps.check-changes.outputs.any_changed == 'true' - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: ./.github/actions/setup-python-poetry with: python-version: '3.12' - cache: 'poetry' - - - name: Install dependencies - if: steps.check-changes.outputs.any_changed == 'true' - run: poetry install --no-root - name: Security scan with Bandit if: steps.check-changes.outputs.any_changed == 'true' @@ -77,7 +83,8 @@ jobs: - name: Security scan with Safety if: steps.check-changes.outputs.any_changed == 'true' - run: poetry run safety check -r pyproject.toml + # Accepted CVEs, severity threshold, and ignore expirations live in .safety-policy.yml + run: poetry run safety check -r pyproject.toml --policy-file .safety-policy.yml - name: Dead code detection with Vulture if: steps.check-changes.outputs.any_changed == 'true' diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index 6f94022a99..0646e5f8b5 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -14,6 +14,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: sdk-tests: if: github.repository == 'prowler-cloud/prowler' @@ -24,12 +26,41 @@ jobs: strategy: matrix: python-version: - - '3.9' - '3.10' - '3.11' - '3.12' steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + pypi.org:443 + files.pythonhosted.org:443 + api.github.com:443 + release-assets.githubusercontent.com:443 + *.amazonaws.com:443 + *.googleapis.com:443 + schema.ocsf.io:443 + registry-1.docker.io:443 + production.cloudflare.docker.com:443 + powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443 + o26192.ingest.us.sentry.io:443 + management.azure.com:443 + login.microsoftonline.com:443 + keybase.io:443 + ingest.codecov.io:443 + graph.microsoft.com:443 + dc.services.visualstudio.com:443 + cloud.mongodb.com:443 + cli.codecov.io:443 + auth.docker.io:443 + api.vercel.com:443 + api.atlassian.com:443 + aka.ms:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -38,7 +69,7 @@ jobs: - name: Check for SDK changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: ./** files_ignore: | @@ -61,26 +92,17 @@ jobs: contrib/** **/AGENTS.md - - name: Install Poetry + - name: Setup Python with Poetry if: steps.check-changes.outputs.any_changed == 'true' - run: pipx install poetry==2.1.1 - - - name: Set up Python ${{ matrix.python-version }} - if: steps.check-changes.outputs.any_changed == 'true' - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: ./.github/actions/setup-python-poetry with: python-version: ${{ matrix.python-version }} - cache: 'poetry' - - - name: Install dependencies - if: steps.check-changes.outputs.any_changed == 'true' - run: poetry install --no-root # AWS Provider - name: Check if AWS files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-aws - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/aws/** @@ -210,7 +232,7 @@ jobs: - name: Check if Azure files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-azure - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/azure/** @@ -234,7 +256,7 @@ jobs: - name: Check if GCP files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-gcp - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/gcp/** @@ -258,7 +280,7 @@ jobs: - name: Check if Kubernetes files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-kubernetes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/kubernetes/** @@ -282,7 +304,7 @@ jobs: - name: Check if GitHub files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-github - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/github/** @@ -306,7 +328,7 @@ jobs: - name: Check if NHN files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-nhn - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/nhn/** @@ -330,7 +352,7 @@ jobs: - name: Check if M365 files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-m365 - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/m365/** @@ -354,7 +376,7 @@ jobs: - name: Check if IaC files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-iac - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/iac/** @@ -378,7 +400,7 @@ jobs: - name: Check if MongoDB Atlas files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-mongodbatlas - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/mongodbatlas/** @@ -402,7 +424,7 @@ jobs: - name: Check if OCI files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-oraclecloud - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/oraclecloud/** @@ -426,7 +448,7 @@ jobs: - name: Check if OpenStack files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-openstack - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/openstack/** @@ -450,7 +472,7 @@ jobs: - name: Check if Google Workspace files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-googleworkspace - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/**/googleworkspace/** @@ -470,11 +492,35 @@ jobs: flags: prowler-py${{ matrix.python-version }}-googleworkspace files: ./googleworkspace_coverage.xml + # Vercel Provider + - name: Check if Vercel files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-vercel + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 + with: + files: | + ./prowler/**/vercel/** + ./tests/**/vercel/** + ./poetry.lock + + - name: Run Vercel tests + if: steps.changed-vercel.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/vercel --cov-report=xml:vercel_coverage.xml tests/providers/vercel + + - name: Upload Vercel coverage to Codecov + if: steps.changed-vercel.outputs.any_changed == 'true' + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-vercel + files: ./vercel_coverage.xml + # Lib - name: Check if Lib files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-lib - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/lib/** @@ -498,7 +544,7 @@ jobs: - name: Check if Config files changed if: steps.check-changes.outputs.any_changed == 'true' id: changed-config - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ./prowler/config/** diff --git a/.github/workflows/test-impact-analysis.yml b/.github/workflows/test-impact-analysis.yml index c8cd8bb651..88500551f2 100644 --- a/.github/workflows/test-impact-analysis.yml +++ b/.github/workflows/test-impact-analysis.yml @@ -31,6 +31,8 @@ on: description: "Whether there are UI E2E tests to run" value: ${{ jobs.analyze.outputs.has-ui-e2e }} +permissions: {} + jobs: analyze: runs-on: ubuntu-latest @@ -45,8 +47,19 @@ jobs: has-sdk-tests: ${{ steps.set-flags.outputs.has-sdk-tests }} has-api-tests: ${{ steps.set-flags.outputs.has-api-tests }} has-ui-e2e: ${{ steps.set-flags.outputs.has-ui-e2e }} + permissions: + contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + pypi.org:443 + files.pythonhosted.org:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -55,7 +68,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/ui-bump-version.yml b/.github/workflows/ui-bump-version.yml index aaa32b8f0b..273778e1c2 100644 --- a/.github/workflows/ui-bump-version.yml +++ b/.github/workflows/ui-bump-version.yml @@ -13,6 +13,8 @@ env: PROWLER_VERSION: ${{ github.event.release.tag_name }} BASE_BRANCH: master +permissions: {} + jobs: detect-release-type: runs-on: ubuntu-latest @@ -26,6 +28,11 @@ jobs: minor_version: ${{ steps.detect.outputs.minor_version }} patch_version: ${{ steps.detect.outputs.patch_version }} steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Detect release type and parse version id: detect run: | @@ -66,6 +73,11 @@ jobs: contents: read pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -89,7 +101,7 @@ jobs: run: | set -e - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=.*|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env echo "Files modified:" git --no-pager diff @@ -143,7 +155,7 @@ jobs: run: | set -e - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=.*|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env echo "Files modified:" git --no-pager diff @@ -179,6 +191,11 @@ jobs: contents: read pull-requests: write steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -208,7 +225,7 @@ jobs: run: | set -e - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=.*|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env echo "Files modified:" git --no-pager diff diff --git a/.github/workflows/ui-codeql.yml b/.github/workflows/ui-codeql.yml index 7d8755e27c..c8b65e77d3 100644 --- a/.github/workflows/ui-codeql.yml +++ b/.github/workflows/ui-codeql.yml @@ -26,6 +26,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: ui-analyze: if: github.repository == 'prowler-cloud/prowler' @@ -44,6 +46,16 @@ jobs: - 'javascript-typescript' steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + release-assets.githubusercontent.com:443 + uploads.github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/ui-container-build-push.yml b/.github/workflows/ui-container-build-push.yml index e3b6589adf..6fab2d1efb 100644 --- a/.github/workflows/ui-container-build-push.yml +++ b/.github/workflows/ui-container-build-push.yml @@ -17,9 +17,6 @@ on: required: true type: string -permissions: - contents: read - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false @@ -38,6 +35,8 @@ env: # Build args NEXT_PUBLIC_API_BASE_URL: http://prowler-api:8080/api/v1 +permissions: {} + jobs: setup: if: github.repository == 'prowler-cloud/prowler' @@ -45,7 +44,14 @@ jobs: timeout-minutes: 5 outputs: short-sha: ${{ steps.set-short-sha.outputs.short-sha }} + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Calculate short SHA id: set-short-sha run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT @@ -57,7 +63,14 @@ jobs: timeout-minutes: 5 outputs: message-ts: ${{ steps.slack-notification.outputs.ts }} + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -96,24 +109,38 @@ jobs: packages: write steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + registry-1.docker.io:443 + production.cloudflare.docker.com:443 + auth.docker.io:443 + registry.npmjs.org:443 + dl-cdn.alpinelinux.org:443 + fonts.googleapis.com:443 + fonts.gstatic.com:443 + github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build and push UI container for ${{ matrix.arch }} id: container-push if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: ${{ env.WORKING_DIRECTORY }} build-args: | @@ -131,17 +158,27 @@ jobs: needs: [setup, container-build-push] if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success' runs-on: ubuntu-latest + permissions: + contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + release-assets.githubusercontent.com:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Create and push manifests for push event if: github.event_name == 'push' run: | @@ -183,7 +220,14 @@ jobs: needs: [setup, notify-release-started, container-build-push, create-manifest] runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -226,6 +270,13 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + - name: Trigger UI deployment uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 with: diff --git a/.github/workflows/ui-container-checks.yml b/.github/workflows/ui-container-checks.yml index c3b45bc62e..eb7b508b58 100644 --- a/.github/workflows/ui-container-checks.yml +++ b/.github/workflows/ui-container-checks.yml @@ -18,6 +18,8 @@ env: UI_WORKING_DIR: ./ui IMAGE_NAME: prowler-ui +permissions: {} + jobs: ui-dockerfile-lint: if: github.repository == 'prowler-cloud/prowler' @@ -27,6 +29,13 @@ jobs: contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -35,7 +44,7 @@ jobs: - name: Check if Dockerfile changed id: dockerfile-changed - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: ui/Dockerfile @@ -65,6 +74,26 @@ jobs: pull-requests: write steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + registry.npmjs.org:443 + dl-cdn.alpinelinux.org:443 + fonts.googleapis.com:443 + fonts.gstatic.com:443 + api.github.com:443 + mirror.gcr.io:443 + check.trivy.dev:443 + get.trivy.dev:443 + release-assets.githubusercontent.com:443 + objects.githubusercontent.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -73,7 +102,7 @@ jobs: - name: Check for UI changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: ui/** files_ignore: | @@ -83,11 +112,11 @@ jobs: - name: Set up Docker Buildx if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build UI container for ${{ matrix.arch }} if: steps.check-changes.outputs.any_changed == 'true' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: ${{ env.UI_WORKING_DIR }} target: prod diff --git a/.github/workflows/ui-e2e-tests-v2.yml b/.github/workflows/ui-e2e-tests-v2.yml index 08b1063903..2a91e460be 100644 --- a/.github/workflows/ui-e2e-tests-v2.yml +++ b/.github/workflows/ui-e2e-tests-v2.yml @@ -15,13 +15,14 @@ on: - 'ui/**' - 'api/**' # API changes can affect UI E2E -permissions: - contents: read +permissions: {} jobs: # First, analyze which tests need to run impact-analysis: if: github.repository == 'prowler-cloud/prowler' + permissions: + contents: read uses: ./.github/workflows/test-impact-analysis.yml # Run E2E tests based on impact analysis @@ -75,8 +76,15 @@ jobs: # Pass E2E paths from impact analysis E2E_TEST_PATHS: ${{ needs.impact-analysis.outputs.ui-e2e }} RUN_ALL_TESTS: ${{ needs.impact-analysis.outputs.run-all }} + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -152,21 +160,21 @@ jobs: ' - name: Setup Node.js - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: '24.13.0' - name: Setup pnpm - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10 + package_json_file: ui/package.json run_install: false - name: Get pnpm store directory run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - name: Setup pnpm and Next.js cache - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: | ${{ env.STORE_PATH }} @@ -186,7 +194,7 @@ jobs: run: pnpm run build - name: Cache Playwright browsers - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 id: playwright-cache with: path: ~/.cache/ms-playwright @@ -253,7 +261,7 @@ jobs: fi - name: Upload test reports - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: failure() with: name: playwright-report @@ -273,7 +281,14 @@ jobs: needs.impact-analysis.outputs.has-ui-e2e != 'true' && needs.impact-analysis.outputs.run-all != 'true' runs-on: ubuntu-latest + permissions: + contents: read steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: No E2E tests needed run: | echo "## E2E Tests Skipped" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 838b125249..c11e90b8f4 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -18,6 +18,8 @@ env: UI_WORKING_DIR: ./ui NODE_VERSION: '24.13.0' +permissions: {} + jobs: ui-tests: runs-on: ubuntu-latest @@ -29,6 +31,18 @@ jobs: working-directory: ./ui steps: + - name: Harden Runner + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + registry.npmjs.org:443 + fonts.googleapis.com:443 + fonts.gstatic.com:443 + api.github.com:443 + release-assets.githubusercontent.com:443 + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -37,7 +51,7 @@ jobs: - name: Check for UI changes id: check-changes - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ui/** @@ -50,7 +64,7 @@ jobs: - name: Get changed source files for targeted tests id: changed-source if: steps.check-changes.outputs.any_changed == 'true' - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ui/**/*.ts @@ -66,7 +80,7 @@ jobs: - name: Check for critical path changes (run all tests) id: critical-changes if: steps.check-changes.outputs.any_changed == 'true' - uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5 with: files: | ui/lib/** @@ -78,15 +92,15 @@ jobs: - name: Setup Node.js ${{ env.NODE_VERSION }} if: steps.check-changes.outputs.any_changed == 'true' - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: ${{ env.NODE_VERSION }} - name: Setup pnpm if: steps.check-changes.outputs.any_changed == 'true' - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: - version: 10 + package_json_file: ui/package.json run_install: false - name: Get pnpm store directory @@ -96,7 +110,7 @@ jobs: - name: Setup pnpm and Next.js cache if: steps.check-changes.outputs.any_changed == 'true' - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: | ${{ env.STORE_PATH }} diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 0000000000..931ef1b94c --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,25 @@ +rules: + secrets-outside-env: + ignore: + - api-bump-version.yml + - api-container-build-push.yml + - api-tests.yml + - backport.yml + - docs-bump-version.yml + - issue-triage.lock.yml + - mcp-container-build-push.yml + - pr-merged.yml + - prepare-release.yml + - sdk-bump-version.yml + - sdk-container-build-push.yml + - sdk-refresh-aws-services-regions.yml + - sdk-refresh-oci-regions.yml + - sdk-tests.yml + - ui-bump-version.yml + - ui-container-build-push.yml + - ui-e2e-tests-v2.yml + superfluous-actions: + ignore: + - pr-check-changelog.yml + - pr-conflict-checker.yml + - prepare-release.yml diff --git a/.gitignore b/.gitignore index 4dfb137b89..9e0e4da849 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ htmlcov/ **/mcp-config.json **/mcpServers.json .mcp/ +.mcp.json # AI Coding Assistants - Cursor .cursorignore @@ -83,6 +84,7 @@ continue.json .continuerc.json # AI Coding Assistants - OpenCode +.opencode/ opencode.json # AI Coding Assistants - GitHub Copilot @@ -149,6 +151,8 @@ node_modules # Persistent data _data/ +/openspec/ +/.gitmodules # AI Instructions (generated by skills/setup.sh from AGENTS.md) CLAUDE.md @@ -163,3 +167,6 @@ GEMINI.md .codex/skills .github/skills .gemini/skills + +# Claude Code +.claude/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb65669765..1f55160b4c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,140 +1,212 @@ +# Priority tiers (lower = runs first, same priority = concurrent): +# P0 — fast file fixers +# P10 — validators and guards +# P20 — auto-formatters +# P30 — linters +# P40 — security scanners +# P50 — dependency validation + +default_install_hook_types: [pre-commit, pre-push] + repos: - ## GENERAL - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + ## GENERAL (prek built-in — no external repo needed) + - repo: builtin hooks: - id: check-merge-conflict + priority: 10 - id: check-yaml - args: ["--unsafe"] - exclude: prowler/config/llm_config.yaml + args: ["--allow-multiple-documents"] + exclude: (prowler/config/llm_config.yaml|contrib/) + priority: 10 - id: check-json + priority: 10 - id: end-of-file-fixer + priority: 0 - id: trailing-whitespace + priority: 0 - id: no-commit-to-branch + priority: 10 - id: pretty-format-json args: ["--autofix", --no-sort-keys, --no-ensure-ascii] + priority: 10 ## TOML - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.13.0 + rev: v2.16.0 hooks: - id: pretty-format-toml args: [--autofix] files: pyproject.toml + priority: 20 + + ## GITHUB ACTIONS + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.24.1 + hooks: + - id: zizmor + files: ^\.github/ + priority: 30 ## BASH - repo: https://github.com/koalaman/shellcheck-precommit - rev: v0.10.0 + rev: v0.11.0 hooks: - id: shellcheck exclude: contrib + priority: 30 - ## PYTHON + ## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/) - repo: https://github.com/myint/autoflake - rev: v2.3.1 + rev: v2.3.3 hooks: - id: autoflake - exclude: ^skills/ + name: "SDK - autoflake" + files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] } args: [ "--in-place", "--remove-all-unused-imports", "--remove-unused-variable", ] + priority: 20 - repo: https://github.com/pycqa/isort - rev: 5.13.2 + rev: 8.0.1 hooks: - id: isort - exclude: ^skills/ + name: "SDK - isort" + files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] } args: ["--profile", "black"] + priority: 20 - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 26.3.1 hooks: - id: black - exclude: ^skills/ + name: "SDK - black" + files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] } + priority: 20 - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + rev: 7.3.0 hooks: - id: flake8 - exclude: (contrib|^skills/) + name: "SDK - flake8" + files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] } args: ["--ignore=E266,W503,E203,E501,W605"] + priority: 30 + ## PYTHON — API + MCP Server (ruff) + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.11 + hooks: + - id: ruff + name: "API + MCP - ruff check" + files: { glob: ["{api,mcp_server}/**/*.py"] } + args: ["--fix"] + priority: 30 + - id: ruff-format + name: "API + MCP - ruff format" + files: { glob: ["{api,mcp_server}/**/*.py"] } + priority: 20 + + ## PYTHON — Poetry - repo: https://github.com/python-poetry/poetry - rev: 2.1.1 + rev: 2.3.4 hooks: - id: poetry-check name: API - poetry-check args: ["--directory=./api"] + files: { glob: ["api/{pyproject.toml,poetry.lock}"] } pass_filenames: false + priority: 50 - id: poetry-lock name: API - poetry-lock args: ["--directory=./api"] + files: { glob: ["api/{pyproject.toml,poetry.lock}"] } pass_filenames: false + priority: 50 - id: poetry-check name: SDK - poetry-check args: ["--directory=./"] + files: { glob: ["{pyproject.toml,poetry.lock}"] } pass_filenames: false + priority: 50 - id: poetry-lock name: SDK - poetry-lock args: ["--directory=./"] + files: { glob: ["{pyproject.toml,poetry.lock}"] } pass_filenames: false + priority: 50 + ## CONTAINERS - repo: https://github.com/hadolint/hadolint - rev: v2.13.0-beta + rev: v2.14.0 hooks: - id: hadolint args: ["--ignore=DL3013"] + priority: 30 + ## LOCAL HOOKS - repo: local hooks: - id: pylint - name: pylint - entry: bash -c 'pylint --disable=W,C,R,E -j 0 -rn -sn prowler/' + name: "SDK - pylint" + entry: pylint --disable=W,C,R,E -j 0 -rn -sn language: system - files: '.*\.py' + types: [python] + files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] } + priority: 30 - id: trufflehog name: TruffleHog description: Detect secrets in your data. - entry: bash -c 'trufflehog --no-update git file://. --only-verified --fail' + entry: bash -c 'trufflehog --no-update git file://. --since-commit HEAD --only-verified --fail' # For running trufflehog in docker, use the following entry instead: # entry: bash -c 'docker run -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --only-verified --fail' language: system + pass_filenames: false stages: ["pre-commit", "pre-push"] + priority: 40 - id: bandit name: bandit description: "Bandit is a tool for finding common security issues in Python code" - entry: bash -c 'bandit -q -lll -x '*_test.py,./contrib/,./.venv/,./skills/' -r .' + entry: bandit -q -lll language: system + types: [python] files: '.*\.py' + exclude: + { glob: ["{contrib,skills}/**", "**/.venv/**", "**/*_test.py"] } + priority: 40 - id: safety name: safety description: "Safety is a tool that checks your installed dependencies for known security vulnerabilities" - # TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X - # TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0 - entry: bash -c 'safety check --ignore 70612,66963,74429,76352,76353,77744,77745,79023,79027' + # Accepted CVEs, severity threshold, and ignore expirations live in .safety-policy.yml + entry: safety check --policy-file .safety-policy.yml language: system + pass_filenames: false + files: + { + glob: + [ + "**/pyproject.toml", + "**/poetry.lock", + "**/requirements*.txt", + ".safety-policy.yml", + ], + } + priority: 40 - id: vulture name: vulture description: "Vulture finds unused code in Python programs." - entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/,skills/" --min-confidence 100 .' + entry: vulture --min-confidence 100 language: system + types: [python] files: '.*\.py' - - - id: ui-checks - name: UI - Husky Pre-commit - description: "Run UI pre-commit checks (Claude Code validation + healthcheck)" - entry: bash -c 'cd ui && .husky/pre-commit' - language: system - files: '^ui/.*\.(ts|tsx|js|jsx|json|css)$' - pass_filenames: false - verbose: true + priority: 40 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 17d338d2e9..5d375cd0df 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,7 +13,7 @@ build: post_create_environment: # Install poetry # https://python-poetry.org/docs/#installing-manually - - python -m pip install poetry + - python -m pip install poetry==2.3.4 post_install: # Install dependencies with 'docs' dependency group # https://python-poetry.org/docs/managing-dependencies/#dependency-groups diff --git a/.safety-policy.yml b/.safety-policy.yml new file mode 100644 index 0000000000..fec97e2fb9 --- /dev/null +++ b/.safety-policy.yml @@ -0,0 +1,58 @@ +# Safety policy for `safety check` (Safety CLI 3.x, v2 schema). +# Applied in: .pre-commit-config.yaml, .github/workflows/api-security.yml, +# .github/workflows/sdk-security.yml via `--policy-file`. +# +# Validate: poetry run safety validate policy_file --path .safety-policy.yml + +security: + # Scan unpinned requirements too. Prowler pins via poetry.lock, so this is + # defensive against accidental unpinned entries. + ignore-unpinned-requirements: False + + # CVSS severity filter. 7 = report only HIGH (7.0–8.9) and CRITICAL (9.0–10.0). + # Reference: 9=CRITICAL only, 7=CRITICAL+HIGH, 4=CRITICAL+HIGH+MEDIUM. + ignore-cvss-severity-below: 7 + + # Unknown severity is unrated, not safe. Keep False so unrated CVEs still fail + # the build and get a human eye. Flip to True only if noise is unmanageable. + ignore-cvss-unknown-severity: False + + # Fail the build when a non-ignored vulnerability is found. + continue-on-vulnerability-error: False + + # Explicit accepted vulnerabilities. Each entry MUST have a reason and an + # expiry. Expired entries fail the scan, forcing re-audit. + ignore-vulnerabilities: + 77744: + reason: "Botocore requires urllib3 1.X. Remove once upgraded to urllib3 2.X." + expires: '2026-10-22' + 77745: + reason: "Botocore requires urllib3 1.X. Remove once upgraded to urllib3 2.X." + expires: '2026-10-22' + 79023: + reason: "knack ReDoS; blocked until azure-cli-core (via cartography) allows knack >=0.13.0." + expires: '2026-10-22' + 79027: + reason: "knack ReDoS; blocked until azure-cli-core (via cartography) allows knack >=0.13.0." + expires: '2026-10-22' + 86217: + reason: "alibabacloud-tea-openapi==0.4.3 blocks upgrade to cryptography >=46.0.0." + expires: '2026-10-22' + 71600: + reason: "CVE-2024-1135 false positive. Fixed in gunicorn 22.0.0; project uses 23.0.0." + expires: '2026-10-22' + 70612: + reason: "TBD - audit required. Reason not documented in prior --ignore list." + expires: '2026-07-22' + 66963: + reason: "TBD - audit required. Reason not documented in prior --ignore list." + expires: '2026-07-22' + 74429: + reason: "TBD - audit required. Reason not documented in prior --ignore list." + expires: '2026-07-22' + 76352: + reason: "TBD - audit required. Reason not documented in prior --ignore list." + expires: '2026-07-22' + 76353: + reason: "TBD - audit required. Reason not documented in prior --ignore list." + expires: '2026-07-22' diff --git a/.worktreeinclude b/.worktreeinclude new file mode 100644 index 0000000000..a9fce75e0f --- /dev/null +++ b/.worktreeinclude @@ -0,0 +1,2 @@ +.envrc +ui/.env.local diff --git a/AGENTS.md b/AGENTS.md index 7eb2262917..1d7a36c667 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -140,7 +140,7 @@ Prowler is an open-source cloud security assessment tool supporting AWS, Azure, | Component | Location | Tech Stack | |-----------|----------|------------| -| SDK | `prowler/` | Python 3.9+, Poetry | +| SDK | `prowler/` | Python 3.10+, Poetry 2.3+ | | API | `api/` | Django 5.1, DRF, Celery | | UI | `ui/` | Next.js 15, React 19, Tailwind 4 | | MCP Server | `mcp_server/` | FastMCP, Python 3.12+ | @@ -153,12 +153,12 @@ Prowler is an open-source cloud security assessment tool supporting AWS, Azure, ```bash # Setup poetry install --with dev -poetry run pre-commit install +poetry run prek install # Code quality poetry run make lint poetry run make format -poetry run pre-commit run --all-files +poetry run prek run --all-files ``` --- diff --git a/Dockerfile b/Dockerfile index 60d72ad469..49fbc5356e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12.11-slim-bookworm AS build +FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 AS build LABEL maintainer="https://github.com/prowler-cloud/prowler" LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler" @@ -9,6 +9,9 @@ ENV POWERSHELL_VERSION=${POWERSHELL_VERSION} ARG TRIVY_VERSION=0.69.2 ENV TRIVY_VERSION=${TRIVY_VERSION} +ARG ZIZMOR_VERSION=1.24.1 +ENV ZIZMOR_VERSION=${ZIZMOR_VERSION} + # hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ wget libicu72 libunwind8 libssl3 libcurl4 ca-certificates apt-transport-https gnupg \ @@ -48,6 +51,22 @@ RUN ARCH=$(uname -m) && \ mkdir -p /tmp/.cache/trivy && \ chmod 777 /tmp/.cache/trivy +# Install zizmor for GitHub Actions workflow scanning +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + ZIZMOR_ARCH="x86_64-unknown-linux-gnu" ; \ + elif [ "$ARCH" = "aarch64" ]; then \ + ZIZMOR_ARCH="aarch64-unknown-linux-gnu" ; \ + else \ + echo "Unsupported architecture for zizmor: $ARCH" && exit 1 ; \ + fi && \ + wget --progress=dot:giga "https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/zizmor-${ZIZMOR_ARCH}.tar.gz" -O /tmp/zizmor.tar.gz && \ + mkdir -p /tmp/zizmor-extract && \ + tar zxf /tmp/zizmor.tar.gz -C /tmp/zizmor-extract && \ + mv /tmp/zizmor-extract/zizmor /usr/local/bin/zizmor && \ + chmod +x /usr/local/bin/zizmor && \ + rm -rf /tmp/zizmor.tar.gz /tmp/zizmor-extract + # Add prowler user RUN addgroup --gid 1000 prowler && \ adduser --uid 1000 --gid 1000 --disabled-password --gecos "" prowler @@ -68,7 +87,7 @@ ENV HOME='/home/prowler' ENV PATH="${HOME}/.local/bin:${PATH}" #hadolint ignore=DL3013 RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir poetry + pip install --no-cache-dir poetry==2.3.4 RUN poetry install --compile && \ rm -rf ~/.cache/pip diff --git a/README.md b/README.md index 79b274e05b..8f4565481d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

- Prowler is the Open Cloud Security platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size. + Prowler is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.

Secure ANY cloud at AI Speed at prowler.com @@ -41,7 +41,7 @@ # Description -**Prowler** is the world’s most widely used _open-source cloud security platform_ that automates security and compliance across **any cloud environment**. With hundreds of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size. +**Prowler** is the world’s most widely used _Open-Source Cloud Security Platform_ that automates security and compliance across **any cloud environment**. With hundreds of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY Cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size. Prowler includes hundreds of built-in controls to ensure compliance with standards and frameworks, including: @@ -104,21 +104,22 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically | Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface | |---|---|---|---|---|---|---| -| AWS | 572 | 83 | 41 | 17 | Official | UI, API, CLI | -| Azure | 165 | 20 | 18 | 13 | Official | UI, API, CLI | -| GCP | 100 | 13 | 15 | 11 | Official | UI, API, CLI | -| Kubernetes | 83 | 7 | 7 | 9 | Official | UI, API, CLI | -| GitHub | 21 | 2 | 1 | 2 | Official | UI, API, CLI | -| M365 | 89 | 9 | 4 | 5 | Official | UI, API, CLI | -| OCI | 48 | 13 | 3 | 10 | Official | UI, API, CLI | -| Alibaba Cloud | 61 | 9 | 3 | 9 | Official | UI, API, CLI | -| Cloudflare | 29 | 2 | 0 | 5 | Official | UI, API, CLI | +| AWS | 595 | 84 | 43 | 17 | Official | UI, API, CLI | +| Azure | 167 | 22 | 19 | 16 | Official | UI, API, CLI | +| GCP | 102 | 18 | 17 | 12 | Official | UI, API, CLI | +| Kubernetes | 83 | 7 | 7 | 11 | Official | UI, API, CLI | +| GitHub | 24 | 3 | 1 | 5 | Official | UI, API, CLI | +| M365 | 101 | 10 | 4 | 10 | Official | UI, API, CLI | +| OCI | 51 | 14 | 4 | 10 | Official | UI, API, CLI | +| Alibaba Cloud | 61 | 9 | 4 | 9 | Official | UI, API, CLI | +| Cloudflare | 29 | 3 | 0 | 5 | Official | UI, API, CLI | | IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI | | MongoDB Atlas | 10 | 3 | 0 | 8 | Official | UI, API, CLI | | LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI | | Image | N/A | N/A | N/A | N/A | Official | CLI, API | -| Google Workspace | 1 | 1 | 0 | 1 | Official | CLI | -| OpenStack | 27 | 4 | 0 | 8 | Official | UI, API, CLI | +| Google Workspace | 25 | 4 | 2 | 4 | Official | CLI | +| OpenStack | 34 | 5 | 0 | 9 | Official | UI, API, CLI | +| Vercel | 26 | 6 | 0 | 5 | Official | CLI | | NHN | 6 | 2 | 1 | 0 | Unofficial | CLI | > [!Note] @@ -239,9 +240,17 @@ pnpm start > Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started. +**Pre-commit Hooks Setup** + +Some pre-commit hooks require tools installed on your system: + +1. **Install [TruffleHog](https://github.com/trufflesecurity/trufflehog#install)** (secret scanning) — see the [official installation options](https://github.com/trufflesecurity/trufflehog#install). + +2. **Install [Hadolint](https://github.com/hadolint/hadolint#install)** (Dockerfile linting) — see the [official installation options](https://github.com/hadolint/hadolint#install). + ## Prowler CLI ### Pip package -Prowler CLI is available as a project in [PyPI](https://pypi.org/project/prowler-cloud/). Consequently, it can be installed using pip with Python >3.9.1, <3.13: +Prowler CLI is available as a project in [PyPI](https://pypi.org/project/prowler-cloud/). Consequently, it can be installed using pip with Python >=3.10, <3.13: ```console pip install prowler @@ -273,7 +282,7 @@ The container images are available here: ### From GitHub -Python >3.9.1, <3.13 is required with pip and Poetry: +Python >=3.10, <3.13 is required with pip and Poetry: ``` console git clone https://github.com/prowler-cloud/prowler @@ -291,6 +300,36 @@ python prowler-cli.py -v > If your Poetry version is below v2.0.0, continue using `poetry shell` to activate your environment. > For further guidance, refer to the Poetry Environment Activation Guide https://python-poetry.org/docs/managing-environments/#activating-the-environment. +# 🛡️ GitHub Action + +The official **Prowler GitHub Action** runs Prowler scans in your GitHub workflows using the official [`prowlercloud/prowler`](https://hub.docker.com/r/prowlercloud/prowler) Docker image. Scans run on any [supported provider](https://docs.prowler.com/user-guide/providers/), with optional [`--push-to-cloud`](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings) to send findings to Prowler Cloud and optional SARIF upload so findings show up in the repo's **Security → Code scanning** tab and as inline PR annotations. + +```yaml +name: Prowler IaC Scan +on: + pull_request: + +permissions: + contents: read + security-events: write + actions: read + +jobs: + prowler: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: prowler-cloud/prowler@5.25 + with: + provider: iac + output-formats: sarif json-ocsf + upload-sarif: true + flags: --severity critical high +``` + +Full configuration, per-provider authentication, and SARIF examples: [Prowler GitHub Action tutorial](docs/user-guide/tutorials/prowler-app-github-action.mdx). Marketplace listing: [Prowler Security Scan](https://github.com/marketplace/actions/prowler-security-scan). + # ✏️ High level architecture ## Prowler App @@ -301,7 +340,10 @@ python prowler-cli.py -v - **Prowler SDK**: A Python SDK designed to extend the functionality of the Prowler CLI for advanced capabilities. - **Prowler MCP Server**: A Model Context Protocol server that provides AI tools for Lighthouse, the AI-powered security assistant. This is a critical dependency for Lighthouse functionality. -![Prowler App Architecture](docs/products/img/prowler-app-architecture.png) +![Prowler App Architecture](docs/images/products/prowler-app-architecture.png) + + + ## Prowler CLI diff --git a/action.yml b/action.yml new file mode 100644 index 0000000000..3b5b8d8bdf --- /dev/null +++ b/action.yml @@ -0,0 +1,307 @@ +name: Prowler Security Scan +description: Run Prowler cloud security scanner using the official Docker image +branding: + icon: cloud + color: green + +inputs: + provider: + description: Cloud provider to scan (e.g. aws, azure, gcp, github, kubernetes, iac). See https://docs.prowler.com for supported providers. + required: true + image-tag: + description: > + Docker image tag for prowlercloud/prowler. + Default is "stable" (latest release). Available tags: + "stable" (latest release), "latest" (master branch, not stable), + "" (pinned release version). + See all tags at https://hub.docker.com/r/prowlercloud/prowler/tags + required: false + default: stable + output-formats: + description: Output format(s) for scan results (e.g. "json-ocsf", "sarif json-ocsf") + required: false + default: json-ocsf + push-to-cloud: + description: Push scan findings to Prowler Cloud. Requires the PROWLER_CLOUD_API_KEY environment variable. See https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli + required: false + default: "false" + flags: + description: 'Additional CLI flags passed to the Prowler scan (e.g. "--severity critical high --compliance cis_aws"). Values containing spaces can be quoted, e.g. "--resource-tag ''Environment=My Server''".' + required: false + default: "" + extra-env: + description: > + Space-, newline-, or comma-separated list of host environment variable NAMES to forward to the Prowler container + (e.g. "AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN" for AWS, + "GITHUB_PERSONAL_ACCESS_TOKEN" for GitHub, "CLOUDFLARE_API_TOKEN" for Cloudflare). + List names only; set the values via `env:` at the workflow or job level (typically from `secrets.*`). + See the README for per-provider examples. + required: false + default: "" + upload-sarif: + description: 'Upload SARIF results to GitHub Code Scanning (requires "sarif" in output-formats and both `security-events: write` and `actions: read` permissions)' + required: false + default: "false" + sarif-file: + description: Path to the SARIF file to upload (auto-detected from output/ if not set) + required: false + default: "" + sarif-category: + description: Category for the SARIF upload (used to distinguish multiple analyses) + required: false + default: prowler + fail-on-findings: + description: Fail the workflow step when Prowler detects findings (exit code 3). By default the action tolerates findings and succeeds. + required: false + default: "false" + +runs: + using: composite + steps: + - name: Validate inputs + shell: bash + env: + INPUT_IMAGE_TAG: ${{ inputs.image-tag }} + INPUT_UPLOAD_SARIF: ${{ inputs.upload-sarif }} + INPUT_OUTPUT_FORMATS: ${{ inputs.output-formats }} + run: | + # Validate image tag format (alphanumeric, dots, hyphens, underscores only) + if [[ ! "$INPUT_IMAGE_TAG" =~ ^[a-zA-Z0-9._-]+$ ]]; then + echo "::error::Invalid image-tag '${INPUT_IMAGE_TAG}'. Must contain only alphanumeric characters, dots, hyphens, and underscores." + exit 1 + fi + + # Warn if upload-sarif is enabled but sarif not in output-formats + if [ "$INPUT_UPLOAD_SARIF" = "true" ]; then + if [[ ! "$INPUT_OUTPUT_FORMATS" =~ (^|[[:space:]])sarif($|[[:space:]]) ]]; then + echo "::warning::upload-sarif is enabled but 'sarif' is not included in output-formats ('${INPUT_OUTPUT_FORMATS}'). SARIF upload will fail unless you add 'sarif' to output-formats." + fi + fi + + - name: Run Prowler scan + shell: bash + env: + INPUT_PROVIDER: ${{ inputs.provider }} + INPUT_IMAGE_TAG: ${{ inputs.image-tag }} + INPUT_OUTPUT_FORMATS: ${{ inputs.output-formats }} + INPUT_PUSH_TO_CLOUD: ${{ inputs.push-to-cloud }} + INPUT_FLAGS: ${{ inputs.flags }} + INPUT_EXTRA_ENV: ${{ inputs.extra-env }} + INPUT_FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} + run: | + set -e + + # Parse space-separated inputs with shlex so values with spaces can be quoted + # (e.g. `--resource-tag 'Environment=My Server'`). + mapfile -t OUTPUT_FORMATS < <(python3 -c 'import shlex, os; [print(t) for t in shlex.split(os.environ.get("INPUT_OUTPUT_FORMATS", ""))]') + mapfile -t EXTRA_FLAGS < <(python3 -c 'import shlex, os; [print(t) for t in shlex.split(os.environ.get("INPUT_FLAGS", ""))]') + mapfile -t EXTRA_ENV_NAMES < <(python3 -c 'import shlex, os; [print(t) for t in shlex.split(os.environ.get("INPUT_EXTRA_ENV", "").replace(",", " "))]') + + env_args=() + for var in "${EXTRA_ENV_NAMES[@]}"; do + [ -z "$var" ] && continue + if [[ ! "$var" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "::error::Invalid env var name '${var}' in extra-env. Names must match ^[A-Za-z_][A-Za-z0-9_]*$." + exit 1 + fi + env_args+=("-e" "$var") + done + + push_args=() + if [ "$INPUT_PUSH_TO_CLOUD" = "true" ]; then + push_args=("--push-to-cloud") + env_args+=("-e" "PROWLER_CLOUD_API_KEY") + fi + + mkdir -p "$GITHUB_WORKSPACE/output" + chmod 777 "$GITHUB_WORKSPACE/output" + + set +e + docker run --rm \ + "${env_args[@]}" \ + -v "$GITHUB_WORKSPACE:/home/prowler/workspace" \ + -v "$GITHUB_WORKSPACE/output:/home/prowler/workspace/output" \ + -w /home/prowler/workspace \ + "prowlercloud/prowler:${INPUT_IMAGE_TAG}" \ + "$INPUT_PROVIDER" \ + --output-formats "${OUTPUT_FORMATS[@]}" \ + "${push_args[@]}" \ + "${EXTRA_FLAGS[@]}" + exit_code=$? + set -e + + # Exit code 3 = findings detected + if [ "$exit_code" -eq 3 ] && [ "$INPUT_FAIL_ON_FINDINGS" != "true" ]; then + echo "::notice::Prowler detected findings (exit code 3). Set fail-on-findings to 'true' to fail the workflow on findings." + exit 0 + fi + exit $exit_code + + - name: Upload scan results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: prowler-${{ inputs.provider }} + path: output/ + retention-days: 30 + if-no-files-found: warn + + - name: Find SARIF file + if: always() && inputs.upload-sarif == 'true' + id: find-sarif + shell: bash + env: + INPUT_SARIF_FILE: ${{ inputs.sarif-file }} + run: | + if [ -n "$INPUT_SARIF_FILE" ]; then + echo "sarif_path=$INPUT_SARIF_FILE" >> "$GITHUB_OUTPUT" + else + sarif_file=$(find output/ -name '*.sarif' -type f | head -1) + if [ -z "$sarif_file" ]; then + echo "::warning::No .sarif file found in output/. Ensure 'sarif' is included in output-formats." + echo "sarif_path=" >> "$GITHUB_OUTPUT" + else + echo "sarif_path=$sarif_file" >> "$GITHUB_OUTPUT" + fi + fi + + - name: Upload SARIF to GitHub Code Scanning + if: always() && inputs.upload-sarif == 'true' && steps.find-sarif.outputs.sarif_path != '' + uses: github/codeql-action/upload-sarif@d4b3ca9fa7f69d38bfcd667bdc45bc373d16277e # v4 + with: + sarif_file: ${{ steps.find-sarif.outputs.sarif_path }} + category: ${{ inputs.sarif-category }} + + - name: Write scan summary + if: always() + shell: bash + env: + INPUT_PROVIDER: ${{ inputs.provider }} + INPUT_UPLOAD_SARIF: ${{ inputs.upload-sarif }} + INPUT_PUSH_TO_CLOUD: ${{ inputs.push-to-cloud }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + BRANCH: ${{ github.head_ref || github.ref_name }} + GH_TOKEN: ${{ github.token }} + run: | + set +e + + # Build a link to the scan step in the workflow logs. Requires `actions: read` + # on the caller's GITHUB_TOKEN; silently skips the link if unavailable. + scan_step_url="" + if [ -n "${GH_TOKEN:-}" ] && command -v gh >/dev/null 2>&1; then + job_info=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT:-1}/jobs" \ + --jq ".jobs[] | select(.runner_name == \"${RUNNER_NAME:-}\")" 2>/dev/null) + if [ -n "$job_info" ]; then + job_id=$(jq -r '.id // empty' <<<"$job_info") + step_number=$(jq -r '[.steps[]? | select((.name // "") | test("Run Prowler scan"; "i")) | .number] | first // empty' <<<"$job_info") + if [ -z "$step_number" ]; then + step_number=$(jq -r '[.steps[]? | select(.status == "in_progress") | .number] | first // empty' <<<"$job_info") + fi + if [ -n "$job_id" ] && [ -n "$step_number" ]; then + scan_step_url="${REPO_URL}/actions/runs/${GITHUB_RUN_ID}/job/${job_id}#step:${step_number}:1" + elif [ -n "$job_id" ]; then + scan_step_url="${REPO_URL}/actions/runs/${GITHUB_RUN_ID}/job/${job_id}" + fi + fi + fi + + # Map provider code to a properly-cased display name. + case "$INPUT_PROVIDER" in + alibabacloud) provider_name="Alibaba Cloud" ;; + aws) provider_name="AWS" ;; + azure) provider_name="Azure" ;; + cloudflare) provider_name="Cloudflare" ;; + gcp) provider_name="GCP" ;; + github) provider_name="GitHub" ;; + googleworkspace) provider_name="Google Workspace" ;; + iac) provider_name="IaC" ;; + image) provider_name="Container Image" ;; + kubernetes) provider_name="Kubernetes" ;; + llm) provider_name="LLM" ;; + m365) provider_name="Microsoft 365" ;; + mongodbatlas) provider_name="MongoDB Atlas" ;; + nhn) provider_name="NHN" ;; + openstack) provider_name="OpenStack" ;; + oraclecloud) provider_name="Oracle Cloud" ;; + vercel) provider_name="Vercel" ;; + *) provider_name="${INPUT_PROVIDER^}" ;; + esac + + ocsf_file=$(find output/ -name '*.ocsf.json' -type f 2>/dev/null | head -1) + + { + echo "## Prowler ${provider_name} Scan Summary" + echo "" + + counts="" + if [ -n "$ocsf_file" ] && [ -s "$ocsf_file" ]; then + counts=$(jq -r '[ + length, + ([.[] | select(.status_code == "FAIL")] | length), + ([.[] | select(.status_code == "PASS")] | length), + ([.[] | select(.status_code == "MUTED")] | length), + ([.[] | select(.status_code == "FAIL" and .severity == "Critical")] | length), + ([.[] | select(.status_code == "FAIL" and .severity == "High")] | length), + ([.[] | select(.status_code == "FAIL" and .severity == "Medium")] | length), + ([.[] | select(.status_code == "FAIL" and .severity == "Low")] | length), + ([.[] | select(.status_code == "FAIL" and .severity == "Informational")] | length) + ] | @tsv' "$ocsf_file" 2>/dev/null) + fi + + if [ -n "$counts" ]; then + read -r total fail pass muted critical high medium low info <<<"$counts" + + line="**${fail:-0} failing** · ${pass:-0} passing" + [ "${muted:-0}" -gt 0 ] && line="${line} · ${muted} muted" + echo "${line} — ${total:-0} checks total" + echo "" + echo "| Severity | Failing |" + echo "|----------|---------|" + echo "| ‼️ Critical | ${critical:-0} |" + echo "| 🔴 High | ${high:-0} |" + echo "| 🟠 Medium | ${medium:-0} |" + echo "| 🔵 Low | ${low:-0} |" + echo "| ⚪ Informational | ${info:-0} |" + echo "" + else + echo "_No findings report was produced. Check the scan logs above._" + echo "" + fi + + if [ -n "$scan_step_url" ]; then + echo "**Scan logs:** [view in workflow run](${scan_step_url})" + echo "" + fi + + echo "**Get the full report:** [\`prowler-${INPUT_PROVIDER}\` artifact](${RUN_URL}#artifacts)" + + if [ "$INPUT_UPLOAD_SARIF" = "true" ] && [ -n "$BRANCH" ]; then + encoded_branch=$(jq -nr --arg b "$BRANCH" '$b|@uri') + echo "" + echo "**See results in GitHub Code Security:** [open alerts on \`${BRANCH}\`](${REPO_URL}/security/code-scanning?query=is%3Aopen+branch%3A${encoded_branch})" + fi + + if [ "$INPUT_PUSH_TO_CLOUD" != "true" ]; then + echo "" + echo "---" + echo "" + echo "### Scale ${provider_name} security with Prowler Cloud ☁️" + echo "" + echo "Send this scan's findings to **[Prowler Cloud](https://cloud.prowler.com)** and get:" + echo "" + echo "- **Unified findings** across every cloud, SaaS provider (M365, Google Workspace, GitHub, MongoDB Atlas), IaC repo, Kubernetes cluster, and container image" + echo "- **Posture over time** with alerts, and notifications" + echo "- **Prowler Lighthouse AI**: agentic assistant that triages findings, explains root cause and helps with remediation" + echo "- **50+ Compliance frameworks** mapped automatically" + echo "- **Enterprise-ready platform**: SOC 2 Type 2, SSO/SAML, AWS Security Hub, S3 and Jira integrations" + echo "" + echo "**Get started in 3 steps:**" + echo "1. Create an account at [cloud.prowler.com](https://cloud.prowler.com)" + echo "2. Generate a Prowler Cloud API key ([docs](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli))" + echo "3. Add \`PROWLER_CLOUD_API_KEY\` to your GitHub secrets and set \`push-to-cloud: true\` on this action" + echo "" + echo "See [prowler.com/pricing](https://prowler.com/pricing) for plan details." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index ab63e8283b..8c57285025 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,6 +2,210 @@ All notable changes to the **Prowler API** are documented in this file. +## [1.27.0] (Prowler UNRELEASED) + +### 🚀 Added + +- New `scan-reset-ephemeral-resources` post-scan task zeroes `failed_findings_count` for resources missing from the latest full-scope scan, keeping ephemeral resources from polluting the Resources page sort [(#10929)](https://github.com/prowler-cloud/prowler/pull/10929) + +--- + +## [1.26.1] (Prowler v5.25.1) + +### 🐞 Fixed + +- Attack Paths: AWS scans no longer fail when enabled regions cannot be retrieved, and scans stuck in `scheduled` state are now cleaned up after the stale threshold [(#10917)](https://github.com/prowler-cloud/prowler/pull/10917) +- Scan report and compliance downloads now redirect to a presigned S3 URL instead of streaming through the API worker, preventing gunicorn timeouts on large files [(#10927)](https://github.com/prowler-cloud/prowler/pull/10927) + +--- + +## [1.26.0] (Prowler v5.25.0) + +### 🚀 Added + +- CIS Benchmark PDF report generation for scans, exposing the latest CIS version per provider via `GET /scans/{id}/cis/{name}/` [(#10650)](https://github.com/prowler-cloud/prowler/pull/10650) +- `/overviews/resource-groups` (resource inventory), `/overviews/categories` and `/overviews/attack-surfaces` now reflect newly-muted findings without waiting for the next scan. The post-mute `reaggregate-all-finding-group-summaries` task now also dispatches `aggregate_scan_resource_group_summaries_task`, `aggregate_scan_category_summaries_task` and `aggregate_attack_surface_task` per latest scan of every `(provider, day)` pair, rebuilding `ScanGroupSummary`, `ScanCategorySummary` and `AttackSurfaceOverview` alongside the tables already covered in #10827 [(#10843)](https://github.com/prowler-cloud/prowler/pull/10843) +- Install zizmor v1.24.1 in API Docker image for GitHub Actions workflow scanning [(#10607)](https://github.com/prowler-cloud/prowler/pull/10607) + +### 🔄 Changed + +- Allows tenant owners to expel users from their organizations [(#10787)](https://github.com/prowler-cloud/prowler/pull/10787) +- `aggregate_findings`, `aggregate_attack_surface`, `aggregate_scan_resource_group_summaries` and `aggregate_scan_category_summaries` now upsert via `bulk_create(update_conflicts=True, ...)` instead of the prior `ignore_conflicts=True` / plain INSERT / `already backfilled` short-circuit. Re-runs triggered by the post-mute reaggregation pipeline no longer trip the `unique_*_per_scan` constraints nor silently drop updates, and are race-safe under concurrent writers (e.g. scan completion overlapping with a fresh mute rule) [(#10843)](https://github.com/prowler-cloud/prowler/pull/10843) +- Rename the scan-category and scan-resource-group summary aggregators from `backfill_*` to `aggregate_*` [(#10843)](https://github.com/prowler-cloud/prowler/pull/10843) + +### 🐞 Fixed + +- `generate_outputs_task` crashing with `KeyError` for compliance frameworks listed by `get_compliance_frameworks` but not loadable by `Compliance.get_bulk` [(#10903)](https://github.com/prowler-cloud/prowler/pull/10903) + +--- + +## [1.25.4] (Prowler v5.24.4) + +### 🚀 Added + +- `DJANGO_SENTRY_TRACES_SAMPLE_RATE` env var (default `0.02`) enables Sentry performance tracing for the API [(#10873)](https://github.com/prowler-cloud/prowler/pull/10873) + +### 🔄 Changed + +- Attack Paths: Neo4j driver `connection_acquisition_timeout` is now configurable via `NEO4J_CONN_ACQUISITION_TIMEOUT` (default lowered from 120 s to 15 s) [(#10873)](https://github.com/prowler-cloud/prowler/pull/10873) + +### 🐞 Fixed + +- `/tmp/prowler_api_output` saturation in compliance report workers: the final `rmtree` in `generate_compliance_reports` now only waits on frameworks actually generated for the provider (so unsupported frameworks no longer leave a placeholder `results` entry that blocks cleanup), output directories are created lazily per enabled framework, and both `generate_compliance_reports` and `generate_outputs_task` run an opportunistic stale cleanup at task start with a 48h age threshold, a per-host `fcntl` throttle, a 50-deletions-per-run cap, and guards that protect EXECUTING scans and scans whose `output_location` still points to a local path (metadata lookups routed through the admin DB so RLS does not hide those rows) [(#10874)](https://github.com/prowler-cloud/prowler/pull/10874) + +--- + +## [1.25.3] (Prowler v5.24.3) + +### 🚀 Added + +- `/overviews/findings`, `/overviews/findings-severity` and `/overviews/services` now reflect newly-muted findings without waiting for the next scan. The post-mute `reaggregate-all-finding-group-summaries` task was extended to re-run the same per-scan pipeline that scan completion runs (`ScanSummary`, `DailySeveritySummary`, `FindingGroupDailySummary`) on the latest scan of every `(provider, day)` pair, keeping the pre-aggregated tables in sync with `Finding.muted` updates [(#10827)](https://github.com/prowler-cloud/prowler/pull/10827) + +### 🐞 Fixed + +- Finding groups aggregated `status` now treats muted findings as resolved: a group is `FAIL` only while at least one non-muted FAIL remains, otherwise it is `PASS` (including fully-muted groups). The `filter[status]` filter and the `sort=status` ordering share the same semantics, keeping `status` consistent with `fail_count` and the orthogonal `muted` flag [(#10825)](https://github.com/prowler-cloud/prowler/pull/10825) +- `aggregate_findings` is now idempotent: it deletes the scan's existing `ScanSummary` rows before `bulk_create`, so re-runs (such as the post-mute reaggregation pipeline) no longer violate the `unique_scan_summary` constraint and no longer abort the downstream `DailySeveritySummary` / `FindingGroupDailySummary` recomputation for the affected scan [(#10827)](https://github.com/prowler-cloud/prowler/pull/10827) +- Attack Paths: Findings on AWS were silently dropped during the Neo4j merge for resources whose Cartography node is keyed by a short identifier (e.g. EC2 instances) rather than the full ARN [(#10839)](https://github.com/prowler-cloud/prowler/pull/10839) + +--- + +## [1.25.2] (Prowler v5.24.2) + +### 🔄 Changed + +- Finding groups `/resources` endpoints now materialize the filtered finding IDs into a Python list before filtering `ResourceFindingMapping`, so PostgreSQL switches from a Merge Semi Join that read hundreds of thousands of RFM index entries to a Nested Loop Index Scan over `finding_id`. The `has_mappings.exists()` pre-check is removed, and a request-scoped cache deduplicates the finding-id round-trip across the helpers that build different RFM querysets [(#10816)](https://github.com/prowler-cloud/prowler/pull/10816) + +### 🐞 Fixed + +- `/finding-groups/latest//resources` now selects the latest completed scan per provider by `-completed_at` (then `-inserted_at`) instead of `-inserted_at`, matching the `/finding-groups/latest` summary path and the daily-summary upsert so overlapping scans no longer produce diverging `delta`/`new_count` between the two endpoints [(#10802)](https://github.com/prowler-cloud/prowler/pull/10802) + + +## [1.25.1] (Prowler v5.24.1) + +### 🔄 Changed + +- Attack Paths: Restore `SYNC_BATCH_SIZE` and `FINDINGS_BATCH_SIZE` defaults to 1000, upgrade Cartography to 0.135.0, enable Celery queue priority for cleanup task, rewrite Finding insertion, remove AWS graph cleanup and add timing logs [(#10729)](https://github.com/prowler-cloud/prowler/pull/10729) + +### 🐞 Fixed + +- Finding group resources endpoints now include findings without associated resources (orphaned IaC findings) as simulated resource rows, and return one row per finding when multiple findings share a resource [(#10708)](https://github.com/prowler-cloud/prowler/pull/10708) +- Attack Paths: Missing `tenant_id` filter while getting related findings after scan completes [(#10722)](https://github.com/prowler-cloud/prowler/pull/10722) +- Finding group counters `pass_count`, `fail_count` and `manual_count` now exclude muted findings [(#10753)](https://github.com/prowler-cloud/prowler/pull/10753) +- Silent data loss in `ResourceFindingMapping` bulk insert that left findings orphaned when `INSERT ... ON CONFLICT DO NOTHING` dropped rows without raising; added explicit `unique_fields` [(#10724)](https://github.com/prowler-cloud/prowler/pull/10724) +- `DELETE /tenants/{tenant_pk}/memberships/{id}` now deletes the expelled user's account when the removed membership was their last one, and blacklists every outstanding refresh token for that user so their existing sessions can no longer mint new access tokens [(#10787)](https://github.com/prowler-cloud/prowler/pull/10787) + +--- + +## [1.25.0] (Prowler v5.24.0) + +### 🔄 Changed + +- Bump Poetry to `2.3.4` in Dockerfile and pre-commit hooks. Regenerate `api/poetry.lock` [(#10681)](https://github.com/prowler-cloud/prowler/pull/10681) +- Attack Paths: Remove dead `cleanup_findings` no-op and its supporting `prowler_finding_lastupdated` index [(#10684)](https://github.com/prowler-cloud/prowler/pull/10684) + +### 🐞 Fixed + +- Worker-beat race condition on cold start: replaced `sleep 15` with API service healthcheck dependency (Docker Compose) and init containers (Helm), aligned Gunicorn default port to `8080` [(#10603)](https://github.com/prowler-cloud/prowler/pull/10603) +- API container startup crash on Linux due to root-owned bind-mount preventing JWT key generation [(#10646)](https://github.com/prowler-cloud/prowler/pull/10646) + +### 🔐 Security + +- `pytest` from 8.2.2 to 9.0.3 to fix CVE-2025-71176 [(#10678)](https://github.com/prowler-cloud/prowler/pull/10678) + +--- + +## [1.24.0] (Prowler v5.23.0) + +### 🚀 Added + +- RBAC role lookup filtered by `tenant_id` to prevent cross-tenant privilege leak [(#10491)](https://github.com/prowler-cloud/prowler/pull/10491) +- `VALKEY_SCHEME`, `VALKEY_USERNAME`, and `VALKEY_PASSWORD` environment variables to configure Celery broker TLS/auth connection details for Valkey/ElastiCache [(#10420)](https://github.com/prowler-cloud/prowler/pull/10420) +- `Vercel` provider support [(#10190)](https://github.com/prowler-cloud/prowler/pull/10190) +- Finding groups list and latest endpoints support `sort=delta`, ordering by `new_count` then `changed_count` so groups with the most new findings rank highest [(#10606)](https://github.com/prowler-cloud/prowler/pull/10606) +- Finding group resources endpoints (`/finding-groups/{check_id}/resources` and `/finding-groups/latest/{check_id}/resources`) now expose `finding_id` per row, pointing to the most recent matching Finding for each resource. UUIDv7 ordering guarantees `Max(finding__id)` resolves to the latest snapshot [(#10630)](https://github.com/prowler-cloud/prowler/pull/10630) +- Handle CIS and CISA SCuBA compliance framework from google workspace [(#10629)](https://github.com/prowler-cloud/prowler/pull/10629) +- Sort support for all finding group counter fields: `pass_muted_count`, `fail_muted_count`, `manual_muted_count`, and all `new_*`/`changed_*` status-mute breakdown counters [(#10655)](https://github.com/prowler-cloud/prowler/pull/10655) + +### 🔄 Changed + +- Finding groups list/latest/resources now expose `status` ∈ `{FAIL, PASS, MANUAL}` and `muted: bool` as orthogonal fields. The aggregated `status` reflects the underlying check outcome regardless of mute state, and `muted=true` signals that every finding in the group/resource is muted. New `manual_count` is exposed alongside `pass_count`/`fail_count`, plus `pass_muted_count`/`fail_muted_count`/`manual_muted_count` siblings so clients can isolate the muted half of each status. The `new_*`/`changed_*` deltas are now broken down by status and mute state via 12 new counters (`new_fail_count`, `new_fail_muted_count`, `new_pass_count`, `new_pass_muted_count`, `new_manual_count`, `new_manual_muted_count` and the matching `changed_*` set). New `filter[muted]=true|false` and `sort=status` (FAIL > PASS > MANUAL) / `sort=muted` are supported. `filter[status]=MUTED` is no longer accepted [(#10630)](https://github.com/prowler-cloud/prowler/pull/10630) +- Attack Paths: Periodic cleanup of stale scans with dead-worker detection via Celery inspect, marking orphaned `EXECUTING` scans as `FAILED` and recovering `graph_data_ready` [(#10387)](https://github.com/prowler-cloud/prowler/pull/10387) +- Attack Paths: Replace `_provider_id` property with `_Provider_{uuid}` label for provider isolation, add regex-based label injection for custom queries [(#10402)](https://github.com/prowler-cloud/prowler/pull/10402) + +### 🐞 Fixed + +- `reaggregate_all_finding_group_summaries_task` now refreshes finding group daily summaries for every `(provider, day)` combination instead of only the latest scan per provider, matching the unbounded scope of `mute_historical_findings_task`. Mute rule operations no longer leave older daily summaries drifting from the underlying muted findings [(#10630)](https://github.com/prowler-cloud/prowler/pull/10630) +- Finding groups list/latest now apply computed status/severity filters and finding-level prefilters (delta, region, service, category, resource group, scan, resource type), plus `check_title` support for sort/filter consistency [(#10428)](https://github.com/prowler-cloud/prowler/pull/10428) +- Populate compliance data inside `check_metadata` for findings, which was always returned as `null` [(#10449)](https://github.com/prowler-cloud/prowler/pull/10449) +- 403 error for admin users listing tenants due to roles query not using the admin database connection [(#10460)](https://github.com/prowler-cloud/prowler/pull/10460) +- Filter transient Neo4j defunct connection logs in Sentry `before_send` to suppress false-positive alerts handled by `RetryableSession` retries [(#10452)](https://github.com/prowler-cloud/prowler/pull/10452) +- `MANAGE_ACCOUNT` permission no longer required for listing and creating tenants [(#10468)](https://github.com/prowler-cloud/prowler/pull/10468) +- Finding groups muted filter, counters, metadata extraction and mute reaggregation [(#10477)](https://github.com/prowler-cloud/prowler/pull/10477) +- Finding groups `check_title__icontains` resolution, `name__icontains` resource filter and `resource_group` field in `/resources` response [(#10486)](https://github.com/prowler-cloud/prowler/pull/10486) +- Membership `post_delete` signal using raw FK ids to avoid `DoesNotExist` during cascade deletions [(#10497)](https://github.com/prowler-cloud/prowler/pull/10497) +- Finding group resources endpoints returning false 404 when filters match no results, and `sort` parameter being ignored [(#10510)](https://github.com/prowler-cloud/prowler/pull/10510) +- Jira integration failing with `JiraInvalidIssueTypeError` on non-English Jira instances due to hardcoded `"Task"` issue type; now dynamically fetches available issue types per project [(#10534)](https://github.com/prowler-cloud/prowler/pull/10534) +- Finding group `first_seen_at` now reflects when a new finding appeared in the scan instead of the oldest carry-forward date across all unchanged findings [(#10595)](https://github.com/prowler-cloud/prowler/pull/10595) +- Attack Paths: Remove `clear_cache` call from read-only query endpoints; cache clearing belongs to the scan/ingestion flow, not API queries [(#10586)](https://github.com/prowler-cloud/prowler/pull/10586) + +### 🔐 Security + +- Pin all unpinned dependencies to exact versions to prevent supply chain attacks and ensure reproducible builds [(#10469)](https://github.com/prowler-cloud/prowler/pull/10469) +- `authlib` bumped from 1.6.6 to 1.6.9 to fix CVE-2026-28802 (JWT `alg: none` validation bypass) [(#10579)](https://github.com/prowler-cloud/prowler/pull/10579) +- `aiohttp` bumped from 3.13.3 to 3.13.5 to fix CVE-2026-34520 (the C parser accepted null bytes and control characters in response headers) [(#10538)](https://github.com/prowler-cloud/prowler/pull/10538) + +--- + +## [1.23.0] (Prowler v5.22.0) + +### 🚀 Added + +- Finding groups support `check_title` substring filtering [(#10377)](https://github.com/prowler-cloud/prowler/pull/10377) + +### 🐞 Fixed + +- Finding groups latest endpoint now aggregates the latest snapshot per provider before check-level totals, keeping impacted resources aligned across providers [(#10419)](https://github.com/prowler-cloud/prowler/pull/10419) +- Mute rule creation now triggers finding-group summary re-aggregation after historical muting, keeping stats in sync after mute operations [(#10419)](https://github.com/prowler-cloud/prowler/pull/10419) +- Attack Paths: Deduplicate nodes before ProwlerFinding lookup in Attack Paths Cypher queries, reducing execution time [(#10424)](https://github.com/prowler-cloud/prowler/pull/10424) + +### 🔐 Security + +- Replace stdlib XML parser with `defusedxml` in SAML metadata parsing to prevent XML bomb (billion laughs) DoS attacks [(#10165)](https://github.com/prowler-cloud/prowler/pull/10165) +- Bump `flask` to 3.1.3 (CVE-2026-27205) and `werkzeug` to 3.1.6 (CVE-2026-27199) [(#10430)](https://github.com/prowler-cloud/prowler/pull/10430) + +--- + +## [1.22.1] (Prowler v5.21.1) + +### 🐞 Fixed + +- Threat score aggregation query to eliminate unnecessary JOINs and `COUNT(DISTINCT)` overhead [(#10394)](https://github.com/prowler-cloud/prowler/pull/10394) + +--- + +## [1.22.0] (Prowler v5.21.0) + +### 🚀 Added + +- `CORS_ALLOWED_ORIGINS` configurable via environment variable [(#10355)](https://github.com/prowler-cloud/prowler/pull/10355) +- Attack Paths: Tenant and provider related labels to the nodes so they can be easily filtered on custom queries [(#10308)](https://github.com/prowler-cloud/prowler/pull/10308) + +### 🔄 Changed + +- Attack Paths: Complete migration to private graph labels and properties, removing deprecated dual-write support [(#10268)](https://github.com/prowler-cloud/prowler/pull/10268) +- Attack Paths: Reduce sync and findings memory usage with smaller batches, cursor iteration, and sequential sessions [(#10359)](https://github.com/prowler-cloud/prowler/pull/10359) + +### 🐞 Fixed + +- Attack Paths: Recover `graph_data_ready` flag when scan fails during graph swap, preventing query endpoints from staying blocked until the next successful scan [(#10354)](https://github.com/prowler-cloud/prowler/pull/10354) + +### 🔐 Security + +- Use `psycopg2.sql` to safely compose DDL in `PostgresEnumMigration`, preventing SQL injection via f-string interpolation [(#10166)](https://github.com/prowler-cloud/prowler/pull/10166) +- Replace stdlib XML parser with `defusedxml` in SAML metadata parsing to prevent XML bomb (billion laughs) DoS attacks [(#10165)](https://github.com/prowler-cloud/prowler/pull/10165) + +--- + ## [1.21.0] (Prowler v5.20.0) ### 🔄 Changed diff --git a/api/Dockerfile b/api/Dockerfile index a07115e9a4..6f8385934d 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12.10-slim-bookworm AS build +FROM python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db AS build LABEL maintainer="https://github.com/prowler-cloud/api" @@ -8,6 +8,9 @@ ENV POWERSHELL_VERSION=${POWERSHELL_VERSION} ARG TRIVY_VERSION=0.69.2 ENV TRIVY_VERSION=${TRIVY_VERSION} +ARG ZIZMOR_VERSION=1.24.1 +ENV ZIZMOR_VERSION=${ZIZMOR_VERSION} + # hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ wget \ @@ -22,6 +25,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libtool \ libxslt1-dev \ python3-dev \ + git \ && rm -rf /var/lib/apt/lists/* # Install PowerShell @@ -57,6 +61,22 @@ RUN ARCH=$(uname -m) && \ mkdir -p /tmp/.cache/trivy && \ chmod 777 /tmp/.cache/trivy +# Install zizmor for GitHub Actions workflow scanning +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + ZIZMOR_ARCH="x86_64-unknown-linux-gnu" ; \ + elif [ "$ARCH" = "aarch64" ]; then \ + ZIZMOR_ARCH="aarch64-unknown-linux-gnu" ; \ + else \ + echo "Unsupported architecture for zizmor: $ARCH" && exit 1 ; \ + fi && \ + wget --progress=dot:giga "https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/zizmor-${ZIZMOR_ARCH}.tar.gz" -O /tmp/zizmor.tar.gz && \ + mkdir -p /tmp/zizmor-extract && \ + tar zxf /tmp/zizmor.tar.gz -C /tmp/zizmor-extract && \ + mv /tmp/zizmor-extract/zizmor /usr/local/bin/zizmor && \ + chmod +x /usr/local/bin/zizmor && \ + rm -rf /tmp/zizmor.tar.gz /tmp/zizmor-extract + # Add prowler user RUN addgroup --gid 1000 prowler && \ adduser --uid 1000 --gid 1000 --disabled-password --gecos "" prowler @@ -71,7 +91,7 @@ RUN mkdir -p /tmp/prowler_api_output COPY pyproject.toml ./ RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir poetry + pip install --no-cache-dir poetry==2.3.4 ENV PATH="/home/prowler/.local/bin:$PATH" diff --git a/api/docker-entrypoint.sh b/api/docker-entrypoint.sh index eea024a4a2..fb1e1693b4 100755 --- a/api/docker-entrypoint.sh +++ b/api/docker-entrypoint.sh @@ -30,14 +30,32 @@ start_prod_server() { poetry run gunicorn -c config/guniconf.py config.wsgi:application } +resolve_worker_hostname() { + TASK_ID="" + + if [ -n "$ECS_CONTAINER_METADATA_URI_V4" ]; then + TASK_ID=$(wget -qO- --timeout=2 "${ECS_CONTAINER_METADATA_URI_V4}/task" | \ + python3 -c "import sys,json; print(json.load(sys.stdin)['TaskARN'].split('/')[-1])" 2>/dev/null) + fi + + if [ -z "$TASK_ID" ]; then + TASK_ID=$(python3 -c "import uuid; print(uuid.uuid4().hex)") + fi + + echo "${TASK_ID}@$(hostname)" +} + start_worker() { echo "Starting the worker..." - poetry run python -m celery -A config.celery worker -l "${DJANGO_LOGGING_LEVEL:-info}" -Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans -E --max-tasks-per-child 1 + poetry run python -m celery -A config.celery worker \ + -n "$(resolve_worker_hostname)" \ + -l "${DJANGO_LOGGING_LEVEL:-info}" \ + -Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans \ + -E --max-tasks-per-child 1 } start_worker_beat() { echo "Starting the worker-beat..." - sleep 15 poetry run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler } diff --git a/api/poetry.lock b/api/poetry.lock index dba0433896..f93e0d21e6 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. [[package]] name = "about-time" @@ -103,132 +103,132 @@ files = [ [[package]] name = "aiohttp" -version = "3.13.3" +version = "3.13.5" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "aiohttp-3.13.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7"}, - {file = "aiohttp-3.13.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821"}, - {file = "aiohttp-3.13.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845"}, - {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af"}, - {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940"}, - {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160"}, - {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7"}, - {file = "aiohttp-3.13.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455"}, - {file = "aiohttp-3.13.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279"}, - {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e"}, - {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d"}, - {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808"}, - {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40"}, - {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29"}, - {file = "aiohttp-3.13.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11"}, - {file = "aiohttp-3.13.3-cp310-cp310-win32.whl", hash = "sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd"}, - {file = "aiohttp-3.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c"}, - {file = "aiohttp-3.13.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b"}, - {file = "aiohttp-3.13.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64"}, - {file = "aiohttp-3.13.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea"}, - {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a"}, - {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540"}, - {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b"}, - {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3"}, - {file = "aiohttp-3.13.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1"}, - {file = "aiohttp-3.13.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3"}, - {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440"}, - {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7"}, - {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c"}, - {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51"}, - {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4"}, - {file = "aiohttp-3.13.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29"}, - {file = "aiohttp-3.13.3-cp311-cp311-win32.whl", hash = "sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239"}, - {file = "aiohttp-3.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f"}, - {file = "aiohttp-3.13.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c"}, - {file = "aiohttp-3.13.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168"}, - {file = "aiohttp-3.13.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d"}, - {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29"}, - {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3"}, - {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d"}, - {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463"}, - {file = "aiohttp-3.13.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc"}, - {file = "aiohttp-3.13.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf"}, - {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033"}, - {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f"}, - {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679"}, - {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423"}, - {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce"}, - {file = "aiohttp-3.13.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a"}, - {file = "aiohttp-3.13.3-cp312-cp312-win32.whl", hash = "sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046"}, - {file = "aiohttp-3.13.3-cp312-cp312-win_amd64.whl", hash = "sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57"}, - {file = "aiohttp-3.13.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c"}, - {file = "aiohttp-3.13.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9"}, - {file = "aiohttp-3.13.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3"}, - {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf"}, - {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6"}, - {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d"}, - {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261"}, - {file = "aiohttp-3.13.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0"}, - {file = "aiohttp-3.13.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730"}, - {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91"}, - {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3"}, - {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4"}, - {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998"}, - {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0"}, - {file = "aiohttp-3.13.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591"}, - {file = "aiohttp-3.13.3-cp313-cp313-win32.whl", hash = "sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf"}, - {file = "aiohttp-3.13.3-cp313-cp313-win_amd64.whl", hash = "sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e"}, - {file = "aiohttp-3.13.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808"}, - {file = "aiohttp-3.13.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415"}, - {file = "aiohttp-3.13.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f"}, - {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6"}, - {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687"}, - {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26"}, - {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a"}, - {file = "aiohttp-3.13.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1"}, - {file = "aiohttp-3.13.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25"}, - {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603"}, - {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a"}, - {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926"}, - {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba"}, - {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c"}, - {file = "aiohttp-3.13.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43"}, - {file = "aiohttp-3.13.3-cp314-cp314-win32.whl", hash = "sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1"}, - {file = "aiohttp-3.13.3-cp314-cp314-win_amd64.whl", hash = "sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984"}, - {file = "aiohttp-3.13.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c"}, - {file = "aiohttp-3.13.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592"}, - {file = "aiohttp-3.13.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f"}, - {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29"}, - {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc"}, - {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2"}, - {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587"}, - {file = "aiohttp-3.13.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8"}, - {file = "aiohttp-3.13.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632"}, - {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64"}, - {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0"}, - {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56"}, - {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72"}, - {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df"}, - {file = "aiohttp-3.13.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa"}, - {file = "aiohttp-3.13.3-cp314-cp314t-win32.whl", hash = "sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767"}, - {file = "aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344"}, - {file = "aiohttp-3.13.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e"}, - {file = "aiohttp-3.13.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7"}, - {file = "aiohttp-3.13.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02"}, - {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43"}, - {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6"}, - {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce"}, - {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80"}, - {file = "aiohttp-3.13.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a"}, - {file = "aiohttp-3.13.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6"}, - {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558"}, - {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7"}, - {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877"}, - {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3"}, - {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704"}, - {file = "aiohttp-3.13.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f"}, - {file = "aiohttp-3.13.3-cp39-cp39-win32.whl", hash = "sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1"}, - {file = "aiohttp-3.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538"}, - {file = "aiohttp-3.13.3.tar.gz", hash = "sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88"}, + {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, + {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, + {file = "aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670"}, + {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274"}, + {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a"}, + {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d"}, + {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796"}, + {file = "aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95"}, + {file = "aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5"}, + {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a"}, + {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73"}, + {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297"}, + {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074"}, + {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e"}, + {file = "aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7"}, + {file = "aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9"}, + {file = "aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76"}, + {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6"}, + {file = "aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d"}, + {file = "aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c"}, + {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb"}, + {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6"}, + {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13"}, + {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174"}, + {file = "aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc"}, + {file = "aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6"}, + {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49"}, + {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8"}, + {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d"}, + {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c"}, + {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac"}, + {file = "aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3"}, + {file = "aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06"}, + {file = "aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8"}, + {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9"}, + {file = "aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416"}, + {file = "aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2"}, + {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4"}, + {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9"}, + {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5"}, + {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e"}, + {file = "aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1"}, + {file = "aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286"}, + {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9"}, + {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88"}, + {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3"}, + {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b"}, + {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe"}, + {file = "aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14"}, + {file = "aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3"}, + {file = "aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1"}, + {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61"}, + {file = "aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832"}, + {file = "aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9"}, + {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090"}, + {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b"}, + {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a"}, + {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8"}, + {file = "aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665"}, + {file = "aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540"}, + {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb"}, + {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46"}, + {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8"}, + {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d"}, + {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6"}, + {file = "aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c"}, + {file = "aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc"}, + {file = "aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83"}, + {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c"}, + {file = "aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be"}, + {file = "aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25"}, + {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56"}, + {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2"}, + {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a"}, + {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be"}, + {file = "aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b"}, + {file = "aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94"}, + {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d"}, + {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7"}, + {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772"}, + {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5"}, + {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1"}, + {file = "aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b"}, + {file = "aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3"}, + {file = "aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162"}, + {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a"}, + {file = "aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254"}, + {file = "aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36"}, + {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f"}, + {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800"}, + {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf"}, + {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b"}, + {file = "aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a"}, + {file = "aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8"}, + {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be"}, + {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b"}, + {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6"}, + {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037"}, + {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500"}, + {file = "aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9"}, + {file = "aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8"}, + {file = "aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9"}, + {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf"}, + {file = "aiohttp-3.13.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1"}, + {file = "aiohttp-3.13.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10"}, + {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f"}, + {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b"}, + {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643"}, + {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031"}, + {file = "aiohttp-3.13.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258"}, + {file = "aiohttp-3.13.5-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a"}, + {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88"}, + {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0"}, + {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f"}, + {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8"}, + {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f"}, + {file = "aiohttp-3.13.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b"}, + {file = "aiohttp-3.13.5-cp39-cp39-win32.whl", hash = "sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83"}, + {file = "aiohttp-3.13.5-cp39-cp39-win_amd64.whl", hash = "sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67"}, + {file = "aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1"}, ] [package.dependencies] @@ -682,21 +682,21 @@ requests = ">=2.21.0,<3.0.0" [[package]] name = "alibabacloud-tea-openapi" -version = "0.4.1" +version = "0.4.4" description = "Alibaba Cloud openapi SDK Library for Python" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "alibabacloud_tea_openapi-0.4.1-py3-none-any.whl", hash = "sha256:e46bfa3ca34086d2c357d217a0b7284ecbd4b3bab5c88e075e73aec637b0e4a0"}, - {file = "alibabacloud_tea_openapi-0.4.1.tar.gz", hash = "sha256:2384b090870fdb089c3c40f3fb8cf0145b8c7d6c14abbac521f86a01abb5edaf"}, + {file = "alibabacloud_tea_openapi-0.4.4-py3-none-any.whl", hash = "sha256:cea6bc1fe35b0319a8752cb99eb0ecb0dab7ca1a71b99c12970ba0867410995f"}, + {file = "alibabacloud_tea_openapi-0.4.4.tar.gz", hash = "sha256:1b0917bc03cd49417da64945e92731716d53e2eb8707b235f54e45b7473221ce"}, ] [package.dependencies] alibabacloud-credentials = ">=1.0.2,<2.0.0" alibabacloud-gateway-spi = ">=0.0.2,<1.0.0" alibabacloud-tea-util = ">=0.3.13,<1.0.0" -cryptography = ">=3.0.0,<45.0.0" +cryptography = {version = ">=3.0.0,<47.0.0", markers = "python_version >= \"3.8\""} darabonba-core = ">=1.0.3,<2.0.0" [[package]] @@ -943,14 +943,14 @@ files = [ [[package]] name = "authlib" -version = "1.6.6" +version = "1.6.9" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "authlib-1.6.6-py2.py3-none-any.whl", hash = "sha256:7d9e9bc535c13974313a87f53e8430eb6ea3d1cf6ae4f6efcd793f2e949143fd"}, - {file = "authlib-1.6.6.tar.gz", hash = "sha256:45770e8e056d0f283451d9996fbb59b70d45722b45d854d58f32878d0a40c38e"}, + {file = "authlib-1.6.9-py2.py3-none-any.whl", hash = "sha256:f08b4c14e08f0861dc18a32357b33fbcfd2ea86cfe3fe149484b4d764c4a0ac3"}, + {file = "authlib-1.6.9.tar.gz", hash = "sha256:d8f2421e7e5980cc1ddb4e32d3f5fa659cfaf60d8eaf3281ebed192e4ab74f04"}, ] [package.dependencies] @@ -1526,19 +1526,19 @@ typing-extensions = ">=4.6.0" [[package]] name = "azure-mgmt-resource" -version = "23.3.0" +version = "24.0.0" description = "Microsoft Azure Resource Management Client Library for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "azure_mgmt_resource-23.3.0-py3-none-any.whl", hash = "sha256:ab216ee28e29db6654b989746e0c85a1181f66653929d2cb6e48fba66d9af323"}, - {file = "azure_mgmt_resource-23.3.0.tar.gz", hash = "sha256:fc4f1fd8b6aad23f8af4ed1f913df5f5c92df117449dc354fea6802a2829fea4"}, + {file = "azure_mgmt_resource-24.0.0-py3-none-any.whl", hash = "sha256:27b32cd223e2784269f5a0db3c282042886ee4072d79cedc638438ece7cd0df4"}, + {file = "azure_mgmt_resource-24.0.0.tar.gz", hash = "sha256:cf6b8995fcdd407ac9ff1dd474087129429a1d90dbb1ac77f97c19b96237b265"}, ] [package.dependencies] azure-common = ">=1.1" -azure-mgmt-core = ">=1.3.2" +azure-mgmt-core = ">=1.5.0" isodate = ">=0.6.1" typing-extensions = ">=4.6.0" @@ -1822,19 +1822,19 @@ crt = ["awscrt (==0.27.6)"] [[package]] name = "cartography" -version = "0.132.0" +version = "0.135.0" description = "Explore assets and their relationships across your technical infrastructure." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cartography-0.132.0-py3-none-any.whl", hash = "sha256:c070aa51d0ab4479cb043cae70b35e7df49f2fb5f1fa95ccf10000bbeb952262"}, - {file = "cartography-0.132.0.tar.gz", hash = "sha256:7c6332bc57fd2629d7b83aee7bd95a7b2edb0d51ef746efa0461399e0b66625c"}, + {file = "cartography-0.135.0-py3-none-any.whl", hash = "sha256:c62c32a6917b8f23a8b98fe2b6c7c4a918b50f55918482966c4dae1cf5f538e1"}, + {file = "cartography-0.135.0.tar.gz", hash = "sha256:3f500cd22c3b392d00e8b49f62acc95fd4dcd559ce514aafe2eb8101133c7a49"}, ] [package.dependencies] adal = ">=1.2.4" -aioboto3 = ">=13.0.0" +aioboto3 = ">=15.0.0" azure-cli-core = ">=2.26.0" azure-identity = ">=1.5.0" azure-keyvault-certificates = ">=4.0.0" @@ -1852,9 +1852,9 @@ azure-mgmt-keyvault = ">=10.0.0" azure-mgmt-logic = ">=10.0.0" azure-mgmt-monitor = ">=3.0.0" azure-mgmt-network = ">=25.0.0" -azure-mgmt-resource = ">=10.2.0,<25.0.0" +azure-mgmt-resource = ">=24.0.0,<25" azure-mgmt-security = ">=5.0.0" -azure-mgmt-sql = ">=3.0.1,<4" +azure-mgmt-sql = ">=3.0.1" azure-mgmt-storage = ">=16.0.0" azure-mgmt-synapse = ">=2.0.0" azure-mgmt-web = ">=7.0.0" @@ -1862,38 +1862,39 @@ azure-synapse-artifacts = ">=0.17.0" backoff = ">=2.1.2" boto3 = ">=1.15.1" botocore = ">=1.18.1" -cloudflare = ">=4.1.0,<5.0.0" +cloudflare = ">=4.1.0" crowdstrike-falconpy = ">=0.5.1" -cryptography = "*" -dnspython = ">=1.15.0" -duo-client = "*" -google-api-python-client = ">=1.7.8" +cryptography = ">=45.0.0" +dnspython = ">=2.0.0" +duo-client = ">=5.5.0" +google-api-python-client = ">=2.0.0" google-auth = ">=2.37.0" google-cloud-asset = ">=1.0.0" google-cloud-resource-manager = ">=1.14.2" httpx = ">=0.24.0" kubernetes = ">=22.6.0" -marshmallow = ">=3.0.0rc7" -msgraph-sdk = "*" +marshmallow = ">=4.0.0" +msgraph-sdk = ">=1.53.0" msrestazure = ">=0.6.4" neo4j = ">=6.0.0" oci = ">=2.71.0" okta = "<1.0.0" -packageurl-python = "*" -packaging = "*" +packageurl-python = ">=0.17.0" +packaging = ">=26.0.0" pagerduty = ">=4.0.1" policyuniverse = ">=1.1.0.0" PyJWT = {version = ">=2.0.0", extras = ["crypto"]} -python-dateutil = "*" +python-dateutil = ">=2.9.0" python-digitalocean = ">=1.16.0" pyyaml = ">=5.3.1" requests = ">=2.22.0" scaleway = ">=2.10.0" slack-sdk = ">=3.37.0" -statsd = "*" +statsd = ">=4.0.0" typer = ">=0.9.0" -types-aiobotocore-ecr = "*" -xmltodict = "*" +types-aiobotocore-ecr = ">=3.1.0" +workos = ">=5.44.0" +xmltodict = ">=1.0.0" [[package]] name = "celery" @@ -2469,22 +2470,18 @@ toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "cron-descriptor" -version = "2.0.6" +version = "1.4.5" description = "A Python library that converts cron expressions into human readable strings." optional = false -python-versions = ">=3.9" +python-versions = "*" groups = ["main"] files = [ - {file = "cron_descriptor-2.0.6-py3-none-any.whl", hash = "sha256:3a1c0d837c0e5a32e415f821b36cf758eb92d510e6beff8fbfe4fa16573d93d6"}, - {file = "cron_descriptor-2.0.6.tar.gz", hash = "sha256:e39d2848e1d8913cfb6e3452e701b5eec662ee18bea8cc5aa53ee1a7bb217157"}, + {file = "cron_descriptor-1.4.5-py3-none-any.whl", hash = "sha256:736b3ae9d1a99bc3dbfc5b55b5e6e7c12031e7ba5de716625772f8b02dcd6013"}, + {file = "cron_descriptor-1.4.5.tar.gz", hash = "sha256:f51ce4ffc1d1f2816939add8524f206c376a42c87a5fca3091ce26725b3b1bca"}, ] -[package.dependencies] -typing_extensions = "*" - [package.extras] -dev = ["mypy", "polib", "ruff"] -test = ["pytest"] +dev = ["polib"] [[package]] name = "crowdstrike-falconpy" @@ -2507,62 +2504,74 @@ dev = ["bandit", "coverage", "flake8", "pydocstyle", "pylint", "pytest", "pytest [[package]] name = "cryptography" -version = "44.0.3" +version = "46.0.6" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false -python-versions = "!=3.9.0,!=3.9.1,>=3.7" +python-versions = "!=3.9.0,!=3.9.1,>=3.8" groups = ["main", "dev"] files = [ - {file = "cryptography-44.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d"}, - {file = "cryptography-44.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904"}, - {file = "cryptography-44.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44"}, - {file = "cryptography-44.0.3-cp37-abi3-win32.whl", hash = "sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d"}, - {file = "cryptography-44.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d"}, - {file = "cryptography-44.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f"}, - {file = "cryptography-44.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5"}, - {file = "cryptography-44.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b"}, - {file = "cryptography-44.0.3-cp39-abi3-win32.whl", hash = "sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028"}, - {file = "cryptography-44.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06"}, - {file = "cryptography-44.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5"}, - {file = "cryptography-44.0.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c"}, - {file = "cryptography-44.0.3.tar.gz", hash = "sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053"}, + {file = "cryptography-46.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:64235194bad039a10bb6d2d930ab3323baaec67e2ce36215fd0952fad0930ca8"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:26031f1e5ca62fcb9d1fcb34b2b60b390d1aacaa15dc8b895a9ed00968b97b30"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9a693028b9cbe51b5a1136232ee8f2bc242e4e19d456ded3fa7c86e43c713b4a"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:67177e8a9f421aa2d3a170c3e56eca4e0128883cf52a071a7cbf53297f18b175"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d9528b535a6c4f8ff37847144b8986a9a143585f0540fbcb1a98115b543aa463"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:22259338084d6ae497a19bae5d4c66b7ca1387d3264d1c2c0e72d9e9b6a77b97"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:760997a4b950ff00d418398ad73fbc91aa2894b5c1db7ccb45b4f68b42a63b3c"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:3dfa6567f2e9e4c5dceb8ccb5a708158a2a871052fa75c8b78cb0977063f1507"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:cdcd3edcbc5d55757e5f5f3d330dd00007ae463a7e7aa5bf132d1f22a4b62b19"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:d4e4aadb7fc1f88687f47ca20bb7227981b03afaae69287029da08096853b738"}, + {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2b417edbe8877cda9022dde3a008e2deb50be9c407eef034aeeb3a8b11d9db3c"}, + {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:380343e0653b1c9d7e1f55b52aaa2dbb2fdf2730088d48c43ca1c7c0abb7cc2f"}, + {file = "cryptography-46.0.6-cp311-abi3-win32.whl", hash = "sha256:bcb87663e1f7b075e48c3be3ecb5f0b46c8fc50b50a97cf264e7f60242dca3f2"}, + {file = "cryptography-46.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:6739d56300662c468fddb0e5e291f9b4d084bead381667b9e654c7dd81705124"}, + {file = "cryptography-46.0.6-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:2ef9e69886cbb137c2aef9772c2e7138dc581fad4fcbcf13cc181eb5a3ab6275"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7f417f034f91dcec1cb6c5c35b07cdbb2ef262557f701b4ecd803ee8cefed4f4"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d24c13369e856b94892a89ddf70b332e0b70ad4a5c43cf3e9cb71d6d7ffa1f7b"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:aad75154a7ac9039936d50cf431719a2f8d4ed3d3c277ac03f3339ded1a5e707"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:3c21d92ed15e9cfc6eb64c1f5a0326db22ca9c2566ca46d845119b45b4400361"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4668298aef7cddeaf5c6ecc244c2302a2b8e40f384255505c22875eebb47888b"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:8ce35b77aaf02f3b59c90b2c8a05c73bac12cea5b4e8f3fbece1f5fddea5f0ca"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:c89eb37fae9216985d8734c1afd172ba4927f5a05cfd9bf0e4863c6d5465b013"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:ed418c37d095aeddf5336898a132fba01091f0ac5844e3e8018506f014b6d2c4"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:69cf0056d6947edc6e6760e5f17afe4bea06b56a9ac8a06de9d2bd6b532d4f3a"}, + {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8e7304c4f4e9490e11efe56af6713983460ee0780f16c63f219984dab3af9d2d"}, + {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b928a3ca837c77a10e81a814a693f2295200adb3352395fad024559b7be7a736"}, + {file = "cryptography-46.0.6-cp314-cp314t-win32.whl", hash = "sha256:97c8115b27e19e592a05c45d0dd89c57f81f841cc9880e353e0d3bf25b2139ed"}, + {file = "cryptography-46.0.6-cp314-cp314t-win_amd64.whl", hash = "sha256:c797e2517cb7880f8297e2c0f43bb910e91381339336f75d2c1c2cbf811b70b4"}, + {file = "cryptography-46.0.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:12cae594e9473bca1a7aceb90536060643128bb274fcea0fc459ab90f7d1ae7a"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:639301950939d844a9e1c4464d7e07f902fe9a7f6b215bb0d4f28584729935d8"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ed3775295fb91f70b4027aeba878d79b3e55c0b3e97eaa4de71f8f23a9f2eb77"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8927ccfbe967c7df312ade694f987e7e9e22b2425976ddbf28271d7e58845290"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b12c6b1e1651e42ab5de8b1e00dc3b6354fdfd778e7fa60541ddacc27cd21410"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:063b67749f338ca9c5a0b7fe438a52c25f9526b851e24e6c9310e7195aad3b4d"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:02fad249cb0e090b574e30b276a3da6a149e04ee2f049725b1f69e7b8351ec70"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e6142674f2a9291463e5e150090b95a8519b2fb6e6aaec8917dd8d094ce750d"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:456b3215172aeefb9284550b162801d62f5f264a081049a3e94307fe20792cfa"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:341359d6c9e68834e204ceaf25936dffeafea3829ab80e9503860dcc4f4dac58"}, + {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9a9c42a2723999a710445bc0d974e345c32adfd8d2fac6d8a251fa829ad31cfb"}, + {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6617f67b1606dfd9fe4dbfa354a9508d4a6d37afe30306fe6c101b7ce3274b72"}, + {file = "cryptography-46.0.6-cp38-abi3-win32.whl", hash = "sha256:7f6690b6c55e9c5332c0b59b9c8a3fb232ebf059094c17f9019a51e9827df91c"}, + {file = "cryptography-46.0.6-cp38-abi3-win_amd64.whl", hash = "sha256:79e865c642cfc5c0b3eb12af83c35c5aeff4fa5c672dc28c43721c2c9fdd2f0f"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:2ea0f37e9a9cf0df2952893ad145fd9627d326a59daec9b0802480fa3bcd2ead"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a3e84d5ec9ba01f8fd03802b2147ba77f0c8f2617b2aff254cedd551844209c8"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:12f0fa16cc247b13c43d56d7b35287ff1569b5b1f4c5e87e92cc4fcc00cd10c0"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:50575a76e2951fe7dbd1f56d181f8c5ceeeb075e9ff88e7ad997d2f42af06e7b"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:90e5f0a7b3be5f40c3a0a0eafb32c681d8d2c181fc2a1bdabe9b3f611d9f6b1a"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6728c49e3b2c180ef26f8e9f0a883a2c585638db64cf265b49c9ba10652d430e"}, + {file = "cryptography-46.0.6.tar.gz", hash = "sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759"}, ] [package.dependencies] -cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} +cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0) ; python_version >= \"3.8\""] +docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_version >= \"3.8\""] -pep8test = ["check-sdist ; python_version >= \"3.8\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +nox = ["nox[uv] (>=2024.4.15)"] +pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==44.0.3)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test = ["certifi (>=2024)", "cryptography-vectors (==46.0.6)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] [[package]] @@ -2700,23 +2709,17 @@ files = [ ] [[package]] -name = "deprecated" -version = "1.3.1" -description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +name = "defusedxml" +version = "0.7.1" +description = "XML bomb protection for Python stdlib modules" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] files = [ - {file = "deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f"}, - {file = "deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223"}, + {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, + {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, ] -[package.dependencies] -wrapt = ">=1.10,<3" - -[package.extras] -dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"] - [[package]] name = "detect-secrets" version = "1.5.0" @@ -2807,14 +2810,14 @@ bcrypt = ["bcrypt"] [[package]] name = "django-allauth" -version = "65.14.0" +version = "65.15.0" description = "Integrated set of Django applications addressing authentication, registration, account management as well as 3rd party (social) account authentication." optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "django_allauth-65.14.0-py3-none-any.whl", hash = "sha256:448f5f7877f95fcbe1657256510fe7822d7871f202521a29e23ef937f3325a97"}, - {file = "django_allauth-65.14.0.tar.gz", hash = "sha256:5529227aba2b1377d900e9274a3f24496c645e65400fbae3cad5789944bc4d0b"}, + {file = "django_allauth-65.15.0-py3-none-any.whl", hash = "sha256:ad9fc49c49a9368eaa5bb95456b76e2a4f377b3c6862ee8443507816578c098d"}, + {file = "django_allauth-65.15.0.tar.gz", hash = "sha256:b404d48cf0c3ee14dacc834c541f30adedba2ff1c433980ecc494d6cb0b395a8"}, ] [package.dependencies] @@ -2837,20 +2840,20 @@ steam = ["python3-openid (>=3.0.8,<4)"] [[package]] name = "django-celery-beat" -version = "2.8.1" +version = "2.9.0" description = "Database-backed Periodic Tasks." optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "django_celery_beat-2.8.1-py3-none-any.whl", hash = "sha256:da2b1c6939495c05a551717509d6e3b79444e114a027f7b77bf3727c2a39d171"}, - {file = "django_celery_beat-2.8.1.tar.gz", hash = "sha256:dfad0201c0ac50c91a34700ef8fa0a10ee098cc7f3375fe5debed79f2204f80a"}, + {file = "django_celery_beat-2.9.0-py3-none-any.whl", hash = "sha256:4a9e5ebe26d6f8d7215e1fc5c46e466016279dc102435a28141108649bdf2157"}, + {file = "django_celery_beat-2.9.0.tar.gz", hash = "sha256:92404650f52fcb44cf08e2b09635cb1558327c54b1a5d570f0e2d3a22130934c"}, ] [package.dependencies] celery = ">=5.2.3,<6.0" -cron-descriptor = ">=1.2.32" -Django = ">=2.2,<6.0" +cron-descriptor = ">=1.2.32,<2.0.0" +Django = ">=2.2,<6.1" django-timezone-field = ">=5.0" python-crontab = ">=2.3.4" tzdata = "*" @@ -2971,7 +2974,7 @@ files = [ [package.dependencies] autopep8 = "*" Django = ">=4.2" -gprof2dot = ">=2017.09.19" +gprof2dot = ">=2017.9.19" sqlparse = "*" [[package]] @@ -3358,14 +3361,14 @@ files = [ [[package]] name = "flask" -version = "3.1.2" +version = "3.1.3" description = "A simple framework for building complex web applications." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "flask-3.1.2-py3-none-any.whl", hash = "sha256:ca1d8112ec8a6158cc29ea4858963350011b5c846a414cdb7a954aa9e967d03c"}, - {file = "flask-3.1.2.tar.gz", hash = "sha256:bf656c15c80190ed628ad08cdfd3aaa35beb087855e2f494910aa3774cc4fd87"}, + {file = "flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c"}, + {file = "flask-3.1.3.tar.gz", hash = "sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb"}, ] [package.dependencies] @@ -3382,62 +3385,62 @@ dotenv = ["python-dotenv"] [[package]] name = "fonttools" -version = "4.61.1" +version = "4.62.1" description = "Tools to manipulate font files" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "fonttools-4.61.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c7db70d57e5e1089a274cbb2b1fd635c9a24de809a231b154965d415d6c6d24"}, - {file = "fonttools-4.61.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5fe9fd43882620017add5eabb781ebfbc6998ee49b35bd7f8f79af1f9f99a958"}, - {file = "fonttools-4.61.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8db08051fc9e7d8bc622f2112511b8107d8f27cd89e2f64ec45e9825e8288da"}, - {file = "fonttools-4.61.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a76d4cb80f41ba94a6691264be76435e5f72f2cb3cab0b092a6212855f71c2f6"}, - {file = "fonttools-4.61.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a13fc8aeb24bad755eea8f7f9d409438eb94e82cf86b08fe77a03fbc8f6a96b1"}, - {file = "fonttools-4.61.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b846a1fcf8beadeb9ea4f44ec5bdde393e2f1569e17d700bfc49cd69bde75881"}, - {file = "fonttools-4.61.1-cp310-cp310-win32.whl", hash = "sha256:78a7d3ab09dc47ac1a363a493e6112d8cabed7ba7caad5f54dbe2f08676d1b47"}, - {file = "fonttools-4.61.1-cp310-cp310-win_amd64.whl", hash = "sha256:eff1ac3cc66c2ac7cda1e64b4e2f3ffef474b7335f92fc3833fc632d595fcee6"}, - {file = "fonttools-4.61.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c6604b735bb12fef8e0efd5578c9fb5d3d8532d5001ea13a19cddf295673ee09"}, - {file = "fonttools-4.61.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5ce02f38a754f207f2f06557523cd39a06438ba3aafc0639c477ac409fc64e37"}, - {file = "fonttools-4.61.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77efb033d8d7ff233385f30c62c7c79271c8885d5c9657d967ede124671bbdfb"}, - {file = "fonttools-4.61.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:75c1a6dfac6abd407634420c93864a1e274ebc1c7531346d9254c0d8f6ca00f9"}, - {file = "fonttools-4.61.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0de30bfe7745c0d1ffa2b0b7048fb7123ad0d71107e10ee090fa0b16b9452e87"}, - {file = "fonttools-4.61.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:58b0ee0ab5b1fc9921eccfe11d1435added19d6494dde14e323f25ad2bc30c56"}, - {file = "fonttools-4.61.1-cp311-cp311-win32.whl", hash = "sha256:f79b168428351d11e10c5aeb61a74e1851ec221081299f4cf56036a95431c43a"}, - {file = "fonttools-4.61.1-cp311-cp311-win_amd64.whl", hash = "sha256:fe2efccb324948a11dd09d22136fe2ac8a97d6c1347cf0b58a911dcd529f66b7"}, - {file = "fonttools-4.61.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f3cb4a569029b9f291f88aafc927dd53683757e640081ca8c412781ea144565e"}, - {file = "fonttools-4.61.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41a7170d042e8c0024703ed13b71893519a1a6d6e18e933e3ec7507a2c26a4b2"}, - {file = "fonttools-4.61.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:10d88e55330e092940584774ee5e8a6971b01fc2f4d3466a1d6c158230880796"}, - {file = "fonttools-4.61.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15acc09befd16a0fb8a8f62bc147e1a82817542d72184acca9ce6e0aeda9fa6d"}, - {file = "fonttools-4.61.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e6bcdf33aec38d16508ce61fd81838f24c83c90a1d1b8c68982857038673d6b8"}, - {file = "fonttools-4.61.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5fade934607a523614726119164ff621e8c30e8fa1ffffbbd358662056ba69f0"}, - {file = "fonttools-4.61.1-cp312-cp312-win32.whl", hash = "sha256:75da8f28eff26defba42c52986de97b22106cb8f26515b7c22443ebc9c2d3261"}, - {file = "fonttools-4.61.1-cp312-cp312-win_amd64.whl", hash = "sha256:497c31ce314219888c0e2fce5ad9178ca83fe5230b01a5006726cdf3ac9f24d9"}, - {file = "fonttools-4.61.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c56c488ab471628ff3bfa80964372fc13504ece601e0d97a78ee74126b2045c"}, - {file = "fonttools-4.61.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dc492779501fa723b04d0ab1f5be046797fee17d27700476edc7ee9ae535a61e"}, - {file = "fonttools-4.61.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:64102ca87e84261419c3747a0d20f396eb024bdbeb04c2bfb37e2891f5fadcb5"}, - {file = "fonttools-4.61.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c1b526c8d3f615a7b1867f38a9410849c8f4aef078535742198e942fba0e9bd"}, - {file = "fonttools-4.61.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41ed4b5ec103bd306bb68f81dc166e77409e5209443e5773cb4ed837bcc9b0d3"}, - {file = "fonttools-4.61.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b501c862d4901792adaec7c25b1ecc749e2662543f68bb194c42ba18d6eec98d"}, - {file = "fonttools-4.61.1-cp313-cp313-win32.whl", hash = "sha256:4d7092bb38c53bbc78e9255a59158b150bcdc115a1e3b3ce0b5f267dc35dd63c"}, - {file = "fonttools-4.61.1-cp313-cp313-win_amd64.whl", hash = "sha256:21e7c8d76f62ab13c9472ccf74515ca5b9a761d1bde3265152a6dc58700d895b"}, - {file = "fonttools-4.61.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fff4f534200a04b4a36e7ae3cb74493afe807b517a09e99cb4faa89a34ed6ecd"}, - {file = "fonttools-4.61.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d9203500f7c63545b4ce3799319fe4d9feb1a1b89b28d3cb5abd11b9dd64147e"}, - {file = "fonttools-4.61.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fa646ecec9528bef693415c79a86e733c70a4965dd938e9a226b0fc64c9d2e6c"}, - {file = "fonttools-4.61.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11f35ad7805edba3aac1a3710d104592df59f4b957e30108ae0ba6c10b11dd75"}, - {file = "fonttools-4.61.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b931ae8f62db78861b0ff1ac017851764602288575d65b8e8ff1963fed419063"}, - {file = "fonttools-4.61.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b148b56f5de675ee16d45e769e69f87623a4944f7443850bf9a9376e628a89d2"}, - {file = "fonttools-4.61.1-cp314-cp314-win32.whl", hash = "sha256:9b666a475a65f4e839d3d10473fad6d47e0a9db14a2f4a224029c5bfde58ad2c"}, - {file = "fonttools-4.61.1-cp314-cp314-win_amd64.whl", hash = "sha256:4f5686e1fe5fce75d82d93c47a438a25bf0d1319d2843a926f741140b2b16e0c"}, - {file = "fonttools-4.61.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e76ce097e3c57c4bcb67c5aa24a0ecdbd9f74ea9219997a707a4061fbe2707aa"}, - {file = "fonttools-4.61.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9cfef3ab326780c04d6646f68d4b4742aae222e8b8ea1d627c74e38afcbc9d91"}, - {file = "fonttools-4.61.1-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a75c301f96db737e1c5ed5fd7d77d9c34466de16095a266509e13da09751bd19"}, - {file = "fonttools-4.61.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:91669ccac46bbc1d09e9273546181919064e8df73488ea087dcac3e2968df9ba"}, - {file = "fonttools-4.61.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c33ab3ca9d3ccd581d58e989d67554e42d8d4ded94ab3ade3508455fe70e65f7"}, - {file = "fonttools-4.61.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:664c5a68ec406f6b1547946683008576ef8b38275608e1cee6c061828171c118"}, - {file = "fonttools-4.61.1-cp314-cp314t-win32.whl", hash = "sha256:aed04cabe26f30c1647ef0e8fbb207516fd40fe9472e9439695f5c6998e60ac5"}, - {file = "fonttools-4.61.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2180f14c141d2f0f3da43f3a81bc8aa4684860f6b0e6f9e165a4831f24e6a23b"}, - {file = "fonttools-4.61.1-py3-none-any.whl", hash = "sha256:17d2bf5d541add43822bcf0c43d7d847b160c9bb01d15d5007d84e2217aaa371"}, - {file = "fonttools-4.61.1.tar.gz", hash = "sha256:6675329885c44657f826ef01d9e4fb33b9158e9d93c537d84ad8399539bc6f69"}, + {file = "fonttools-4.62.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ad5cca75776cd453b1b035b530e943334957ae152a36a88a320e779d61fc980c"}, + {file = "fonttools-4.62.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0b3ae47e8636156a9accff64c02c0924cbebad62854c4a6dbdc110cd5b4b341a"}, + {file = "fonttools-4.62.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9b9e288b4da2f64fd6180644221749de651703e8d0c16bd4b719533a3a7d6e3"}, + {file = "fonttools-4.62.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bca7a1c1faf235ffe25d4f2e555246b4750220b38de8261d94ebc5ce8a23c23"}, + {file = "fonttools-4.62.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b4e0fcf265ad26e487c56cb12a42dffe7162de708762db951e1b3f755319507d"}, + {file = "fonttools-4.62.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2d850f66830a27b0d498ee05adb13a3781637b1826982cd7e2b3789ef0cc71ae"}, + {file = "fonttools-4.62.1-cp310-cp310-win32.whl", hash = "sha256:486f32c8047ccd05652aba17e4a8819a3a9d78570eb8a0e3b4503142947880ed"}, + {file = "fonttools-4.62.1-cp310-cp310-win_amd64.whl", hash = "sha256:5a648bde915fba9da05ae98856987ca91ba832949a9e2888b48c47ef8b96c5a9"}, + {file = "fonttools-4.62.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:40975849bac44fb0b9253d77420c6d8b523ac4dcdcefeff6e4d706838a5b80f7"}, + {file = "fonttools-4.62.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9dde91633f77fa576879a0c76b1d89de373cae751a98ddf0109d54e173b40f14"}, + {file = "fonttools-4.62.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6acb4109f8bee00fec985c8c7afb02299e35e9c94b57287f3ea542f28bd0b0a7"}, + {file = "fonttools-4.62.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1c5c25671ce8805e0d080e2ffdeca7f1e86778c5cbfbeae86d7f866d8830517b"}, + {file = "fonttools-4.62.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a5d8825e1140f04e6c99bb7d37a9e31c172f3bc208afbe02175339e699c710e1"}, + {file = "fonttools-4.62.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:268abb1cb221e66c014acc234e872b7870d8b5d4657a83a8f4205094c32d2416"}, + {file = "fonttools-4.62.1-cp311-cp311-win32.whl", hash = "sha256:942b03094d7edbb99bdf1ae7e9090898cad7bf9030b3d21f33d7072dbcb51a53"}, + {file = "fonttools-4.62.1-cp311-cp311-win_amd64.whl", hash = "sha256:e8514f4924375f77084e81467e63238b095abda5107620f49421c368a6017ed2"}, + {file = "fonttools-4.62.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:90365821debbd7db678809c7491ca4acd1e0779b9624cdc6ddaf1f31992bf974"}, + {file = "fonttools-4.62.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12859ff0b47dd20f110804c3e0d0970f7b832f561630cd879969011541a464a9"}, + {file = "fonttools-4.62.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c125ffa00c3d9003cdaaf7f2c79e6e535628093e14b5de1dccb08859b680936"}, + {file = "fonttools-4.62.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:149f7d84afca659d1a97e39a4778794a2f83bf344c5ee5134e09995086cc2392"}, + {file = "fonttools-4.62.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0aa72c43a601cfa9273bb1ae0518f1acadc01ee181a6fc60cd758d7fdadffc04"}, + {file = "fonttools-4.62.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:19177c8d96c7c36359266e571c5173bcee9157b59cfc8cb0153c5673dc5a3a7d"}, + {file = "fonttools-4.62.1-cp312-cp312-win32.whl", hash = "sha256:a24decd24d60744ee8b4679d38e88b8303d86772053afc29b19d23bb8207803c"}, + {file = "fonttools-4.62.1-cp312-cp312-win_amd64.whl", hash = "sha256:9e7863e10b3de72376280b515d35b14f5eeed639d1aa7824f4cf06779ec65e42"}, + {file = "fonttools-4.62.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c22b1014017111c401469e3acc5433e6acf6ebcc6aa9efb538a533c800971c79"}, + {file = "fonttools-4.62.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68959f5fc58ed4599b44aad161c2837477d7f35f5f79402d97439974faebfebe"}, + {file = "fonttools-4.62.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef46db46c9447103b8f3ff91e8ba009d5fe181b1920a83757a5762551e32bb68"}, + {file = "fonttools-4.62.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6706d1cb1d5e6251a97ad3c1b9347505c5615c112e66047abbef0f8545fa30d1"}, + {file = "fonttools-4.62.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2e7abd2b1e11736f58c1de27819e1955a53267c21732e78243fa2fa2e5c1e069"}, + {file = "fonttools-4.62.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:403d28ce06ebfc547fbcb0cb8b7f7cc2f7a2d3e1a67ba9a34b14632df9e080f9"}, + {file = "fonttools-4.62.1-cp313-cp313-win32.whl", hash = "sha256:93c316e0f5301b2adbe6a5f658634307c096fd5aae60a5b3412e4f3e1728ab24"}, + {file = "fonttools-4.62.1-cp313-cp313-win_amd64.whl", hash = "sha256:7aa21ff53e28a9c2157acbc44e5b401149d3c9178107130e82d74ceb500e5056"}, + {file = "fonttools-4.62.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fa1d16210b6b10a826d71bed68dd9ec24a9e218d5a5e2797f37c573e7ec215ca"}, + {file = "fonttools-4.62.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:aa69d10ed420d8121118e628ad47d86e4caa79ba37f968597b958f6cceab7eca"}, + {file = "fonttools-4.62.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd13b7999d59c5eb1c2b442eb2d0c427cb517a0b7a1f5798fc5c9e003f5ff782"}, + {file = "fonttools-4.62.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d337fdd49a79b0d51c4da87bc38169d21c3abbf0c1aa9367eff5c6656fb6dae"}, + {file = "fonttools-4.62.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d241cdc4a67b5431c6d7f115fdf63335222414995e3a1df1a41e1182acd4bcc7"}, + {file = "fonttools-4.62.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c05557a78f8fa514da0f869556eeda40887a8abc77c76ee3f74cf241778afd5a"}, + {file = "fonttools-4.62.1-cp314-cp314-win32.whl", hash = "sha256:49a445d2f544ce4a69338694cad575ba97b9a75fff02720da0882d1a73f12800"}, + {file = "fonttools-4.62.1-cp314-cp314-win_amd64.whl", hash = "sha256:1eecc128c86c552fb963fe846ca4e011b1be053728f798185a1687502f6d398e"}, + {file = "fonttools-4.62.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1596aeaddf7f78e21e68293c011316a25267b3effdaccaf4d59bc9159d681b82"}, + {file = "fonttools-4.62.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8f8fca95d3bb3208f59626a4b0ea6e526ee51f5a8ad5d91821c165903e8d9260"}, + {file = "fonttools-4.62.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee91628c08e76f77b533d65feb3fbe6d9dad699f95be51cf0d022db94089cdc4"}, + {file = "fonttools-4.62.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5f37df1cac61d906e7b836abe356bc2f34c99d4477467755c216b72aa3dc748b"}, + {file = "fonttools-4.62.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:92bb00a947e666169c99b43753c4305fc95a890a60ef3aeb2a6963e07902cc87"}, + {file = "fonttools-4.62.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:bdfe592802ef939a0e33106ea4a318eeb17822c7ee168c290273cbd5fabd746c"}, + {file = "fonttools-4.62.1-cp314-cp314t-win32.whl", hash = "sha256:b820fcb92d4655513d8402d5b219f94481c4443d825b4372c75a2072aa4b357a"}, + {file = "fonttools-4.62.1-cp314-cp314t-win_amd64.whl", hash = "sha256:59b372b4f0e113d3746b88985f1c796e7bf830dd54b28374cd85c2b8acd7583e"}, + {file = "fonttools-4.62.1-py3-none-any.whl", hash = "sha256:7487782e2113861f4ddcc07c3436450659e3caa5e470b27dc2177cade2d8e7fd"}, + {file = "fonttools-4.62.1.tar.gz", hash = "sha256:e54c75fd6041f1122476776880f7c3c3295ffa31962dc6ebe2543c00dca58b5d"}, ] [package.extras] @@ -3750,19 +3753,19 @@ urllib3 = ["packaging", "urllib3"] [[package]] name = "google-auth-httplib2" -version = "0.2.1" +version = "0.2.0" description = "Google Authentication Library: httplib2 transport" optional = false -python-versions = ">=3.7" +python-versions = "*" groups = ["main"] files = [ - {file = "google_auth_httplib2-0.2.1-py3-none-any.whl", hash = "sha256:1be94c611db91c01f9703e7f62b0a59bbd5587a95571c7b6fade510d648bc08b"}, - {file = "google_auth_httplib2-0.2.1.tar.gz", hash = "sha256:5ef03be3927423c87fb69607b42df23a444e434ddb2555b73b3679793187b7de"}, + {file = "google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05"}, + {file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"}, ] [package.dependencies] -google-auth = ">=1.32.0,<3.0.0" -httplib2 = ">=0.19.0,<1.0.0" +google-auth = "*" +httplib2 = ">=0.19.0" [[package]] name = "google-cloud-access-context-manager" @@ -4579,7 +4582,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -4787,7 +4790,7 @@ librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""] mongodb = ["pymongo (==4.15.3)"] msgpack = ["msgpack (==1.1.2)"] pyro = ["pyro4 (==4.82)"] -qpid = ["qpid-python (==1.36.0-1)", "qpid-tools (==1.36.0-1)"] +qpid = ["qpid-python (==1.36.0.post1)", "qpid-tools (==1.36.0.post1)"] redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<6.5)"] slmq = ["softlayer_messaging (>=1.0.3)"] sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] @@ -4808,7 +4811,7 @@ files = [ ] [package.dependencies] -certifi = ">=14.05.14" +certifi = ">=14.5.14" durationpy = ">=0.7" google-auth = ">=1.0.1" oauthlib = ">=3.2.2" @@ -5052,18 +5055,18 @@ tests = ["psutil", "pytest (!=3.3.0)", "pytest-cov"] [[package]] name = "markdown" -version = "3.9" +version = "3.10.2" description = "Python implementation of John Gruber's Markdown." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280"}, - {file = "markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a"}, + {file = "markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36"}, + {file = "markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950"}, ] [package.extras] -docs = ["mdx_gh_links (>=0.2)", "mkdocs (>=1.6)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] +docs = ["mdx_gh_links (>=0.2)", "mkdocs (>=1.6)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python] (>=0.28.3)"] testing = ["coverage", "pyyaml"] [[package]] @@ -5191,24 +5194,16 @@ files = [ [[package]] name = "marshmallow" -version = "3.26.2" +version = "4.3.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73"}, - {file = "marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57"}, + {file = "marshmallow-4.3.0-py3-none-any.whl", hash = "sha256:46c4fe6984707e3cbd485dfebbf0a59874f58d695aad05c1668d15e8c6e13b46"}, + {file = "marshmallow-4.3.0.tar.gz", hash = "sha256:fb43c53b3fe240b8f6af37223d6ef1636f927ad9bea8ab323afad95dff090880"}, ] -[package.dependencies] -packaging = ">=17.0" - -[package.extras] -dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"] -docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"] -tests = ["pytest", "simplejson"] - [[package]] name = "matplotlib" version = "3.10.8" @@ -5502,14 +5497,14 @@ dev = ["bumpver", "isort", "mypy", "pylint", "pytest", "yapf"] [[package]] name = "msgraph-sdk" -version = "1.23.0" +version = "1.55.0" description = "The Microsoft Graph Python SDK" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "msgraph_sdk-1.23.0-py3-none-any.whl", hash = "sha256:58e0047b4ca59fd82022c02cd73fec0170a3d84f3b76721e3db2a0314df9a58a"}, - {file = "msgraph_sdk-1.23.0.tar.gz", hash = "sha256:6dd1ba9a46f5f0ce8599fd9610133adbd9d1493941438b5d3632fce9e55ed607"}, + {file = "msgraph_sdk-1.55.0-py3-none-any.whl", hash = "sha256:c8e68ebc4b88af5111de312e7fa910a4e76ddf48a4534feadb1fb8a411c48cfc"}, + {file = "msgraph_sdk-1.55.0.tar.gz", hash = "sha256:6df691a31954a050d26b8a678968017e157d940fb377f2a8a4e17a9741b98756"}, ] [package.dependencies] @@ -5837,14 +5832,14 @@ files = [ [[package]] name = "nltk" -version = "3.9.2" +version = "3.9.4" description = "Natural Language Toolkit" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "nltk-3.9.2-py3-none-any.whl", hash = "sha256:1e209d2b3009110635ed9709a67a1a3e33a10f799490fa71cf4bec218c11c88a"}, - {file = "nltk-3.9.2.tar.gz", hash = "sha256:0f409e9b069ca4177c1903c3e843eef90c7e92992fa4931ae607da6de49e1419"}, + {file = "nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f"}, + {file = "nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0"}, ] [package.dependencies] @@ -5935,23 +5930,24 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "oci" -version = "2.160.3" +version = "2.169.0" description = "Oracle Cloud Infrastructure Python SDK" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "oci-2.160.3-py3-none-any.whl", hash = "sha256:858bff3e697098bdda44833d2476bfb4632126f0182178e7dbde4dbd156d71f0"}, - {file = "oci-2.160.3.tar.gz", hash = "sha256:57514889be3b713a8385d86e3ba8a33cf46e3563c2a7e29a93027fb30b8a2537"}, + {file = "oci-2.169.0-py3-none-any.whl", hash = "sha256:c71bb5143f307791082b3e33cc1545c2490a518cfed85ab1948ef5107c36d30b"}, + {file = "oci-2.169.0.tar.gz", hash = "sha256:f3c5fff00b01783b5325ea7b13bf140053ec1e9f41da20bfb9c8a349ee7662fa"}, ] [package.dependencies] certifi = "*" circuitbreaker = {version = ">=1.3.1,<3.0.0", markers = "python_version >= \"3.7\""} -cryptography = ">=3.2.1,<46.0.0" -pyOpenSSL = ">=17.5.0,<25.0.0" +cryptography = ">=3.2.1,<47.0.0" +pyOpenSSL = ">=17.5.0,<27.0.0" python-dateutil = ">=2.5.3,<3.0.0" pytz = ">=2016.10" +urllib3 = {version = ">=2.6.3", markers = "python_version >= \"3.10.0\""} [package.extras] adk = ["docstring-parser (>=0.16) ; python_version >= \"3.10\" and python_version < \"4\"", "mcp (>=1.6.0) ; python_version >= \"3.10\" and python_version < \"4\"", "pydantic (>=2.10.6) ; python_version >= \"3.10\" and python_version < \"4\"", "rich (>=13.9.4) ; python_version >= \"3.10\" and python_version < \"4\""] @@ -6455,6 +6451,33 @@ docs = ["sphinx (>=1.7.1)"] redis = ["redis"] tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)", "types-redis"] +[[package]] +name = "prek" +version = "0.3.9" +description = "A Git hook manager written in Rust, designed as a drop-in alternative to pre-commit." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "prek-0.3.9-py3-none-linux_armv6l.whl", hash = "sha256:3ed793d51bfaa27bddb64d525d7acb77a7c8644f549412d82252e3eb0b88aad8"}, + {file = "prek-0.3.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:399c58400c0bd0b82a93a3c09dc1bfd88d8d0cfb242d414d2ed247187b06ead1"}, + {file = "prek-0.3.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ea1ffb124e92f081b8e2ca5b5a623a733efb3be0c5b1f4b7ffe2ee17d1f20c"}, + {file = "prek-0.3.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:aaf639f95b7301639298311d8d44aad0d0b4864e9736083ad3c71ce9765d37ab"}, + {file = "prek-0.3.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff104863b187fa443ea8451ca55d51e2c6e94f99f00d88784b5c3c4c623f1ebe"}, + {file = "prek-0.3.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:039ecaf87c63a3e67cca645ebd5bc5eb6aafa6c9d929e9a27b2921e7849d7ef9"}, + {file = "prek-0.3.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3bde2a3d045705095983c7f78ba04f72a7565fe1c2b4e85f5628502a254754ff"}, + {file = "prek-0.3.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28a0960a21543563e2c8e19aaad176cc8423a87aac3c914d0f313030d7a9244a"}, + {file = "prek-0.3.9-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:0dfb5d5171d7523271909246ee306b4dc3d5b63752e7dd7c7e8a8908fc9490d1"}, + {file = "prek-0.3.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82b791bd36c1430c84d3ae7220a85152babc7eaf00f70adcb961bd594e756ba3"}, + {file = "prek-0.3.9-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:6eac6d2f736b041118f053a1487abed468a70dd85a8688eaf87bb42d3dcecf20"}, + {file = "prek-0.3.9-py3-none-musllinux_1_1_i686.whl", hash = "sha256:5517e46e761367a3759b3168eabc120840ffbca9dfbc53187167298a98f87dc4"}, + {file = "prek-0.3.9-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:92024778cf78683ca32687bb249ab6a7d5c33887b5ee1d1a9f6d0c14228f4cf3"}, + {file = "prek-0.3.9-py3-none-win32.whl", hash = "sha256:7f89c55e5f480f5d073769e319924ad69d4bf9f98c5cb46a83082e26e634c958"}, + {file = "prek-0.3.9-py3-none-win_amd64.whl", hash = "sha256:7722f3372eaa83b147e70a43cb7b9fe2128c13d0c78d8a1cdbf2a8ec2ee071eb"}, + {file = "prek-0.3.9-py3-none-win_arm64.whl", hash = "sha256:0bced6278d6cc8a4b46048979e36bc9da034611dc8facd77ab123177b833a929"}, + {file = "prek-0.3.9.tar.gz", hash = "sha256:f82b92d81f42f1f90a47f5fbbf492373e25ef1f790080215b2722dd6da66510e"}, +] + [[package]] name = "prompt-toolkit" version = "3.0.52" @@ -6642,10 +6665,10 @@ files = [ [[package]] name = "prowler" -version = "5.19.0" +version = "5.25.0" description = "Prowler is an Open Source security tool to perform AWS, GCP and Azure security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMP, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme) and your custom security frameworks." optional = false -python-versions = ">3.9.1,<3.13" +python-versions = ">=3.10,<3.13" groups = ["main"] files = [] develop = false @@ -6662,7 +6685,7 @@ alibabacloud-rds20140815 = "12.0.0" alibabacloud_sas20181203 = "6.1.0" alibabacloud-sls20201230 = "5.9.0" alibabacloud_sts20150401 = "1.1.6" -alibabacloud_tea_openapi = "0.4.1" +alibabacloud_tea_openapi = "0.4.4" alibabacloud_vpc20160428 = "6.13.0" alive-progress = "3.3.0" awsipranges = "0.3.3" @@ -6684,7 +6707,7 @@ azure-mgmt-postgresqlflexibleservers = "1.1.0" azure-mgmt-rdbms = "10.1.0" azure-mgmt-recoveryservices = "3.1.0" azure-mgmt-recoveryservicesbackup = "9.2.0" -azure-mgmt-resource = "23.3.0" +azure-mgmt-resource = "24.0.0" azure-mgmt-search = "9.1.0" azure-mgmt-security = "7.0.0" azure-mgmt-sql = "3.0.1" @@ -6697,40 +6720,42 @@ boto3 = "1.40.61" botocore = "1.40.61" cloudflare = "4.3.1" colorama = "0.4.6" -cryptography = "44.0.3" +cryptography = "46.0.6" dash = "3.1.1" dash-bootstrap-components = "2.0.3" +defusedxml = "0.7.1" detect-secrets = "1.5.0" dulwich = "0.23.0" google-api-python-client = "2.163.0" -google-auth-httplib2 = ">=0.1,<0.3" +google-auth-httplib2 = "0.2.0" h2 = "4.3.0" jsonschema = "4.23.0" kubernetes = "32.0.1" -markdown = "3.9.0" +markdown = "3.10.2" microsoft-kiota-abstractions = "1.9.2" -msgraph-sdk = "1.23.0" +msgraph-sdk = "1.55.0" numpy = "2.0.2" -oci = "2.160.3" +oci = "2.169.0" openstacksdk = "4.2.0" pandas = "2.2.3" py-iam-expand = "0.1.0" py-ocsf-models = "0.8.1" -pydantic = ">=2.0,<3.0" -pygithub = "2.5.0" -python-dateutil = ">=2.9.0.post0,<3.0.0" +pydantic = "2.12.5" +pygithub = "2.8.0" +python-dateutil = "2.9.0.post0" pytz = "2025.1" schema = "0.7.5" shodan = "1.31.0" slack-sdk = "3.39.0" tabulate = "0.9.0" tzlocal = "5.3.1" +uuid6 = "2024.7.10" [package.source] type = "git" url = "https://github.com/prowler-cloud/prowler.git" reference = "master" -resolved_reference = "6962622fd21401886371add25463f77228cd9c1f" +resolved_reference = "ca29e354b622198ff6a70e2ea5eb04e4a44a0903" [[package]] name = "psutil" @@ -6895,14 +6920,14 @@ pydantic = ">=2.12.0,<3.0.0" [[package]] name = "pyasn1" -version = "0.6.2" +version = "0.6.3" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf"}, - {file = "pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b"}, + {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, + {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, ] [[package]] @@ -7103,34 +7128,33 @@ typing-extensions = ">=4.14.1" [[package]] name = "pygithub" -version = "2.5.0" +version = "2.8.0" description = "Use the full Github API v3" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "PyGithub-2.5.0-py3-none-any.whl", hash = "sha256:b0b635999a658ab8e08720bdd3318893ff20e2275f6446fcf35bf3f44f2c0fd2"}, - {file = "pygithub-2.5.0.tar.gz", hash = "sha256:e1613ac508a9be710920d26eb18b1905ebd9926aa49398e88151c1b526aad3cf"}, + {file = "pygithub-2.8.0-py3-none-any.whl", hash = "sha256:11a3473c1c2f1c39c525d0ee8c559f369c6d46c272cb7321c9b0cabc7aa1ce7d"}, + {file = "pygithub-2.8.0.tar.gz", hash = "sha256:72f5f2677d86bc3a8843aa720c6ce4c1c42fb7500243b136e3d5e14ddb5c3386"}, ] [package.dependencies] -Deprecated = "*" pyjwt = {version = ">=2.4.0", extras = ["crypto"]} pynacl = ">=1.4.0" requests = ">=2.14.0" -typing-extensions = ">=4.0.0" +typing-extensions = ">=4.5.0" urllib3 = ">=1.26.0" [[package]] name = "pygments" -version = "2.19.2" +version = "2.20.0" description = "Pygments is a syntax highlighting package written in Python." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, - {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, + {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, + {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, ] [package.extras] @@ -7138,14 +7162,14 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyjwt" -version = "2.11.0" +version = "2.12.1" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pyjwt-2.11.0-py3-none-any.whl", hash = "sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469"}, - {file = "pyjwt-2.11.0.tar.gz", hash = "sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623"}, + {file = "pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c"}, + {file = "pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b"}, ] [package.dependencies] @@ -7170,7 +7194,7 @@ files = [ ] [package.dependencies] -astroid = ">=3.2.2,<=3.3.0-dev0" +astroid = ">=3.2.2,<=3.3.0.dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, @@ -7192,7 +7216,7 @@ description = "The MSALRuntime Python Interop Package" optional = false python-versions = ">=3.6" groups = ["main"] -markers = "(platform_system == \"Windows\" or platform_system == \"Darwin\" or platform_system == \"Linux\") and sys_platform == \"win32\"" +markers = "sys_platform == \"win32\" and (platform_system == \"Windows\" or platform_system == \"Darwin\" or platform_system == \"Linux\")" files = [ {file = "pymsalruntime-0.18.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:0c22e2e83faa10de422bbfaacc1bb2887c9025ee8a53f0fc2e4f7db01c4a7b66"}, {file = "pymsalruntime-0.18.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:8ce2944a0f944833d047bb121396091e00287e2b6373716106da86ea99abf379"}, @@ -7270,18 +7294,19 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=7.4.0)", "pytest-cov (>=2.10.1)", " [[package]] name = "pyopenssl" -version = "24.3.0" +version = "26.0.0" description = "Python wrapper module around the OpenSSL library" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyOpenSSL-24.3.0-py3-none-any.whl", hash = "sha256:e474f5a473cd7f92221cc04976e48f4d11502804657a08a989fb3be5514c904a"}, - {file = "pyopenssl-24.3.0.tar.gz", hash = "sha256:49f7a019577d834746bc55c5fce6ecbcec0f2b4ec5ce1cf43a9a173b8138bb36"}, + {file = "pyopenssl-26.0.0-py3-none-any.whl", hash = "sha256:df94d28498848b98cc1c0ffb8ef1e71e40210d3b0a8064c9d29571ed2904bf81"}, + {file = "pyopenssl-26.0.0.tar.gz", hash = "sha256:f293934e52936f2e3413b89c6ce36df66a0b34ae1ea3a053b8c5020ff2f513fc"}, ] [package.dependencies] -cryptography = ">=41.0.5,<45" +cryptography = ">=46.0.0,<47" +typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and python_version >= \"3.8\""} [package.extras] docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] @@ -7333,35 +7358,36 @@ files = [ [[package]] name = "pytest" -version = "8.2.2" +version = "9.0.3" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, - {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, + {file = "pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9"}, + {file = "pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c"}, ] [package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=1.5,<2.0" +colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} +iniconfig = ">=1.0.1" +packaging = ">=22" +pluggy = ">=1.5,<2" +pygments = ">=2.7.2" [package.extras] -dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-celery" -version = "1.2.1" +version = "1.3.0" description = "Pytest plugin for Celery" optional = false -python-versions = "<4.0,>=3.8" +python-versions = "<4.0,>=3.9" groups = ["main"] files = [ - {file = "pytest_celery-1.2.1-py3-none-any.whl", hash = "sha256:0441ab0c2a712b775be16ffda3d7deb31995fd7b5e9d71630e7ea98b474346a3"}, - {file = "pytest_celery-1.2.1.tar.gz", hash = "sha256:7873fb3cf4fbfe9b0dd15d359bdb8bbab4a41c7e48f5b0adb7d36138d3704d52"}, + {file = "pytest_celery-1.3.0-py3-none-any.whl", hash = "sha256:f02201d7770584a0c412a1ded329a142170c24012467c7046f2c72cc8205ad5d"}, + {file = "pytest_celery-1.3.0.tar.gz", hash = "sha256:bd9e5b0f594ec5de9ab97cf27e3a11c644718a761bab6b997d01800fd7394f64"}, ] [package.dependencies] @@ -7372,14 +7398,13 @@ kombu = "*" psutil = ">=7.0.0" pytest-docker-tools = ">=3.1.3" redis = {version = "*", optional = true, markers = "extra == \"all\" or extra == \"redis\""} -setuptools = {version = ">=75.8.0", markers = "python_version >= \"3.9\" and python_version < \"4.0\""} tenacity = ">=9.0.0" [package.extras] -all = ["boto3", "botocore", "python-memcached", "redis", "urllib3 (>=1.26.16,<2.0)"] +all = ["boto3", "botocore", "pycurl (>=7.43) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\"", "python-memcached", "redis", "urllib3 (>=1.26.16,<2.0)"] memcached = ["python-memcached"] redis = ["redis"] -sqs = ["boto3", "botocore", "urllib3 (>=1.26.16,<2.0)"] +sqs = ["boto3", "botocore", "pycurl (>=7.43) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\"", "urllib3 (>=1.26.16,<2.0)"] [[package]] name = "pytest-cov" @@ -7864,14 +7889,14 @@ files = [ [[package]] name = "reportlab" -version = "4.4.9" +version = "4.4.10" description = "The Reportlab Toolkit" optional = false python-versions = "<4,>=3.9" groups = ["main"] files = [ - {file = "reportlab-4.4.9-py3-none-any.whl", hash = "sha256:68e2d103ae8041a37714e8896ec9b79a1c1e911d68c3bd2ea17546568cf17bfd"}, - {file = "reportlab-4.4.9.tar.gz", hash = "sha256:7cf487764294ee791a4781f5a157bebce262a666ae4bbb87786760a9676c9378"}, + {file = "reportlab-4.4.10-py3-none-any.whl", hash = "sha256:5abc815746ae2bc44e7ff25db96814f921349ca814c992c7eac3c26029bf7c24"}, + {file = "reportlab-4.4.10.tar.gz", hash = "sha256:5cbbb34ac3546039d0086deb2938cdec06b12da3cdb836e813258eb33cd28487"}, ] [package.dependencies] @@ -8184,10 +8209,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.37.4,<2.0a.0" +botocore = ">=1.37.4,<2.0a0" [package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] [[package]] name = "safety" @@ -8293,14 +8318,14 @@ contextlib2 = ">=0.5.5" [[package]] name = "sentry-sdk" -version = "2.51.0" +version = "2.56.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = ">=3.6" groups = ["main"] files = [ - {file = "sentry_sdk-2.51.0-py2.py3-none-any.whl", hash = "sha256:e21016d318a097c2b617bb980afd9fc737e1efc55f9b4f0cdc819982c9717d5f"}, - {file = "sentry_sdk-2.51.0.tar.gz", hash = "sha256:b89d64577075fd8c13088bc3609a2ce77a154e5beb8cba7cc16560b0539df4f7"}, + {file = "sentry_sdk-2.56.0-py2.py3-none-any.whl", hash = "sha256:5afafb744ceb91d22f4cc650c6bd048ac6af5f7412dcc6c59305a2e36f4dbc02"}, + {file = "sentry_sdk-2.56.0.tar.gz", hash = "sha256:fdab72030b69625665b2eeb9738bdde748ad254e8073085a0ce95382678e8168"}, ] [package.dependencies] @@ -8773,14 +8798,14 @@ test = ["pytest", "websockets"] [[package]] name = "werkzeug" -version = "3.1.5" +version = "3.1.7" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "werkzeug-3.1.5-py3-none-any.whl", hash = "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc"}, - {file = "werkzeug-3.1.5.tar.gz", hash = "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67"}, + {file = "werkzeug-3.1.7-py3-none-any.whl", hash = "sha256:4b314d81163a3e1a169b6a0be2a000a0e204e8873c5de6586f453c55688d422f"}, + {file = "werkzeug-3.1.7.tar.gz", hash = "sha256:fb8c01fe6ab13b9b7cdb46892b99b1d66754e1d7ab8e542e865ec13f526b5351"}, ] [package.dependencies] @@ -8789,6 +8814,23 @@ markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] +[[package]] +name = "workos" +version = "6.0.4" +description = "WorkOS Python Client" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "workos-6.0.4-py3-none-any.whl", hash = "sha256:548668b3702673536f853ba72a7b5bbbc269e467aaf9ac4f477b6e0177df5e21"}, + {file = "workos-6.0.4.tar.gz", hash = "sha256:b0bfe8fd212b8567422c4ea3732eb33608794033eb3a69900c6b04db183c32d6"}, +] + +[package.dependencies] +cryptography = ">=46.0,<47.0" +httpx = ">=0.28,<1.0" +pyjwt = ">=2.12,<3.0" + [[package]] name = "wrapt" version = "1.17.3" @@ -9382,4 +9424,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "6e38c38b1f8dc05b881f49703fa445eec299527e6697992b18e4613534fbcdb6" +content-hash = "a3ab982d11a87d951ff15694d2ca7fd51f1f51a451abb0baa067ccf6966367a8" diff --git a/api/pyproject.toml b/api/pyproject.toml index 2dcb958474..838a30fdf7 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -5,43 +5,44 @@ requires = ["poetry-core"] [project] authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}] dependencies = [ - "celery (>=5.4.0,<6.0.0)", + "celery (==5.6.2)", "dj-rest-auth[with_social,jwt] (==7.0.1)", "django (==5.1.15)", - "django-allauth[saml] (>=65.13.0,<66.0.0)", - "django-celery-beat (>=2.7.0,<3.0.0)", - "django-celery-results (>=2.5.1,<3.0.0)", + "django-allauth[saml] (==65.15.0)", + "django-celery-beat (==2.9.0)", + "django-celery-results (==2.6.0)", "django-cors-headers==4.4.0", "django-environ==0.11.2", "django-filter==24.3", "django-guid==3.5.0", - "django-postgres-extra (>=2.0.8,<3.0.0)", + "django-postgres-extra (==2.0.9)", "djangorestframework==3.15.2", "djangorestframework-jsonapi==7.0.2", - "djangorestframework-simplejwt (>=5.3.1,<6.0.0)", - "drf-nested-routers (>=0.94.1,<1.0.0)", + "djangorestframework-simplejwt (==5.5.1)", + "drf-nested-routers (==0.95.0)", "drf-spectacular==0.27.2", "drf-spectacular-jsonapi==0.5.1", + "defusedxml==0.7.1", "gunicorn==23.0.0", "lxml==5.3.2", "prowler @ git+https://github.com/prowler-cloud/prowler.git@master", "psycopg2-binary==2.9.9", - "pytest-celery[redis] (>=1.0.1,<2.0.0)", - "sentry-sdk[django] (>=2.20.0,<3.0.0)", + "pytest-celery[redis] (==1.3.0)", + "sentry-sdk[django] (==2.56.0)", "uuid6==2024.7.10", - "openai (>=1.82.0,<2.0.0)", + "openai (==1.109.1)", "xmlsec==1.3.14", "h2 (==4.3.0)", - "markdown (>=3.9,<4.0)", + "markdown (==3.10.2)", "drf-simple-apikey (==2.2.1)", - "matplotlib (>=3.10.6,<4.0.0)", - "reportlab (>=4.4.4,<5.0.0)", - "neo4j (>=6.0.0,<7.0.0)", - "cartography (==0.132.0)", - "gevent (>=25.9.1,<26.0.0)", - "werkzeug (>=3.1.4)", - "sqlparse (>=0.5.4)", - "fonttools (>=4.60.2)" + "matplotlib (==3.10.8)", + "reportlab (==4.4.10)", + "neo4j (==6.1.0)", + "cartography (==0.135.0)", + "gevent (==25.9.1)", + "werkzeug (==3.1.7)", + "sqlparse (==0.5.5)", + "fonttools (==4.62.1)" ] description = "Prowler's API (Django/DRF)" license = "Apache-2.0" @@ -49,7 +50,7 @@ name = "prowler-api" package-mode = false # Needed for the SDK compatibility requires-python = ">=3.11,<3.13" -version = "1.22.0" +version = "1.27.0" [project.scripts] celery = "src.backend.config.settings.celery" @@ -61,10 +62,9 @@ django-silk = "5.3.2" docker = "7.1.0" filelock = "3.20.3" freezegun = "1.5.1" -marshmallow = ">=3.15.0,<4.0.0" mypy = "1.10.1" pylint = "3.2.5" -pytest = "8.2.2" +pytest = "9.0.3" pytest-cov = "5.0.0" pytest-django = "4.8.0" pytest-env = "1.1.3" @@ -74,3 +74,4 @@ ruff = "0.5.0" safety = "3.7.0" tqdm = "4.67.1" vulture = "2.14" +prek = "0.3.9" diff --git a/api/src/backend/api/apps.py b/api/src/backend/api/apps.py index 543c10ab88..3209f75888 100644 --- a/api/src/backend/api/apps.py +++ b/api/src/backend/api/apps.py @@ -52,7 +52,7 @@ class ApiConfig(AppConfig): "check_and_fix_socialaccount_sites_migration", ] - # Skip Neo4j initialization during tests, some Django commands, and Celery + # Skip eager Neo4j init for tests, some Django commands, and Celery (prefork pool: driver must stay lazy, no post_fork hook) if getattr(settings, "TESTING", False) or ( len(sys.argv) > 1 and ( @@ -64,7 +64,7 @@ class ApiConfig(AppConfig): ) ): logger.info( - "Skipping Neo4j initialization because tests, some Django commands or Celery" + "Skipping eager Neo4j init: tests, some Django commands, or Celery prefork pool (driver stays lazy)" ) else: diff --git a/api/src/backend/api/attack_paths/cypher_sanitizer.py b/api/src/backend/api/attack_paths/cypher_sanitizer.py new file mode 100644 index 0000000000..3772b4cbef --- /dev/null +++ b/api/src/backend/api/attack_paths/cypher_sanitizer.py @@ -0,0 +1,170 @@ +""" +Cypher sanitizer for custom (user-supplied) Attack Paths queries. + +Two responsibilities: + +1. **Validation** - reject queries containing SSRF or dangerous procedure + patterns (defense-in-depth; the primary control is ``neo4j.READ_ACCESS``). + +2. **Provider-scoped label injection** - inject a dynamic + ``_Provider_{uuid}`` label into every node pattern so the database can + use its native label index for provider isolation. + +Label-injection pipeline: + +1. **Protect** string literals and line comments (placeholder replacement). +2. **Split** by top-level clause keywords to track clause context. +3. **Pass A** - inject into *labeled* node patterns in ALL segments. +4. **Pass B** - inject into *bare* node patterns in MATCH segments only. +5. **Restore** protected regions. +""" + +import re + +from rest_framework.exceptions import ValidationError + +from tasks.jobs.attack_paths.config import get_provider_label + + +# Step 1 - String / comment protection +# Single combined regex: strings first, then line comments. +# The regex engine finds the leftmost match, so a string like 'https://prowler.com' +# is consumed as a string before the // inside it can match as a comment. +_PROTECTED_RE = re.compile(r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\"|//[^\n]*") + +# Step 2 - Clause splitting +# OPTIONAL MATCH must come before MATCH to avoid partial matching. +_CLAUSE_RE = re.compile( + r"\b(OPTIONAL\s+MATCH|MATCH|WHERE|RETURN|WITH|ORDER\s+BY" + r"|SKIP|LIMIT|UNION|UNWIND|CALL)\b", + re.IGNORECASE, +) + +# Pass A - Labeled node patterns (all segments) +# Matches node patterns that have at least one :Label. +# (? str: + """Inject provider label into all node patterns that have existing labels.""" + return _LABELED_NODE_RE.sub(rf"(\1:{label}\2", segment) + + +def _inject_bare(segment: str, label: str) -> str: + """Inject provider label into bare `(identifier)` node patterns.""" + + def _replace(match): + var = match.group(1) + props = match.group(2).strip() + if props: + return f"({var}:{label} {props})" + return f"({var}:{label})" + + return _BARE_NODE_RE.sub(_replace, segment) + + +def inject_provider_label(cypher: str, provider_id: str) -> str: + """Rewrite a Cypher query to scope every node pattern to a provider. + + Args: + cypher: The original Cypher query string. + provider_id: The provider UUID (will be converted to a label via + `get_provider_label`). + + Returns: + The rewritten Cypher with `:_Provider_{uuid}` appended to every + node pattern. + """ + label = get_provider_label(provider_id) + + # Step 1: Protect strings and comments (single pass, leftmost-first) + protected: list[str] = [] + + def _save(match): + protected.append(match.group(0)) + return f"\x00P{len(protected) - 1}\x00" + + work = _PROTECTED_RE.sub(_save, cypher) + + # Step 2: Split by clause keywords + parts = _CLAUSE_RE.split(work) + + # Steps 3-4: Apply injection passes per segment + result: list[str] = [] + current_clause: str | None = None + + for i, part in enumerate(parts): + if i % 2 == 1: + # Keyword token - normalize for clause tracking + current_clause = re.sub(r"\s+", " ", part.strip()).upper() + result.append(part) + else: + # Content segment - apply injection based on clause context + part = _inject_labeled(part, label) + if current_clause in _MATCH_CLAUSES: + part = _inject_bare(part, label) + result.append(part) + + work = "".join(result) + + # Step 5: Restore protected regions + for i, original in enumerate(protected): + work = work.replace(f"\x00P{i}\x00", original) + + return work + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + +# Patterns that indicate SSRF or dangerous procedure calls +# Defense-in-depth layer - the primary control is `neo4j.READ_ACCESS` +_BLOCKED_PATTERNS = [ + re.compile(r"\bLOAD\s+CSV\b", re.IGNORECASE), + re.compile(r"\bapoc\.load\b", re.IGNORECASE), + re.compile(r"\bapoc\.import\b", re.IGNORECASE), + re.compile(r"\bapoc\.export\b", re.IGNORECASE), + re.compile(r"\bapoc\.cypher\b", re.IGNORECASE), + re.compile(r"\bapoc\.systemdb\b", re.IGNORECASE), + re.compile(r"\bapoc\.config\b", re.IGNORECASE), + re.compile(r"\bapoc\.periodic\b", re.IGNORECASE), + re.compile(r"\bapoc\.do\b", re.IGNORECASE), + re.compile(r"\bapoc\.trigger\b", re.IGNORECASE), + re.compile(r"\bapoc\.custom\b", re.IGNORECASE), +] + + +def validate_custom_query(cypher: str) -> None: + """Reject queries containing known SSRF or dangerous procedure patterns. + + Raises ValidationError if a blocked pattern is found. + String literals and comments are stripped before matching to avoid + false positives. + """ + stripped = _PROTECTED_RE.sub("", cypher) + for pattern in _BLOCKED_PATTERNS: + if pattern.search(stripped): + raise ValidationError({"query": "Query contains a blocked operation"}) diff --git a/api/src/backend/api/attack_paths/database.py b/api/src/backend/api/attack_paths/database.py index eb82912267..f5fddd0613 100644 --- a/api/src/backend/api/attack_paths/database.py +++ b/api/src/backend/api/attack_paths/database.py @@ -1,25 +1,22 @@ import atexit import logging import threading - -from typing import Any - from contextlib import contextmanager -from typing import Iterator +from typing import Any, Iterator from uuid import UUID import neo4j import neo4j.exceptions - -from django.conf import settings - -from api.attack_paths.retryable_session import RetryableSession from config.env import env +from django.conf import settings from tasks.jobs.attack_paths.config import ( BATCH_SIZE, - DEPRECATED_PROVIDER_RESOURCE_LABEL, + PROVIDER_RESOURCE_LABEL, + get_provider_label, ) +from api.attack_paths.retryable_session import RetryableSession + # Without this Celery goes crazy with Neo4j logging logging.getLogger("neo4j").setLevel(logging.ERROR) logging.getLogger("neo4j").propagate = False @@ -31,6 +28,7 @@ READ_QUERY_TIMEOUT_SECONDS = env.int( "ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30 ) MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250) +CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15) READ_EXCEPTION_CODES = [ "Neo.ClientError.Statement.AccessMode", "Neo.ClientError.Procedure.ProcedureNotFound", @@ -65,7 +63,7 @@ def init_driver() -> neo4j.Driver: auth=(config["USER"], config["PASSWORD"]), keep_alive=True, max_connection_lifetime=7200, - connection_acquisition_timeout=120, + connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT, max_connection_pool_size=50, ) _driver.verify_connectivity() @@ -166,11 +164,8 @@ def drop_subgraph(database: str, provider_id: str) -> int: Uses batched deletion to avoid memory issues with large graphs. Silently returns 0 if the database doesn't exist. """ + provider_label = get_provider_label(provider_id) deleted_nodes = 0 - parameters = { - "provider_id": provider_id, - "batch_size": BATCH_SIZE, - } try: with get_session(database) as session: @@ -178,12 +173,12 @@ def drop_subgraph(database: str, provider_id: str) -> int: while deleted_count > 0: result = session.run( f""" - MATCH (n:{DEPRECATED_PROVIDER_RESOURCE_LABEL} {{provider_id: $provider_id}}) + MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`) WITH n LIMIT $batch_size DETACH DELETE n RETURN COUNT(n) AS deleted_nodes_count """, - parameters, + {"batch_size": BATCH_SIZE}, ) deleted_count = result.single().get("deleted_nodes_count", 0) deleted_nodes += deleted_count @@ -196,6 +191,26 @@ def drop_subgraph(database: str, provider_id: str) -> int: return deleted_nodes +def has_provider_data(database: str, provider_id: str) -> bool: + """ + Check if any ProviderResource node exists for this provider. + + Returns `False` if the database doesn't exist. + """ + provider_label = get_provider_label(provider_id) + query = f"MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`) RETURN 1 LIMIT 1" + + try: + with get_session(database, default_access_mode=neo4j.READ_ACCESS) as session: + result = session.run(query) + return result.single() is not None + + except GraphDatabaseQueryException as exc: + if exc.code == "Neo.ClientError.Database.DatabaseNotFound": + return False + raise + + def clear_cache(database: str) -> None: query = "CALL db.clearQueryCaches()" diff --git a/api/src/backend/api/attack_paths/queries/aws.py b/api/src/backend/api/attack_paths/queries/aws.py index 1a442f7bd6..f50935c49f 100644 --- a/api/src/backend/api/attack_paths/queries/aws.py +++ b/api/src/backend/api/attack_paths/queries/aws.py @@ -16,8 +16,6 @@ AWS_INTERNET_EXPOSED_EC2_SENSITIVE_S3_ACCESS = AttackPathsQueryDefinition( description="Detect EC2 instances with SSH exposed to the internet that can assume higher-privileged roles to read tagged sensitive S3 buckets despite bucket-level public access blocks.", provider="aws", cypher=f""" - OPTIONAL MATCH (internet:Internet {{_provider_id: $provider_id}}) - MATCH path_s3 = (aws:AWSAccount {{id: $provider_uid}})--(s3:S3Bucket)--(t:AWSTag) WHERE toLower(t.key) = toLower($tag_key) AND toLower(t.value) = toLower($tag_value) @@ -31,12 +29,18 @@ AWS_INTERNET_EXPOSED_EC2_SENSITIVE_S3_ACCESS = AttackPathsQueryDefinition( MATCH path_assume_role = (ec2)-[p:STS_ASSUMEROLE_ALLOW*1..9]-(r:AWSRole) - OPTIONAL MATCH (internet)-[can_access:CAN_ACCESS]->(ec2) + OPTIONAL MATCH (internet:Internet)-[can_access:CAN_ACCESS]->(ec2) - UNWIND nodes(path_s3) + nodes(path_ec2) + nodes(path_role) + nodes(path_assume_role) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_s3) + collect(path_ec2) + collect(path_role) + collect(path_assume_role) AS paths, + head(collect(internet)) AS internet, collect(can_access) AS can_access + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_s3, path_ec2, path_role, path_assume_role, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access + WITH paths, internet, can_access, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access """, parameters=[ AttackPathsQueryParameterDefinition( @@ -67,10 +71,15 @@ AWS_RDS_INSTANCES = AttackPathsQueryDefinition( cypher=f""" MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(rds:RDSInstance) - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -85,10 +94,15 @@ AWS_RDS_UNENCRYPTED_STORAGE = AttackPathsQueryDefinition( MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(rds:RDSInstance) WHERE rds.storage_encrypted = false - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -103,10 +117,15 @@ AWS_S3_ANONYMOUS_ACCESS_BUCKETS = AttackPathsQueryDefinition( MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(s3:S3Bucket) WHERE s3.anonymous_access = true - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -122,10 +141,15 @@ AWS_IAM_STATEMENTS_ALLOW_ALL_ACTIONS = AttackPathsQueryDefinition( WHERE stmt.effect = 'Allow' AND any(x IN stmt.action WHERE x = '*') - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -141,10 +165,15 @@ AWS_IAM_STATEMENTS_ALLOW_DELETE_POLICY = AttackPathsQueryDefinition( WHERE stmt.effect = 'Allow' AND any(x IN stmt.action WHERE x = "iam:DeletePolicy") - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -160,10 +189,15 @@ AWS_IAM_STATEMENTS_ALLOW_CREATE_ACTIONS = AttackPathsQueryDefinition( WHERE stmt.effect = "Allow" AND any(x IN stmt.action WHERE toLower(x) CONTAINS "create") - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -179,17 +213,20 @@ AWS_EC2_INSTANCES_INTERNET_EXPOSED = AttackPathsQueryDefinition( description="Find EC2 instances flagged as exposed to the internet within the selected account.", provider="aws", cypher=f""" - OPTIONAL MATCH (internet:Internet {{_provider_id: $provider_id}}) - MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(ec2:EC2Instance) WHERE ec2.exposed_internet = true - OPTIONAL MATCH (internet)-[can_access:CAN_ACCESS]->(ec2) + OPTIONAL MATCH (internet:Internet)-[can_access:CAN_ACCESS]->(ec2) - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths, head(collect(internet)) AS internet, collect(can_access) AS can_access + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access + WITH paths, internet, can_access, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access """, parameters=[], ) @@ -201,19 +238,21 @@ AWS_SECURITY_GROUPS_OPEN_INTERNET_FACING = AttackPathsQueryDefinition( description="Find internet-facing resources associated with security groups that allow inbound access from '0.0.0.0/0'.", provider="aws", cypher=f""" - OPTIONAL MATCH (internet:Internet {{_provider_id: $provider_id}}) - - // Match EC2 instances that are internet-exposed with open security groups (0.0.0.0/0) - MATCH path_ec2 = (aws:AWSAccount {{id: $provider_uid}})--(ec2:EC2Instance)--(sg:EC2SecurityGroup)--(ipi:IpPermissionInbound)--(ir:IpRange) + MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(ec2:EC2Instance)--(sg:EC2SecurityGroup)--(ipi:IpPermissionInbound)--(ir:IpRange) WHERE ec2.exposed_internet = true AND ir.range = "0.0.0.0/0" - OPTIONAL MATCH (internet)-[can_access:CAN_ACCESS]->(ec2) + OPTIONAL MATCH (internet:Internet)-[can_access:CAN_ACCESS]->(ec2) - UNWIND nodes(path_ec2) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths, head(collect(internet)) AS internet, collect(can_access) AS can_access + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_ec2, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access + WITH paths, internet, can_access, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access """, parameters=[], ) @@ -225,17 +264,20 @@ AWS_CLASSIC_ELB_INTERNET_EXPOSED = AttackPathsQueryDefinition( description="Find Classic Load Balancers exposed to the internet along with their listeners.", provider="aws", cypher=f""" - OPTIONAL MATCH (internet:Internet {{_provider_id: $provider_id}}) - MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(elb:LoadBalancer)--(listener:ELBListener) WHERE elb.exposed_internet = true - OPTIONAL MATCH (internet)-[can_access:CAN_ACCESS]->(elb) + OPTIONAL MATCH (internet:Internet)-[can_access:CAN_ACCESS]->(elb) - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths, head(collect(internet)) AS internet, collect(can_access) AS can_access + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access + WITH paths, internet, can_access, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access """, parameters=[], ) @@ -247,17 +289,20 @@ AWS_ELBV2_INTERNET_EXPOSED = AttackPathsQueryDefinition( description="Find ELBv2 load balancers exposed to the internet along with their listeners.", provider="aws", cypher=f""" - OPTIONAL MATCH (internet:Internet {{_provider_id: $provider_id}}) - MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(elbv2:LoadBalancerV2)--(listener:ELBV2Listener) WHERE elbv2.exposed_internet = true - OPTIONAL MATCH (internet)-[can_access:CAN_ACCESS]->(elbv2) + OPTIONAL MATCH (internet:Internet)-[can_access:CAN_ACCESS]->(elbv2) - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths, head(collect(internet)) AS internet, collect(can_access) AS can_access + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access + WITH paths, internet, can_access, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access """, parameters=[], ) @@ -269,20 +314,23 @@ AWS_PUBLIC_IP_RESOURCE_LOOKUP = AttackPathsQueryDefinition( description="Given a public IP address, find the related AWS resource and its adjacent node within the selected account.", provider="aws", cypher=f""" - OPTIONAL MATCH (internet:Internet {{_provider_id: $provider_id}}) - MATCH path = (aws:AWSAccount {{id: $provider_uid}})-[r]-(x)-[q]-(y) WHERE (x:EC2PrivateIp AND x.public_ip = $ip) OR (x:EC2Instance AND x.publicipaddress = $ip) OR (x:NetworkInterface AND x.public_ip = $ip) OR (x:ElasticIPAddress AND x.public_ip = $ip) - OPTIONAL MATCH (internet)-[can_access:CAN_ACCESS]->(x) + OPTIONAL MATCH (internet:Internet)-[can_access:CAN_ACCESS]->(x) - UNWIND nodes(path) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths, head(collect(internet)) AS internet, collect(can_access) AS can_access + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access + WITH paths, internet, can_access, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access """, parameters=[ AttackPathsQueryParameterDefinition( @@ -336,11 +384,16 @@ AWS_APPRUNNER_PRIVESC_PASSROLE_CREATE_SERVICE = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -369,11 +422,16 @@ AWS_APPRUNNER_PRIVESC_UPDATE_SERVICE = AttackPathsQueryDefinition( // Find existing App Runner services with roles attached (potential targets) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'tasks.apprunner.amazonaws.com'}}) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -434,11 +492,16 @@ AWS_BEDROCK_PRIVESC_PASSROLE_CODE_INTERPRETER = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -476,11 +539,16 @@ AWS_BEDROCK_PRIVESC_INVOKE_CODE_INTERPRETER = AttackPathsQueryDefinition( // Find roles that trust Bedrock service (already attached to existing code interpreters) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock.amazonaws.com'}}) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -523,11 +591,16 @@ AWS_CLOUDFORMATION_PRIVESC_PASSROLE_CREATE_STACK = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -556,11 +629,16 @@ AWS_CLOUDFORMATION_PRIVESC_UPDATE_STACK = AttackPathsQueryDefinition( // Find roles that trust CloudFormation service (already attached to existing stacks) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}}) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -612,11 +690,16 @@ AWS_CLOUDFORMATION_PRIVESC_PASSROLE_CREATE_STACKSET = AttackPathsQueryDefinition OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -659,11 +742,16 @@ AWS_CLOUDFORMATION_PRIVESC_PASSROLE_UPDATE_STACKSET = AttackPathsQueryDefinition OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -701,11 +789,16 @@ AWS_CLOUDFORMATION_PRIVESC_CHANGESET = AttackPathsQueryDefinition( // Find roles that trust CloudFormation service (already attached to existing stacks) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}}) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -757,11 +850,16 @@ AWS_CODEBUILD_PRIVESC_PASSROLE_CREATE_PROJECT = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -790,11 +888,16 @@ AWS_CODEBUILD_PRIVESC_START_BUILD = AttackPathsQueryDefinition( // Find roles that trust CodeBuild service (already attached to existing projects) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}}) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -823,11 +926,16 @@ AWS_CODEBUILD_PRIVESC_START_BUILD_BATCH = AttackPathsQueryDefinition( // Find roles that trust CodeBuild service (already attached to existing projects) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}}) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -879,11 +987,16 @@ AWS_CODEBUILD_PRIVESC_PASSROLE_CREATE_PROJECT_BATCH = AttackPathsQueryDefinition OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -945,11 +1058,16 @@ AWS_DATAPIPELINE_PRIVESC_PASSROLE_CREATE_PIPELINE = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -992,11 +1110,16 @@ AWS_EC2_PRIVESC_PASSROLE_IAM = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1043,11 +1166,16 @@ AWS_EC2_PRIVESC_MODIFY_INSTANCE_ATTRIBUTE = AttackPathsQueryDefinition( // Find EC2 instances with instance profiles (potential targets) MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1090,11 +1218,16 @@ AWS_EC2_PRIVESC_PASSROLE_SPOT_INSTANCES = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1132,11 +1265,16 @@ AWS_EC2_PRIVESC_LAUNCH_TEMPLATE = AttackPathsQueryDefinition( // Find launch templates in the account (potential targets) MATCH path_target = (aws)--(template:LaunchTemplate) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1165,11 +1303,16 @@ AWS_EC2INSTANCECONNECT_PRIVESC_SEND_SSH_PUBLIC_KEY = AttackPathsQueryDefinition( // Find EC2 instances with attached roles (targets for credential theft via IMDS) MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1230,11 +1373,16 @@ AWS_ECS_PRIVESC_PASSROLE_CREATE_SERVICE = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1295,11 +1443,16 @@ AWS_ECS_PRIVESC_PASSROLE_RUN_TASK = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1351,11 +1504,16 @@ AWS_ECS_PRIVESC_PASSROLE_CREATE_SERVICE_EXISTING_CLUSTER = AttackPathsQueryDefin OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1407,11 +1565,16 @@ AWS_ECS_PRIVESC_PASSROLE_RUN_TASK_EXISTING_CLUSTER = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1463,11 +1626,16 @@ AWS_ECS_PRIVESC_PASSROLE_START_TASK_EXISTING_CLUSTER = AttackPathsQueryDefinitio OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1505,11 +1673,16 @@ AWS_ECS_PRIVESC_EXECUTE_COMMAND = AttackPathsQueryDefinition( // Find roles that trust ECS tasks service (already attached to running tasks) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ecs-tasks.amazonaws.com'}}) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1552,11 +1725,16 @@ AWS_GLUE_PRIVESC_PASSROLE_DEV_ENDPOINT = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1585,11 +1763,16 @@ AWS_GLUE_PRIVESC_UPDATE_DEV_ENDPOINT = AttackPathsQueryDefinition( // Find roles that trust Glue service (already attached to existing dev endpoints) MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'glue.amazonaws.com'}}) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1641,11 +1824,16 @@ AWS_GLUE_PRIVESC_PASSROLE_CREATE_JOB = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1697,11 +1885,16 @@ AWS_GLUE_PRIVESC_PASSROLE_CREATE_JOB_TRIGGER = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1753,11 +1946,16 @@ AWS_GLUE_PRIVESC_PASSROLE_UPDATE_JOB = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1809,11 +2007,16 @@ AWS_GLUE_PRIVESC_PASSROLE_UPDATE_JOB_TRIGGER = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1847,11 +2050,16 @@ AWS_IAM_PRIVESC_CREATE_POLICY_VERSION = AttackPathsQueryDefinition( OR target_policy.arn CONTAINS resource ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1885,11 +2093,16 @@ AWS_IAM_PRIVESC_CREATE_ACCESS_KEY = AttackPathsQueryDefinition( OR resource CONTAINS target_user.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1937,11 +2150,16 @@ AWS_IAM_PRIVESC_DELETE_CREATE_ACCESS_KEY = AttackPathsQueryDefinition( OR resource CONTAINS target_user.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1975,11 +2193,16 @@ AWS_IAM_PRIVESC_CREATE_LOGIN_PROFILE = AttackPathsQueryDefinition( OR resource CONTAINS target_user.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -1997,7 +2220,7 @@ AWS_IAM_PRIVESC_PUT_ROLE_POLICY = AttackPathsQueryDefinition( provider="aws", cypher=f""" // Find roles with iam:PutRolePolicy permission scoped to themselves - MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(role:AWSRole)--(policy:AWSPolicy)--(stmt:AWSPolicyStatement) + MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(role:AWSRole)--(policy:AWSPolicy)--(stmt:AWSPolicyStatement) WHERE stmt.effect = 'Allow' AND any(action IN stmt.action WHERE toLower(action) = 'iam:putrolepolicy' @@ -2010,11 +2233,16 @@ AWS_IAM_PRIVESC_PUT_ROLE_POLICY = AttackPathsQueryDefinition( OR resource CONTAINS role.name ) - UNWIND nodes(path_principal) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2048,11 +2276,16 @@ AWS_IAM_PRIVESC_UPDATE_LOGIN_PROFILE = AttackPathsQueryDefinition( OR resource CONTAINS target_user.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2070,7 +2303,7 @@ AWS_IAM_PRIVESC_PUT_USER_POLICY = AttackPathsQueryDefinition( provider="aws", cypher=f""" // Find users with iam:PutUserPolicy permission scoped to themselves - MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(user:AWSUser)--(policy:AWSPolicy)--(stmt:AWSPolicyStatement) + MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(user:AWSUser)--(policy:AWSPolicy)--(stmt:AWSPolicyStatement) WHERE stmt.effect = 'Allow' AND any(action IN stmt.action WHERE toLower(action) = 'iam:putuserpolicy' @@ -2083,11 +2316,16 @@ AWS_IAM_PRIVESC_PUT_USER_POLICY = AttackPathsQueryDefinition( OR resource CONTAINS user.name ) - UNWIND nodes(path_principal) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2105,7 +2343,7 @@ AWS_IAM_PRIVESC_ATTACH_USER_POLICY = AttackPathsQueryDefinition( provider="aws", cypher=f""" // Find users with iam:AttachUserPolicy permission scoped to themselves - MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(user:AWSUser)--(policy:AWSPolicy)--(stmt:AWSPolicyStatement) + MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(user:AWSUser)--(policy:AWSPolicy)--(stmt:AWSPolicyStatement) WHERE stmt.effect = 'Allow' AND any(action IN stmt.action WHERE toLower(action) = 'iam:attachuserpolicy' @@ -2118,11 +2356,16 @@ AWS_IAM_PRIVESC_ATTACH_USER_POLICY = AttackPathsQueryDefinition( OR resource CONTAINS user.name ) - UNWIND nodes(path_principal) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2140,7 +2383,7 @@ AWS_IAM_PRIVESC_ATTACH_ROLE_POLICY = AttackPathsQueryDefinition( provider="aws", cypher=f""" // Find roles with iam:AttachRolePolicy permission scoped to themselves - MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(role:AWSRole)--(policy:AWSPolicy)--(stmt:AWSPolicyStatement) + MATCH path = (aws:AWSAccount {{id: $provider_uid}})--(role:AWSRole)--(policy:AWSPolicy)--(stmt:AWSPolicyStatement) WHERE stmt.effect = 'Allow' AND any(action IN stmt.action WHERE toLower(action) = 'iam:attachrolepolicy' @@ -2153,11 +2396,16 @@ AWS_IAM_PRIVESC_ATTACH_ROLE_POLICY = AttackPathsQueryDefinition( OR resource CONTAINS role.name ) - UNWIND nodes(path_principal) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2191,11 +2439,16 @@ AWS_IAM_PRIVESC_ATTACH_GROUP_POLICY = AttackPathsQueryDefinition( OR resource CONTAINS target_group.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2229,11 +2482,16 @@ AWS_IAM_PRIVESC_PUT_GROUP_POLICY = AttackPathsQueryDefinition( OR resource CONTAINS target_group.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2267,11 +2525,16 @@ AWS_IAM_PRIVESC_UPDATE_ASSUME_ROLE_POLICY = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2305,11 +2568,16 @@ AWS_IAM_PRIVESC_ADD_USER_TO_GROUP = AttackPathsQueryDefinition( OR resource CONTAINS target_group.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2343,11 +2611,16 @@ AWS_IAM_PRIVESC_ATTACH_ROLE_POLICY_ASSUME_ROLE = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2395,11 +2668,16 @@ AWS_IAM_PRIVESC_ATTACH_USER_POLICY_CREATE_ACCESS_KEY = AttackPathsQueryDefinitio OR resource CONTAINS target_user.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2434,11 +2712,16 @@ AWS_IAM_PRIVESC_CREATE_POLICY_VERSION_ASSUME_ROLE = AttackPathsQueryDefinition( OR target_policy.arn CONTAINS resource ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2472,11 +2755,16 @@ AWS_IAM_PRIVESC_PUT_ROLE_POLICY_ASSUME_ROLE = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2524,11 +2812,16 @@ AWS_IAM_PRIVESC_PUT_USER_POLICY_CREATE_ACCESS_KEY = AttackPathsQueryDefinition( OR resource CONTAINS target_user.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2576,11 +2869,16 @@ AWS_IAM_PRIVESC_ATTACH_ROLE_POLICY_UPDATE_ASSUME_ROLE = AttackPathsQueryDefiniti OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2629,11 +2927,16 @@ AWS_IAM_PRIVESC_CREATE_POLICY_VERSION_UPDATE_ASSUME_ROLE = AttackPathsQueryDefin OR target_policy.arn CONTAINS resource ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2681,11 +2984,16 @@ AWS_IAM_PRIVESC_PUT_ROLE_POLICY_UPDATE_ASSUME_ROLE = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2737,11 +3045,16 @@ AWS_LAMBDA_PRIVESC_PASSROLE_CREATE_FUNCTION = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2793,11 +3106,16 @@ AWS_LAMBDA_PRIVESC_PASSROLE_CREATE_FUNCTION_EVENT_SOURCE = AttackPathsQueryDefin OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2831,11 +3149,16 @@ AWS_LAMBDA_PRIVESC_UPDATE_FUNCTION_CODE = AttackPathsQueryDefinition( OR resource CONTAINS lambda_fn.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2883,11 +3206,16 @@ AWS_LAMBDA_PRIVESC_UPDATE_FUNCTION_CODE_INVOKE = AttackPathsQueryDefinition( OR resource CONTAINS lambda_fn.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2935,11 +3263,16 @@ AWS_LAMBDA_PRIVESC_UPDATE_FUNCTION_CODE_ADD_PERMISSION = AttackPathsQueryDefinit OR resource CONTAINS lambda_fn.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -2991,11 +3324,16 @@ AWS_LAMBDA_PRIVESC_PASSROLE_CREATE_FUNCTION_ADD_PERMISSION = AttackPathsQueryDef OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -3038,11 +3376,16 @@ AWS_SAGEMAKER_PRIVESC_PASSROLE_CREATE_NOTEBOOK = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -3085,11 +3428,16 @@ AWS_SAGEMAKER_PRIVESC_PASSROLE_CREATE_TRAINING_JOB = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -3132,11 +3480,16 @@ AWS_SAGEMAKER_PRIVESC_PASSROLE_CREATE_PROCESSING_JOB = AttackPathsQueryDefinitio OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -3170,11 +3523,16 @@ AWS_SAGEMAKER_PRIVESC_PRESIGNED_NOTEBOOK_URL = AttackPathsQueryDefinition( OR resource CONTAINS notebook.notebook_instance_name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -3235,11 +3593,16 @@ AWS_SAGEMAKER_PRIVESC_LIFECYCLE_CONFIG_NOTEBOOK = AttackPathsQueryDefinition( OR resource CONTAINS notebook.notebook_instance_name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -3268,11 +3631,16 @@ AWS_SSM_PRIVESC_START_SESSION = AttackPathsQueryDefinition( // Find EC2 instances with attached roles (targets for credential theft via IMDS) MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -3301,11 +3669,16 @@ AWS_SSM_PRIVESC_SEND_COMMAND = AttackPathsQueryDefinition( // Find EC2 instances with attached roles (targets for credential theft via IMDS) MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) @@ -3339,11 +3712,16 @@ AWS_STS_PRIVESC_ASSUME_ROLE = AttackPathsQueryDefinition( OR resource CONTAINS target_role.name ) - UNWIND nodes(path_principal) + nodes(path_target) as n - OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL', provider_uid: $provider_uid}}) + WITH collect(path_principal) + collect(path_target) AS paths + UNWIND paths AS p + UNWIND nodes(p) AS n - RETURN path_principal, path_target, - collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr + WITH paths, collect(DISTINCT n) AS unique_nodes + UNWIND unique_nodes AS n + + OPTIONAL MATCH (n)-[pfr]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}}) + + RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr """, parameters=[], ) diff --git a/api/src/backend/api/attack_paths/queries/schema.py b/api/src/backend/api/attack_paths/queries/schema.py index 1ed227458b..5373d17508 100644 --- a/api/src/backend/api/attack_paths/queries/schema.py +++ b/api/src/backend/api/attack_paths/queries/schema.py @@ -1,13 +1,18 @@ -from tasks.jobs.attack_paths.config import DEPRECATED_PROVIDER_RESOURCE_LABEL +from tasks.jobs.attack_paths.config import PROVIDER_RESOURCE_LABEL, get_provider_label + + +def get_cartography_schema_query(provider_id: str) -> str: + """Build the Cartography schema metadata query scoped to a provider label.""" + provider_label = get_provider_label(provider_id) + return f""" + MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`) + WHERE n._module_name STARTS WITH 'cartography:' + AND NOT n._module_name IN ['cartography:ontology', 'cartography:prowler'] + AND n._module_version IS NOT NULL + RETURN n._module_name AS module_name, n._module_version AS module_version + LIMIT 1 + """ -CARTOGRAPHY_SCHEMA_METADATA = f""" - MATCH (n:{DEPRECATED_PROVIDER_RESOURCE_LABEL} {{provider_id: $provider_id}}) - WHERE n._module_name STARTS WITH 'cartography:' - AND NOT n._module_name IN ['cartography:ontology', 'cartography:prowler'] - AND n._module_version IS NOT NULL - RETURN n._module_name AS module_name, n._module_version AS module_version - LIMIT 1 -""" GITHUB_SCHEMA_URL = ( "https://github.com/cartography-cncf/cartography/blob/" diff --git a/api/src/backend/api/attack_paths/views_helpers.py b/api/src/backend/api/attack_paths/views_helpers.py index b24392955e..201527885e 100644 --- a/api/src/backend/api/attack_paths/views_helpers.py +++ b/api/src/backend/api/attack_paths/views_helpers.py @@ -1,19 +1,28 @@ import logging -import re from typing import Any, Iterable import neo4j + from rest_framework.exceptions import APIException, PermissionDenied, ValidationError from api.attack_paths import database as graph_database, AttackPathsQueryDefinition +from api.attack_paths.cypher_sanitizer import ( + inject_provider_label, + validate_custom_query, +) from api.attack_paths.queries.schema import ( - CARTOGRAPHY_SCHEMA_METADATA, GITHUB_SCHEMA_URL, RAW_SCHEMA_URL, + get_cartography_schema_query, ) from config.custom_logging import BackendLogger -from tasks.jobs.attack_paths.config import INTERNAL_LABELS, INTERNAL_PROPERTIES +from tasks.jobs.attack_paths.config import ( + INTERNAL_LABELS, + INTERNAL_PROPERTIES, + get_provider_label, + is_dynamic_isolation_label, +) logger = logging.getLogger(BackendLogger.API) @@ -67,7 +76,6 @@ def prepare_parameters( clean_parameters = { "provider_uid": str(provider_uid), - "provider_id": str(provider_id), } for definition_parameter in definition.parameters: @@ -118,38 +126,6 @@ def execute_query( # Custom query helpers -# Patterns that indicate SSRF or dangerous procedure calls -# Defense-in-depth layer - the primary control is `neo4j.READ_ACCESS` -_BLOCKED_PATTERNS = [ - re.compile(r"\bLOAD\s+CSV\b", re.IGNORECASE), - re.compile(r"\bapoc\.load\b", re.IGNORECASE), - re.compile(r"\bapoc\.import\b", re.IGNORECASE), - re.compile(r"\bapoc\.export\b", re.IGNORECASE), - re.compile(r"\bapoc\.cypher\b", re.IGNORECASE), - re.compile(r"\bapoc\.systemdb\b", re.IGNORECASE), - re.compile(r"\bapoc\.config\b", re.IGNORECASE), - re.compile(r"\bapoc\.periodic\b", re.IGNORECASE), - re.compile(r"\bapoc\.do\b", re.IGNORECASE), - re.compile(r"\bapoc\.trigger\b", re.IGNORECASE), - re.compile(r"\bapoc\.custom\b", re.IGNORECASE), -] - -# Strip string literals so patterns inside quotes don't cause false positives -# Handles escaped quotes (\' and \") inside strings -_STRING_LITERALS = re.compile(r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\"") - - -def validate_custom_query(cypher: str) -> None: - """Reject queries containing known SSRF or dangerous procedure patterns. - - Raises ValidationError if a blocked pattern is found. - String literals are stripped before matching to avoid false positives. - """ - stripped = _STRING_LITERALS.sub("", cypher) - for pattern in _BLOCKED_PATTERNS: - if pattern.search(stripped): - raise ValidationError({"query": "Query contains a blocked operation"}) - def normalize_custom_query_payload(raw_data): if not isinstance(raw_data, dict): @@ -168,7 +144,15 @@ def execute_custom_query( cypher: str, provider_id: str, ) -> dict[str, Any]: + # Defense-in-depth for custom queries: + # 1. neo4j.READ_ACCESS — prevents mutations at the driver level + # 2. inject_provider_label() — regex-based label injection scopes node patterns + # 3. _serialize_graph() — post-query filter drops nodes without the provider label + # + # Layer 2 is best-effort (regex can't fully parse Cypher); + # layer 3 is the safety net that guarantees provider isolation. validate_custom_query(cypher) + cypher = inject_provider_label(cypher, provider_id) try: graph = graph_database.execute_read_query( @@ -203,10 +187,7 @@ def get_cartography_schema( with graph_database.get_session( database_name, default_access_mode=neo4j.READ_ACCESS ) as session: - result = session.run( - CARTOGRAPHY_SCHEMA_METADATA, - {"provider_id": provider_id}, - ) + result = session.run(get_cartography_schema_query(provider_id)) record = result.single() except graph_database.GraphDatabaseQueryException as exc: logger.error(f"Cartography schema query failed: {exc}") @@ -250,10 +231,12 @@ def _truncate_graph(graph: dict[str, Any]) -> dict[str, Any]: def _serialize_graph(graph, provider_id: str) -> dict[str, Any]: + provider_label = get_provider_label(provider_id) + nodes = [] kept_node_ids = set() for node in graph.nodes: - if node._properties.get("provider_id") != provider_id: + if provider_label not in node.labels: continue kept_node_ids.add(node.element_id) @@ -268,14 +251,11 @@ def _serialize_graph(graph, provider_id: str) -> dict[str, Any]: filtered_count = len(graph.nodes) - len(nodes) if filtered_count > 0: logger.debug( - f"Filtered {filtered_count} nodes without matching provider_id={provider_id}" + f"Filtered {filtered_count} nodes without provider label {provider_label}" ) relationships = [] for relationship in graph.relationships: - if relationship._properties.get("provider_id") != provider_id: - continue - if ( relationship.start_node.element_id not in kept_node_ids or relationship.end_node.element_id not in kept_node_ids @@ -301,7 +281,11 @@ def _serialize_graph(graph, provider_id: str) -> dict[str, Any]: def _filter_labels(labels: Iterable[str]) -> list[str]: - return [label for label in labels if label not in INTERNAL_LABELS] + return [ + label + for label in labels + if label not in INTERNAL_LABELS and not is_dynamic_isolation_label(label) + ] def _serialize_properties(properties: dict[str, Any]) -> dict[str, Any]: diff --git a/api/src/backend/api/base_views.py b/api/src/backend/api/base_views.py index 7cdf8fdd7f..b14cc39529 100644 --- a/api/src/backend/api/base_views.py +++ b/api/src/backend/api/base_views.py @@ -1,10 +1,10 @@ from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from rest_framework import permissions from rest_framework.exceptions import NotAuthenticated from rest_framework.filters import SearchFilter from rest_framework.permissions import SAFE_METHODS +from rest_framework.response import Response from rest_framework_json_api import filters from rest_framework_json_api.views import ModelViewSet @@ -12,7 +12,7 @@ from api.authentication import CombinedJWTOrAPIKeyAuthentication from api.db_router import MainRouter, reset_read_db_alias, set_read_db_alias from api.db_utils import POSTGRES_USER_VAR, rls_transaction from api.filters import CustomDjangoFilterBackend -from api.models import Role, Tenant +from api.models import Role, UserRoleRelationship from api.rbac.permissions import HasPermissions @@ -113,27 +113,22 @@ class BaseTenantViewset(BaseViewSet): if request is not None: request.db_alias = self.db_alias - with transaction.atomic(using=self.db_alias): - tenant = super().dispatch(request, *args, **kwargs) - - try: - # If the request is a POST, create the admin role - if request.method == "POST": - isinstance(tenant, dict) and self._create_admin_role( - tenant.data["id"] - ) - except Exception as e: - self._handle_creation_error(e, tenant) - raise - - return tenant + if request.method == "POST": + with transaction.atomic(using=MainRouter.admin_db): + tenant = super().dispatch(request, *args, **kwargs) + if isinstance(tenant, Response) and tenant.status_code == 201: + self._create_admin_role(tenant.data["id"]) + return tenant + else: + with transaction.atomic(using=self.db_alias): + return super().dispatch(request, *args, **kwargs) finally: if alias_token is not None: reset_read_db_alias(alias_token) self.db_alias = MainRouter.default_db def _create_admin_role(self, tenant_id): - Role.objects.using(MainRouter.admin_db).create( + admin_role = Role.objects.using(MainRouter.admin_db).create( name="admin", tenant_id=tenant_id, manage_users=True, @@ -144,15 +139,11 @@ class BaseTenantViewset(BaseViewSet): manage_scans=True, unlimited_visibility=True, ) - - def _handle_creation_error(self, error, tenant): - if tenant.data.get("id"): - try: - Tenant.objects.using(MainRouter.admin_db).filter( - id=tenant.data["id"] - ).delete() - except ObjectDoesNotExist: - pass # Tenant might not exist, handle gracefully + UserRoleRelationship.objects.using(MainRouter.admin_db).create( + user=self.request.user, + role=admin_role, + tenant_id=tenant_id, + ) def initial(self, request, *args, **kwargs): if request.auth is None: diff --git a/api/src/backend/api/compliance.py b/api/src/backend/api/compliance.py index 1705ed2e8f..25b8fb6735 100644 --- a/api/src/backend/api/compliance.py +++ b/api/src/backend/api/compliance.py @@ -1,7 +1,6 @@ from collections.abc import Iterable, Mapping from api.models import Provider -from prowler.config.config import get_available_compliance_frameworks from prowler.lib.check.compliance_models import Compliance from prowler.lib.check.models import CheckMetadata @@ -95,12 +94,12 @@ PROWLER_CHECKS = LazyChecksMapping() def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]: - """ - Retrieve and cache the list of available compliance frameworks for a specific cloud provider. + """List compliance frameworks the API can load for `provider_type`. - This function lazily loads and caches the available compliance frameworks (e.g., CIS, MITRE, ISO) - for each provider type (AWS, Azure, GCP, etc.) on first access. Subsequent calls for the same - provider will return the cached result. + The list is sourced from `Compliance.get_bulk` so that the names + returned here are guaranteed to be loadable by the bulk loader. This + prevents downstream key mismatches (e.g. CSV report generation iterating + framework names and looking them up in the bulk dict). Args: provider_type (Provider.ProviderChoices): The cloud provider type for which to retrieve @@ -112,8 +111,8 @@ def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[s """ global AVAILABLE_COMPLIANCE_FRAMEWORKS if provider_type not in AVAILABLE_COMPLIANCE_FRAMEWORKS: - AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] = ( - get_available_compliance_frameworks(provider_type) + AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] = list( + Compliance.get_bulk(provider_type).keys() ) return AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] diff --git a/api/src/backend/api/db_utils.py b/api/src/backend/api/db_utils.py index 7a71084ccd..e3b11d7084 100644 --- a/api/src/backend/api/db_utils.py +++ b/api/src/backend/api/db_utils.py @@ -18,6 +18,7 @@ from django.db import ( ) from django_celery_beat.models import PeriodicTask from psycopg2 import connect as psycopg2_connect +from psycopg2 import sql as psycopg2_sql from psycopg2.extensions import AsIs, new_type, register_adapter, register_type from rest_framework_json_api.serializers import ValidationError @@ -280,15 +281,23 @@ class PostgresEnumMigration: self.enum_values = enum_values def create_enum_type(self, apps, schema_editor): # noqa: F841 - string_enum_values = ", ".join([f"'{value}'" for value in self.enum_values]) with schema_editor.connection.cursor() as cursor: cursor.execute( - f"CREATE TYPE {self.enum_name} AS ENUM ({string_enum_values});" + psycopg2_sql.SQL("CREATE TYPE {} AS ENUM ({})").format( + psycopg2_sql.Identifier(self.enum_name), + psycopg2_sql.SQL(", ").join( + psycopg2_sql.Literal(v) for v in self.enum_values + ), + ) ) def drop_enum_type(self, apps, schema_editor): # noqa: F841 with schema_editor.connection.cursor() as cursor: - cursor.execute(f"DROP TYPE {self.enum_name};") + cursor.execute( + psycopg2_sql.SQL("DROP TYPE {}").format( + psycopg2_sql.Identifier(self.enum_name) + ) + ) class PostgresEnumField(models.Field): diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index a64cc0ea13..c787d714b1 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -15,6 +15,7 @@ from django_filters.rest_framework import ( from rest_framework_json_api.django_filters.backends import DjangoFilterBackend from rest_framework_json_api.serializers import ValidationError +from api.constants import SEVERITY_ORDER from api.db_utils import ( FindingDeltaEnumField, InvitationStateEnumField, @@ -43,6 +44,7 @@ from api.models import ( ProviderGroup, ProviderSecret, Resource, + ResourceFindingMapping, ResourceTag, Role, Scan, @@ -196,17 +198,13 @@ class CommonFindingFilters(FilterSet): field_name="resource_services", lookup_expr="icontains" ) - resource_uid = CharFilter(field_name="resources__uid") - resource_uid__in = CharInFilter(field_name="resources__uid", lookup_expr="in") - resource_uid__icontains = CharFilter( - field_name="resources__uid", lookup_expr="icontains" - ) + resource_uid = CharFilter(method="filter_resource_uid") + resource_uid__in = CharInFilter(method="filter_resource_uid_in") + resource_uid__icontains = CharFilter(method="filter_resource_uid_icontains") - resource_name = CharFilter(field_name="resources__name") - resource_name__in = CharInFilter(field_name="resources__name", lookup_expr="in") - resource_name__icontains = CharFilter( - field_name="resources__name", lookup_expr="icontains" - ) + resource_name = CharFilter(method="filter_resource_name") + resource_name__in = CharInFilter(method="filter_resource_name_in") + resource_name__icontains = CharFilter(method="filter_resource_name_icontains") resource_type = CharFilter(method="filter_resource_type") resource_type__in = CharInFilter(field_name="resource_types", lookup_expr="overlap") @@ -264,6 +262,52 @@ class CommonFindingFilters(FilterSet): ) return queryset.filter(overall_query).distinct() + def filter_check_title_icontains(self, queryset, name, value): + # Resolve from the summary table (has check_title column + trigram + # GIN index) instead of scanning JSON in the findings table. + matching_check_ids = ( + FindingGroupDailySummary.objects.filter( + check_title__icontains=value, + ) + .values_list("check_id", flat=True) + .distinct() + ) + return queryset.filter(check_id__in=matching_check_ids) + + # --- Resource subquery filters --- + # Resolve resource → RFM → finding_ids first, then filter findings + # by id__in. This avoids a 3-way JOIN driven from the (huge) + # findings side and lets PostgreSQL start from the resources + # unique-constraint index instead. + + @staticmethod + def _finding_ids_for_resources(**lookup): + return ResourceFindingMapping.objects.filter( + resource__in=Resource.objects.filter(**lookup).values("id") + ).values("finding_id") + + def filter_resource_uid(self, queryset, name, value): + return queryset.filter(id__in=self._finding_ids_for_resources(uid=value)) + + def filter_resource_uid_in(self, queryset, name, value): + return queryset.filter(id__in=self._finding_ids_for_resources(uid__in=value)) + + def filter_resource_uid_icontains(self, queryset, name, value): + return queryset.filter( + id__in=self._finding_ids_for_resources(uid__icontains=value) + ) + + def filter_resource_name(self, queryset, name, value): + return queryset.filter(id__in=self._finding_ids_for_resources(name=value)) + + def filter_resource_name_in(self, queryset, name, value): + return queryset.filter(id__in=self._finding_ids_for_resources(name__in=value)) + + def filter_resource_name_icontains(self, queryset, name, value): + return queryset.filter( + id__in=self._finding_ids_for_resources(name__icontains=value) + ) + class TenantFilter(FilterSet): inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") @@ -286,6 +330,7 @@ class MembershipFilter(FilterSet): model = Membership fields = { "tenant": ["exact"], + "user": ["exact"], "role": ["exact"], "date_joined": ["date", "gte", "lte"], } @@ -390,6 +435,7 @@ class ScanFilter(ProviderRelationshipFilterSet): class Meta: model = Scan fields = { + "id": ["exact", "in"], "provider": ["exact", "in"], "name": ["exact", "icontains"], "started_at": ["gte", "lte"], @@ -803,11 +849,15 @@ class FindingGroupFilter(CommonFindingFilters): check_id = CharFilter(field_name="check_id", lookup_expr="exact") check_id__in = CharInFilter(field_name="check_id", lookup_expr="in") check_id__icontains = CharFilter(field_name="check_id", lookup_expr="icontains") + check_title__icontains = CharFilter(method="filter_check_title_icontains") + scan = UUIDFilter(field_name="scan_id", lookup_expr="exact") + scan__in = UUIDInFilter(field_name="scan_id", lookup_expr="in") class Meta: model = Finding fields = { "check_id": ["exact", "in", "icontains"], + "scan": ["exact", "in"], } def filter_queryset(self, queryset): @@ -895,15 +945,31 @@ class LatestFindingGroupFilter(CommonFindingFilters): check_id = CharFilter(field_name="check_id", lookup_expr="exact") check_id__in = CharInFilter(field_name="check_id", lookup_expr="in") check_id__icontains = CharFilter(field_name="check_id", lookup_expr="icontains") + check_title__icontains = CharFilter(method="filter_check_title_icontains") + scan = UUIDFilter(field_name="scan_id", lookup_expr="exact") + scan__in = UUIDInFilter(field_name="scan_id", lookup_expr="in") class Meta: model = Finding fields = { "check_id": ["exact", "in", "icontains"], + "scan": ["exact", "in"], } -class FindingGroupSummaryFilter(FilterSet): +class _CheckTitleToCheckIdMixin: + """Resolve check_title search to check_ids so all provider rows are kept.""" + + def filter_check_title_to_check_ids(self, queryset, name, value): + matching_check_ids = ( + queryset.filter(check_title__icontains=value) + .values_list("check_id", flat=True) + .distinct() + ) + return queryset.filter(check_id__in=matching_check_ids) + + +class FindingGroupSummaryFilter(_CheckTitleToCheckIdMixin, FilterSet): """ Filter for FindingGroupDailySummary queries. @@ -926,6 +992,7 @@ class FindingGroupSummaryFilter(FilterSet): check_id = CharFilter(field_name="check_id", lookup_expr="exact") check_id__in = CharInFilter(field_name="check_id", lookup_expr="in") check_id__icontains = CharFilter(field_name="check_id", lookup_expr="icontains") + check_title__icontains = CharFilter(method="filter_check_title_to_check_ids") # Provider filters provider_id = UUIDFilter(field_name="provider_id", lookup_expr="exact") @@ -1013,7 +1080,7 @@ class FindingGroupSummaryFilter(FilterSet): return dt -class LatestFindingGroupSummaryFilter(FilterSet): +class LatestFindingGroupSummaryFilter(_CheckTitleToCheckIdMixin, FilterSet): """ Filter for FindingGroupDailySummary /latest endpoint. @@ -1025,6 +1092,7 @@ class LatestFindingGroupSummaryFilter(FilterSet): check_id = CharFilter(field_name="check_id", lookup_expr="exact") check_id__in = CharInFilter(field_name="check_id", lookup_expr="in") check_id__icontains = CharFilter(field_name="check_id", lookup_expr="icontains") + check_title__icontains = CharFilter(method="filter_check_title_to_check_ids") # Provider filters provider_id = UUIDFilter(field_name="provider_id", lookup_expr="exact") @@ -1042,6 +1110,99 @@ class LatestFindingGroupSummaryFilter(FilterSet): } +class FindingGroupAggregatedComputedFilter(FilterSet): + """Filter aggregated finding-group rows by computed status/severity/muted.""" + + STATUS_CHOICES = ( + ("FAIL", "Fail"), + ("PASS", "Pass"), + ("MANUAL", "Manual"), + ) + + status = ChoiceFilter(method="filter_status", choices=STATUS_CHOICES) + status__in = CharInFilter(method="filter_status_in", lookup_expr="in") + severity = ChoiceFilter(method="filter_severity", choices=SeverityChoices) + severity__in = CharInFilter(method="filter_severity_in", lookup_expr="in") + muted = BooleanFilter(field_name="muted") + include_muted = BooleanFilter(method="filter_include_muted") + + def filter_status(self, queryset, name, value): + return queryset.filter(aggregated_status=value) + + def filter_status_in(self, queryset, name, value): + values = value + if isinstance(value, str): + values = [part.strip() for part in value.split(",") if part.strip()] + + allowed = {choice[0] for choice in self.STATUS_CHOICES} + invalid = [ + status_value for status_value in values if status_value not in allowed + ] + if invalid: + raise ValidationError( + [ + { + "detail": f"invalid status filter: {invalid[0]}", + "status": "400", + "source": {"pointer": "/data"}, + "code": "invalid", + } + ] + ) + + if not values: + return queryset + + return queryset.filter(aggregated_status__in=values) + + def filter_severity(self, queryset, name, value): + severity_order = SEVERITY_ORDER.get(value) + if severity_order is None: + raise ValidationError( + [ + { + "detail": f"invalid severity filter: {value}", + "status": "400", + "source": {"pointer": "/data"}, + "code": "invalid", + } + ] + ) + return queryset.filter(severity_order=severity_order) + + def filter_severity_in(self, queryset, name, value): + values = value + if isinstance(value, str): + values = [part.strip() for part in value.split(",") if part.strip()] + + orders = [] + for severity_value in values: + severity_order = SEVERITY_ORDER.get(severity_value) + if severity_order is None: + raise ValidationError( + [ + { + "detail": f"invalid severity filter: {severity_value}", + "status": "400", + "source": {"pointer": "/data"}, + "code": "invalid", + } + ] + ) + orders.append(severity_order) + + if not orders: + return queryset + + return queryset.filter(severity_order__in=orders) + + def filter_include_muted(self, queryset, name, value): + if value is True: + return queryset + # include_muted=false: exclude fully-muted groups + return queryset.exclude(muted=True) + + class ProviderSecretFilter(FilterSet): inserted_at = DateFilter( field_name="inserted_at", diff --git a/api/src/backend/api/migrations/0085_finding_group_daily_summary_trgm_indexes.py b/api/src/backend/api/migrations/0085_finding_group_daily_summary_trgm_indexes.py new file mode 100644 index 0000000000..f6511184cd --- /dev/null +++ b/api/src/backend/api/migrations/0085_finding_group_daily_summary_trgm_indexes.py @@ -0,0 +1,31 @@ +# Generated by Django 5.1.15 on 2026-03-18 + +from django.contrib.postgres.indexes import GinIndex, OpClass +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations +from django.db.models.functions import Upper + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0084_googleworkspace_provider"), + ] + + operations = [ + AddIndexConcurrently( + model_name="findinggroupdailysummary", + index=GinIndex( + OpClass(Upper("check_id"), name="gin_trgm_ops"), + name="fgds_check_id_trgm_idx", + ), + ), + AddIndexConcurrently( + model_name="findinggroupdailysummary", + index=GinIndex( + OpClass(Upper("check_title"), name="gin_trgm_ops"), + name="fgds_check_title_trgm_idx", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0086_attack_paths_cleanup_periodic_task.py b/api/src/backend/api/migrations/0086_attack_paths_cleanup_periodic_task.py new file mode 100644 index 0000000000..1c550f283a --- /dev/null +++ b/api/src/backend/api/migrations/0086_attack_paths_cleanup_periodic_task.py @@ -0,0 +1,49 @@ +from django.db import migrations + + +TASK_NAME = "attack-paths-cleanup-stale-scans" +INTERVAL_HOURS = 1 + + +def create_periodic_task(apps, schema_editor): + IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + + schedule, _ = IntervalSchedule.objects.get_or_create( + every=INTERVAL_HOURS, + period="hours", + ) + + PeriodicTask.objects.update_or_create( + name=TASK_NAME, + defaults={ + "task": TASK_NAME, + "interval": schedule, + "enabled": True, + }, + ) + + +def delete_periodic_task(apps, schema_editor): + IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + + PeriodicTask.objects.filter(name=TASK_NAME).delete() + + # Clean up the schedule if no other task references it + IntervalSchedule.objects.filter( + every=INTERVAL_HOURS, + period="hours", + periodictask__isnull=True, + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0085_finding_group_daily_summary_trgm_indexes"), + ("django_celery_beat", "0019_alter_periodictasks_options"), + ] + + operations = [ + migrations.RunPython(create_periodic_task, delete_periodic_task), + ] diff --git a/api/src/backend/api/migrations/0087_vercel_provider.py b/api/src/backend/api/migrations/0087_vercel_provider.py new file mode 100644 index 0000000000..84a07b3194 --- /dev/null +++ b/api/src/backend/api/migrations/0087_vercel_provider.py @@ -0,0 +1,40 @@ +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0086_attack_paths_cleanup_periodic_task"), + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("mongodbatlas", "MongoDB Atlas"), + ("iac", "IaC"), + ("oraclecloud", "Oracle Cloud Infrastructure"), + ("alibabacloud", "Alibaba Cloud"), + ("cloudflare", "Cloudflare"), + ("openstack", "OpenStack"), + ("image", "Image"), + ("googleworkspace", "Google Workspace"), + ("vercel", "Vercel"), + ], + default="aws", + ), + ), + migrations.RunSQL( + "ALTER TYPE provider ADD VALUE IF NOT EXISTS 'vercel';", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/api/src/backend/api/migrations/0088_finding_group_status_muted_fields.py b/api/src/backend/api/migrations/0088_finding_group_status_muted_fields.py new file mode 100644 index 0000000000..ff3b981435 --- /dev/null +++ b/api/src/backend/api/migrations/0088_finding_group_status_muted_fields.py @@ -0,0 +1,95 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0087_vercel_provider"), + ] + + operations = [ + migrations.AddField( + model_name="findinggroupdailysummary", + name="manual_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="pass_muted_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="fail_muted_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="manual_muted_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="muted", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="new_fail_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="new_fail_muted_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="new_pass_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="new_pass_muted_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="new_manual_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="new_manual_muted_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="changed_fail_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="changed_fail_muted_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="changed_pass_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="changed_pass_muted_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="changed_manual_count", + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name="findinggroupdailysummary", + name="changed_manual_muted_count", + field=models.IntegerField(default=0), + ), + ] diff --git a/api/src/backend/api/migrations/0089_backfill_finding_group_status_muted.py b/api/src/backend/api/migrations/0089_backfill_finding_group_status_muted.py new file mode 100644 index 0000000000..501fcf3cb4 --- /dev/null +++ b/api/src/backend/api/migrations/0089_backfill_finding_group_status_muted.py @@ -0,0 +1,31 @@ +from django.db import migrations +from tasks.tasks import backfill_finding_group_summaries_task + +from api.db_router import MainRouter +from api.rls import Tenant + + +def trigger_backfill_task(apps, schema_editor): + """ + Re-dispatch the finding-group backfill task for every tenant so the new + `manual_count` and `muted` columns added in 0088 get populated from the + last 10 days of completed scans. + + The aggregator (`aggregate_finding_group_summaries`) recomputes every + column on each call, so it back-populates the new fields without touching + the existing ones beyond a normal upsert. + """ + tenant_ids = Tenant.objects.using(MainRouter.admin_db).values_list("id", flat=True) + + for tenant_id in tenant_ids: + backfill_finding_group_summaries_task.delay(tenant_id=str(tenant_id), days=10) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0088_finding_group_status_muted_fields"), + ] + + operations = [ + migrations.RunPython(trigger_backfill_task, migrations.RunPython.noop), + ] diff --git a/api/src/backend/api/migrations/0090_attack_paths_cleanup_priority.py b/api/src/backend/api/migrations/0090_attack_paths_cleanup_priority.py new file mode 100644 index 0000000000..5ef8529b08 --- /dev/null +++ b/api/src/backend/api/migrations/0090_attack_paths_cleanup_priority.py @@ -0,0 +1,23 @@ +from django.db import migrations + +TASK_NAME = "attack-paths-cleanup-stale-scans" + + +def set_cleanup_priority(apps, schema_editor): + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + PeriodicTask.objects.filter(name=TASK_NAME).update(priority=0) + + +def unset_cleanup_priority(apps, schema_editor): + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + PeriodicTask.objects.filter(name=TASK_NAME).update(priority=None) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0089_backfill_finding_group_status_muted"), + ] + + operations = [ + migrations.RunPython(set_cleanup_priority, unset_cleanup_priority), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index e736ef403a..e7195cff5f 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -1,14 +1,15 @@ import json import logging import re -import xml.etree.ElementTree as ET from datetime import datetime, timedelta, timezone from uuid import UUID, uuid4 +import defusedxml from allauth.socialaccount.models import SocialApp from config.custom_logging import BackendLogger from config.settings.social_login import SOCIALACCOUNT_PROVIDERS from cryptography.fernet import Fernet, InvalidToken +from defusedxml import ElementTree as ET from django.conf import settings from django.contrib.auth.models import AbstractBaseUser from django.contrib.postgres.fields import ArrayField @@ -294,6 +295,7 @@ class Provider(RowLevelSecurityProtectedModel): OPENSTACK = "openstack", _("OpenStack") IMAGE = "image", _("Image") GOOGLEWORKSPACE = "googleworkspace", _("Google Workspace") + VERCEL = "vercel", _("Vercel") @staticmethod def validate_aws_uid(value): @@ -437,6 +439,15 @@ class Provider(RowLevelSecurityProtectedModel): pointer="/data/attributes/uid", ) + @staticmethod + def validate_vercel_uid(value): + if not re.match(r"^team_[a-zA-Z0-9]{16,32}$", value): + raise ModelValidationError( + detail="Vercel provider ID must be a valid Vercel Team ID (e.g., team_xxxxxxxxxxxxxxxxxxxxxxxx).", + code="vercel-uid", + pointer="/data/attributes/uid", + ) + @staticmethod def validate_image_uid(value): if not re.match(r"^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{2,249}$", value): @@ -584,10 +595,40 @@ class Scan(RowLevelSecurityProtectedModel): objects = ActiveProviderManager() all_objects = models.Manager() + _SCOPING_SCANNER_ARG_KEYS_CACHE: tuple[str, ...] | None = None + + @classmethod + def get_scoping_scanner_arg_keys(cls) -> tuple[str, ...]: + """Return the scanner_args keys that mark a scan as scoped. + + Derived from ``prowler.lib.scan.scan.Scan.__init__`` so the API stays + in sync with whatever the SDK actually accepts as filters. Cached at + class level — the signature is stable for the process lifetime. + """ + if cls._SCOPING_SCANNER_ARG_KEYS_CACHE is None: + import inspect + + from prowler.lib.scan.scan import Scan as ProwlerScan + + params = inspect.signature(ProwlerScan.__init__).parameters + cls._SCOPING_SCANNER_ARG_KEYS_CACHE = tuple( + name for name in params if name not in ("self", "provider") + ) + return cls._SCOPING_SCANNER_ARG_KEYS_CACHE + class TriggerChoices(models.TextChoices): SCHEDULED = "scheduled", _("Scheduled") MANUAL = "manual", _("Manual") + # Trigger values for scans that ran the SDK end-to-end. Imported scans (or + # any future trigger) are intentionally NOT in this set — they may carry + # only a partial slice of resources, so post-scan logic that depends on a + # full-scope sweep (e.g. resetting ephemeral resource findings) must skip + # them by default. + LIVE_SCAN_TRIGGERS = frozenset( + (TriggerChoices.SCHEDULED.value, TriggerChoices.MANUAL.value) + ) + id = models.UUIDField(primary_key=True, default=uuid7, editable=False) name = models.CharField( blank=True, null=True, max_length=100, validators=[MinLengthValidator(3)] @@ -670,6 +711,24 @@ class Scan(RowLevelSecurityProtectedModel): class JSONAPIMeta: resource_name = "scans" + def is_full_scope(self) -> bool: + """Return True if this scan ran with no scoping filters at all. + + Used to gate post-scan operations (such as resetting the + failed_findings_count of resources missing from the scan) that are only + safe when the scan covered every check, service, and category. Imported + scans are NOT full-scope by definition — they may carry only a partial + slice of resources, so they're rejected via ``trigger`` even before the + scanner_args check. + """ + if self.trigger not in self.LIVE_SCAN_TRIGGERS: + return False + scanner_args = self.scanner_args or {} + for key in self.get_scoping_scanner_arg_keys(): + if scanner_args.get(key): + return False + return True + class AttackPathsScan(RowLevelSecurityProtectedModel): objects = ActiveProviderManager() @@ -1737,15 +1796,45 @@ class FindingGroupDailySummary(RowLevelSecurityProtectedModel): # Severity stored as integer for MAX aggregation (5=critical, 4=high, etc.) severity_order = models.SmallIntegerField(default=1) - # Finding counts + # Finding counts (inclusive of muted findings; use the `muted` flag to + # tell whether the group has any actionable findings). pass_count = models.IntegerField(default=0) fail_count = models.IntegerField(default=0) + manual_count = models.IntegerField(default=0) muted_count = models.IntegerField(default=0) - # Delta counts + # Status counts restricted to muted findings, so clients can isolate the + # muted half of each status (e.g. `pass_count - pass_muted_count` gives the + # actionable PASS findings). + pass_muted_count = models.IntegerField(default=0) + fail_muted_count = models.IntegerField(default=0) + manual_muted_count = models.IntegerField(default=0) + + # Whether every finding for this (provider, check, day) is muted. + muted = models.BooleanField(default=False) + + # Delta counts (non-muted, kept for convenience and as a "total" view). new_count = models.IntegerField(default=0) changed_count = models.IntegerField(default=0) + # Delta breakdown by (status, muted) so clients can answer questions like + # "how many new failing findings appeared in this scan?" without scanning + # the underlying findings table. Mirrors the existing pass/fail/manual + # naming, with `_muted_count` siblings tracking the muted half of each + # bucket explicitly. + new_fail_count = models.IntegerField(default=0) + new_fail_muted_count = models.IntegerField(default=0) + new_pass_count = models.IntegerField(default=0) + new_pass_muted_count = models.IntegerField(default=0) + new_manual_count = models.IntegerField(default=0) + new_manual_muted_count = models.IntegerField(default=0) + changed_fail_count = models.IntegerField(default=0) + changed_fail_muted_count = models.IntegerField(default=0) + changed_pass_count = models.IntegerField(default=0) + changed_pass_muted_count = models.IntegerField(default=0) + changed_manual_count = models.IntegerField(default=0) + changed_manual_muted_count = models.IntegerField(default=0) + # Resource counts resources_fail = models.IntegerField(default=0) resources_total = models.IntegerField(default=0) @@ -1783,6 +1872,15 @@ class FindingGroupDailySummary(RowLevelSecurityProtectedModel): fields=["tenant_id", "provider", "inserted_at"], name="fgds_tenant_prov_ins_idx", ), + # Trigram indexes for case-insensitive search + GinIndex( + OpClass(Upper("check_id"), name="gin_trgm_ops"), + name="fgds_check_id_trgm_idx", + ), + GinIndex( + OpClass(Upper("check_title"), name="gin_trgm_ops"), + name="fgds_check_title_trgm_idx", + ), ] class JSONAPIMeta: @@ -2058,6 +2156,8 @@ class SAMLConfiguration(RowLevelSecurityProtectedModel): root = ET.fromstring(self.metadata_xml) except ET.ParseError as e: raise ValidationError({"metadata_xml": f"Invalid XML: {e}"}) + except defusedxml.DefusedXmlException as e: + raise ValidationError({"metadata_xml": f"Unsafe XML content rejected: {e}"}) # Entity ID entity_id = root.attrib.get("entityID") diff --git a/api/src/backend/api/rbac/permissions.py b/api/src/backend/api/rbac/permissions.py index 97d7d785e0..cfbabf6c0b 100644 --- a/api/src/backend/api/rbac/permissions.py +++ b/api/src/backend/api/rbac/permissions.py @@ -1,7 +1,7 @@ from enum import Enum -from typing import Optional from django.db.models import QuerySet +from rest_framework.exceptions import PermissionDenied from rest_framework.permissions import BasePermission from api.db_router import MainRouter @@ -29,8 +29,17 @@ class HasPermissions(BasePermission): if not required_permissions: return True + tenant_id = getattr(request, "tenant_id", None) + if not tenant_id: + tenant_id = request.auth.get("tenant_id") if request.auth else None + if not tenant_id: + return False + user_roles = ( - User.objects.using(MainRouter.admin_db).get(id=request.user.id).roles.all() + User.objects.using(MainRouter.admin_db) + .get(id=request.user.id) + .roles.using(MainRouter.admin_db) + .filter(tenant_id=tenant_id) ) if not user_roles: return False @@ -42,14 +51,17 @@ class HasPermissions(BasePermission): return True -def get_role(user: User) -> Optional[Role]: +def get_role(user: User, tenant_id: str) -> Role: """ - Retrieve the first role assigned to the given user. + Retrieve the role assigned to the given user in the specified tenant. - Returns: - The user's first Role instance if the user has any roles, otherwise None. + Raises: + PermissionDenied: If the user has no role in the given tenant. """ - return user.roles.first() + role = user.roles.using(MainRouter.admin_db).filter(tenant_id=tenant_id).first() + if role is None: + raise PermissionDenied("User has no role in this tenant.") + return role def get_providers(role: Role) -> QuerySet[Provider]: diff --git a/api/src/backend/api/signals.py b/api/src/backend/api/signals.py index d449144bf4..7bca0da0a6 100644 --- a/api/src/backend/api/signals.py +++ b/api/src/backend/api/signals.py @@ -61,7 +61,7 @@ def revoke_membership_api_keys(sender, instance, **kwargs): # noqa: F841 in that tenant should be revoked to prevent further access. """ TenantAPIKey.objects.filter( - entity=instance.user, tenant_id=instance.tenant.id + entity_id=instance.user_id, tenant_id=instance.tenant_id ).update(revoked=True) diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index be17679f42..c436c4cd8f 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.22.0 + version: 1.27.0 description: |- Prowler API specification. @@ -356,7 +356,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -372,6 +372,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -387,13 +388,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -409,6 +411,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -426,6 +429,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -1263,20 +1267,50 @@ paths: - check_description - severity - status + - muted - impacted_providers - resources_fail - resources_total - pass_count - fail_count + - manual_count + - pass_muted_count + - fail_muted_count + - manual_muted_count - muted_count - new_count - changed_count + - new_fail_count + - new_fail_muted_count + - new_pass_count + - new_pass_muted_count + - new_manual_count + - new_manual_muted_count + - changed_fail_count + - changed_fail_muted_count + - changed_pass_count + - changed_pass_muted_count + - changed_manual_count + - changed_manual_muted_count - first_seen_at - last_seen_at - failing_since description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false + - in: query + name: filter[category] + schema: + type: string + - in: query + name: filter[category__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[check_id] schema: @@ -1294,6 +1328,36 @@ paths: description: Multiple values may be separated by commas. explode: false style: form + - in: query + name: filter[check_title__icontains] + schema: + type: string + - in: query + name: filter[delta] + schema: + type: string + enum: + - changed + - new + description: |- + * `new` - New + * `changed` - Changed + - in: query + name: filter[impact] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational - in: query name: filter[inserted_at] schema: @@ -1316,6 +1380,44 @@ paths: type: string format: date description: Maximum date range is 7 days. + - in: query + name: filter[muted] + schema: + type: boolean + description: If this filter is not provided, muted and non-muted findings + will be returned. + - in: query + name: filter[provider] + schema: + type: string + format: uuid + - in: query + name: filter[provider__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_alias] + schema: + type: string + - in: query + name: filter[provider_alias__icontains] + schema: + type: string + - in: query + name: filter[provider_alias__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[provider_id] schema: @@ -1335,7 +1437,6 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 enum: - alibabacloud - aws @@ -1351,6 +1452,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -1366,8 +1468,59 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - alibabacloud + - aws + - azure + - cloudflare + - gcp + - github + - googleworkspace + - iac + - image + - kubernetes + - m365 + - mongodbatlas + - openstack + - oraclecloud + - vercel + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + * `cloudflare` - Cloudflare + * `openstack` - OpenStack + * `image` - Image + * `googleworkspace` - Google Workspace + * `vercel` - Vercel + explode: false + style: form + - in: query + name: filter[provider_uid] + schema: + type: string + - in: query + name: filter[provider_uid__icontains] + schema: + type: string + - in: query + name: filter[provider_uid__in] schema: type: array items: @@ -1375,12 +1528,166 @@ paths: description: Multiple values may be separated by commas. explode: false style: form - - name: filter[search] - required: false - in: query - description: A search term. + - in: query + name: filter[region] schema: type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_groups] + schema: + type: string + - in: query + name: filter[resource_groups__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_name] + schema: + type: string + - in: query + name: filter[resource_name__icontains] + schema: + type: string + - in: query + name: filter[resource_name__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_type] + schema: + type: string + - in: query + name: filter[resource_type__icontains] + schema: + type: string + - in: query + name: filter[resource_type__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_uid] + schema: + type: string + - in: query + name: filter[resource_uid__icontains] + schema: + type: string + - in: query + name: filter[resource_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resources] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[scan] + schema: + type: string + format: uuid + - in: query + name: filter[scan__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[service] + schema: + type: string + - in: query + name: filter[service__icontains] + schema: + type: string + - in: query + name: filter[service__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[severity] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[status] + schema: + type: string + enum: + - FAIL + - MANUAL + - PASS + description: |- + * `FAIL` - Fail + * `PASS` - Pass + * `MANUAL` - Manual + - in: query + name: filter[uid] + schema: + type: string + - in: query + name: filter[updated_at] + schema: + type: string + format: date - name: page[number] required: false in: query @@ -1414,6 +1721,8 @@ paths: - -severity - status - -status + - muted + - -muted - impacted_providers - -impacted_providers - resources_fail @@ -1424,12 +1733,44 @@ paths: - -pass_count - fail_count - -fail_count + - manual_count + - -manual_count + - pass_muted_count + - -pass_muted_count + - fail_muted_count + - -fail_muted_count + - manual_muted_count + - -manual_muted_count - muted_count - -muted_count - new_count - -new_count - changed_count - -changed_count + - new_fail_count + - -new_fail_count + - new_fail_muted_count + - -new_fail_muted_count + - new_pass_count + - -new_pass_count + - new_pass_muted_count + - -new_pass_muted_count + - new_manual_count + - -new_manual_count + - new_manual_muted_count + - -new_manual_muted_count + - changed_fail_count + - -changed_fail_count + - changed_fail_muted_count + - -changed_fail_muted_count + - changed_pass_count + - -changed_pass_count + - changed_pass_muted_count + - -changed_pass_muted_count + - changed_manual_count + - -changed_manual_count + - changed_manual_muted_count + - -changed_manual_muted_count - first_seen_at - -first_seen_at - last_seen_at @@ -1470,20 +1811,427 @@ paths: - check_description - severity - status + - muted - impacted_providers - resources_fail - resources_total - pass_count - fail_count + - manual_count + - pass_muted_count + - fail_muted_count + - manual_muted_count - muted_count - new_count - changed_count + - new_fail_count + - new_fail_muted_count + - new_pass_count + - new_pass_muted_count + - new_manual_count + - new_manual_muted_count + - changed_fail_count + - changed_fail_muted_count + - changed_pass_count + - changed_pass_muted_count + - changed_manual_count + - changed_manual_muted_count - first_seen_at - last_seen_at - failing_since description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false + - in: query + name: filter[category] + schema: + type: string + - in: query + name: filter[category__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[check_id] + schema: + type: string + - in: query + name: filter[check_id__icontains] + schema: + type: string + - in: query + name: filter[check_id__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[check_title__icontains] + schema: + type: string + - in: query + name: filter[delta] + schema: + type: string + enum: + - changed + - new + description: |- + * `new` - New + * `changed` - Changed + - in: query + name: filter[impact] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[inserted_at] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__date] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__gte] + schema: + type: string + format: date + description: Maximum date range is 7 days. + - in: query + name: filter[inserted_at__lte] + schema: + type: string + format: date + description: Maximum date range is 7 days. + - in: query + name: filter[muted] + schema: + type: boolean + description: If this filter is not provided, muted and non-muted findings + will be returned. + - in: query + name: filter[provider] + schema: + type: string + format: uuid + - in: query + name: filter[provider__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_alias] + schema: + type: string + - in: query + name: filter[provider_alias__icontains] + schema: + type: string + - in: query + name: filter[provider_alias__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - alibabacloud + - aws + - azure + - cloudflare + - gcp + - github + - googleworkspace + - iac + - image + - kubernetes + - m365 + - mongodbatlas + - openstack + - oraclecloud + - vercel + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + * `cloudflare` - Cloudflare + * `openstack` - OpenStack + * `image` - Image + * `googleworkspace` - Google Workspace + * `vercel` - Vercel + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - alibabacloud + - aws + - azure + - cloudflare + - gcp + - github + - googleworkspace + - iac + - image + - kubernetes + - m365 + - mongodbatlas + - openstack + - oraclecloud + - vercel + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + * `cloudflare` - Cloudflare + * `openstack` - OpenStack + * `image` - Image + * `googleworkspace` - Google Workspace + * `vercel` - Vercel + explode: false + style: form + - in: query + name: filter[provider_uid] + schema: + type: string + - in: query + name: filter[provider_uid__icontains] + schema: + type: string + - in: query + name: filter[provider_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_groups] + schema: + type: string + - in: query + name: filter[resource_groups__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_name] + schema: + type: string + - in: query + name: filter[resource_name__icontains] + schema: + type: string + - in: query + name: filter[resource_name__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_type] + schema: + type: string + - in: query + name: filter[resource_type__icontains] + schema: + type: string + - in: query + name: filter[resource_type__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_uid] + schema: + type: string + - in: query + name: filter[resource_uid__icontains] + schema: + type: string + - in: query + name: filter[resource_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resources] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[scan] + schema: + type: string + format: uuid + - in: query + name: filter[scan__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[service] + schema: + type: string + - in: query + name: filter[service__icontains] + schema: + type: string + - in: query + name: filter[service__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[severity] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[status] + schema: + type: string + enum: + - FAIL + - MANUAL + - PASS + description: |- + * `FAIL` - Fail + * `PASS` - Pass + * `MANUAL` - Manual + - in: query + name: filter[uid] + schema: + type: string + - in: query + name: filter[updated_at] + schema: + type: string + format: date - in: path name: id schema: @@ -1491,6 +2239,84 @@ paths: format: uuid description: A UUID string identifying this finding group daily summary. required: true + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - check_id + - -check_id + - check_title + - -check_title + - check_description + - -check_description + - severity + - -severity + - status + - -status + - muted + - -muted + - impacted_providers + - -impacted_providers + - resources_fail + - -resources_fail + - resources_total + - -resources_total + - pass_count + - -pass_count + - fail_count + - -fail_count + - manual_count + - -manual_count + - pass_muted_count + - -pass_muted_count + - fail_muted_count + - -fail_muted_count + - manual_muted_count + - -manual_muted_count + - muted_count + - -muted_count + - new_count + - -new_count + - changed_count + - -changed_count + - new_fail_count + - -new_fail_count + - new_fail_muted_count + - -new_fail_muted_count + - new_pass_count + - -new_pass_count + - new_pass_muted_count + - -new_pass_muted_count + - new_manual_count + - -new_manual_count + - new_manual_muted_count + - -new_manual_muted_count + - changed_fail_count + - -changed_fail_count + - changed_fail_muted_count + - -changed_fail_muted_count + - changed_pass_count + - -changed_pass_count + - changed_pass_muted_count + - -changed_pass_muted_count + - changed_manual_count + - -changed_manual_count + - changed_manual_muted_count + - -changed_manual_muted_count + - first_seen_at + - -first_seen_at + - last_seen_at + - -last_seen_at + - failing_since + - -failing_since + explode: false tags: - Finding Groups security: @@ -1525,14 +2351,31 @@ paths: - check_description - severity - status + - muted - impacted_providers - resources_fail - resources_total - pass_count - fail_count + - manual_count + - pass_muted_count + - fail_muted_count + - manual_muted_count - muted_count - new_count - changed_count + - new_fail_count + - new_fail_muted_count + - new_pass_count + - new_pass_muted_count + - new_manual_count + - new_manual_muted_count + - changed_fail_count + - changed_fail_muted_count + - changed_pass_count + - changed_pass_muted_count + - changed_manual_count + - changed_manual_muted_count - first_seen_at - last_seen_at - failing_since @@ -1577,20 +2420,483 @@ paths: - check_description - severity - status + - muted - impacted_providers - resources_fail - resources_total - pass_count - fail_count + - manual_count + - pass_muted_count + - fail_muted_count + - manual_muted_count - muted_count - new_count - changed_count + - new_fail_count + - new_fail_muted_count + - new_pass_count + - new_pass_muted_count + - new_manual_count + - new_manual_muted_count + - changed_fail_count + - changed_fail_muted_count + - changed_pass_count + - changed_pass_muted_count + - changed_manual_count + - changed_manual_muted_count - first_seen_at - last_seen_at - failing_since description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false + - in: query + name: filter[category] + schema: + type: string + - in: query + name: filter[category__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[check_id] + schema: + type: string + - in: query + name: filter[check_id__icontains] + schema: + type: string + - in: query + name: filter[check_id__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[check_title__icontains] + schema: + type: string + - in: query + name: filter[delta] + schema: + type: string + enum: + - changed + - new + description: |- + * `new` - New + * `changed` - Changed + - in: query + name: filter[impact] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[muted] + schema: + type: boolean + description: If this filter is not provided, muted and non-muted findings + will be returned. + - in: query + name: filter[provider] + schema: + type: string + format: uuid + - in: query + name: filter[provider__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_alias] + schema: + type: string + - in: query + name: filter[provider_alias__icontains] + schema: + type: string + - in: query + name: filter[provider_alias__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - alibabacloud + - aws + - azure + - cloudflare + - gcp + - github + - googleworkspace + - iac + - image + - kubernetes + - m365 + - mongodbatlas + - openstack + - oraclecloud + - vercel + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + * `cloudflare` - Cloudflare + * `openstack` - OpenStack + * `image` - Image + * `googleworkspace` - Google Workspace + * `vercel` - Vercel + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - alibabacloud + - aws + - azure + - cloudflare + - gcp + - github + - googleworkspace + - iac + - image + - kubernetes + - m365 + - mongodbatlas + - openstack + - oraclecloud + - vercel + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + * `cloudflare` - Cloudflare + * `openstack` - OpenStack + * `image` - Image + * `googleworkspace` - Google Workspace + * `vercel` - Vercel + explode: false + style: form + - in: query + name: filter[provider_uid] + schema: + type: string + - in: query + name: filter[provider_uid__icontains] + schema: + type: string + - in: query + name: filter[provider_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_groups] + schema: + type: string + - in: query + name: filter[resource_groups__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_name] + schema: + type: string + - in: query + name: filter[resource_name__icontains] + schema: + type: string + - in: query + name: filter[resource_name__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_type] + schema: + type: string + - in: query + name: filter[resource_type__icontains] + schema: + type: string + - in: query + name: filter[resource_type__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_uid] + schema: + type: string + - in: query + name: filter[resource_uid__icontains] + schema: + type: string + - in: query + name: filter[resource_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resources] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[scan] + schema: + type: string + format: uuid + - in: query + name: filter[scan__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[service] + schema: + type: string + - in: query + name: filter[service__icontains] + schema: + type: string + - in: query + name: filter[service__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[severity] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[status] + schema: + type: string + enum: + - FAIL + - MANUAL + - PASS + description: |- + * `FAIL` - Fail + * `PASS` - Pass + * `MANUAL` - Manual + - in: query + name: filter[uid] + schema: + type: string + - in: query + name: filter[updated_at] + schema: + type: string + format: date + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - check_id + - -check_id + - check_title + - -check_title + - check_description + - -check_description + - severity + - -severity + - status + - -status + - muted + - -muted + - impacted_providers + - -impacted_providers + - resources_fail + - -resources_fail + - resources_total + - -resources_total + - pass_count + - -pass_count + - fail_count + - -fail_count + - manual_count + - -manual_count + - pass_muted_count + - -pass_muted_count + - fail_muted_count + - -fail_muted_count + - manual_muted_count + - -manual_muted_count + - muted_count + - -muted_count + - new_count + - -new_count + - changed_count + - -changed_count + - new_fail_count + - -new_fail_count + - new_fail_muted_count + - -new_fail_muted_count + - new_pass_count + - -new_pass_count + - new_pass_muted_count + - -new_pass_muted_count + - new_manual_count + - -new_manual_count + - new_manual_muted_count + - -new_manual_muted_count + - changed_fail_count + - -changed_fail_count + - changed_fail_muted_count + - -changed_fail_muted_count + - changed_pass_count + - -changed_pass_count + - changed_pass_muted_count + - -changed_pass_muted_count + - changed_manual_count + - -changed_manual_count + - changed_manual_muted_count + - -changed_manual_muted_count + - first_seen_at + - -first_seen_at + - last_seen_at + - -last_seen_at + - failing_since + - -failing_since + explode: false tags: - Finding Groups security: @@ -1811,7 +3117,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -1827,6 +3133,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -1842,13 +3149,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -1864,6 +3172,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -1881,6 +3190,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -2413,7 +3723,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -2429,6 +3739,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -2444,13 +3755,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -2466,6 +3778,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -2483,6 +3796,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -2923,7 +4237,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -2939,6 +4253,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -2954,13 +4269,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -2976,6 +4292,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -2993,6 +4310,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -3431,7 +4749,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -3447,6 +4765,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -3462,13 +4781,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -3484,6 +4804,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -3501,6 +4822,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -3927,7 +5249,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -3943,6 +5265,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -3958,13 +5281,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -3980,6 +5304,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -3997,6 +5322,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -4410,8 +5736,16 @@ paths: /api/v1/integrations/{integration_pk}/jira/dispatches: post: operationId: integrations_jira_dispatches_create - description: Send a set of filtered findings to the given integration. At least - one finding filter must be provided. + description: |- + Send a set of filtered findings to the given integration. At least one finding filter must be provided. + + ## Known Limitations + + ### Issue Types with Required Custom Fields + + Certain Jira issue types (such as Epic) may require mandatory custom fields that Prowler does not currently populate when creating work items. If a selected issue type enforces required fields beyond the standard set (e.g., "Team", "Epic Name"), the work item creation will fail. + + To avoid this, select an issue type that does not require additional custom fields - **Task**, **Bug**, or **Story** typically work without restrictions. If unsure which issue types are available for a project, Prowler automatically fetches and displays them in the "Issue Type" selector when sending a finding. summary: Send findings to a Jira integration parameters: - in: query @@ -4472,6 +5806,47 @@ paths: task_args: null metadata: null description: '' + /api/v1/integrations/{integration_pk}/jira/issue_types: + get: + operationId: integrations_jira_issue_types_retrieve + description: Fetch the available issue types from Jira for a given project key + and update the integration configuration. + summary: Get available issue types for a Jira project + parameters: + - in: query + name: fields[jira-issue-types] + schema: + type: array + items: + type: string + enum: + - project_key + - issue_types + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: integration_pk + schema: + type: string + required: true + - in: query + name: project_key + schema: + type: string + description: The Jira project key to fetch issue types for. + required: true + tags: + - Integration + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/IntegrationJiraIssueTypesResponse' + description: '' /api/v1/integrations/{id}: get: operationId: integrations_retrieve @@ -5764,7 +7139,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -5780,6 +7155,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -5795,13 +7171,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -5817,6 +7194,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -5834,6 +7212,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - name: filter[search] @@ -5939,7 +7318,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -5955,6 +7334,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -5970,13 +7350,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -5992,6 +7373,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -6009,6 +7391,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - name: filter[search] @@ -6119,6 +7502,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -6134,6 +7518,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: @@ -6155,6 +7540,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -6172,6 +7558,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - name: filter[search] @@ -6298,7 +7685,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -6314,6 +7701,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -6329,13 +7717,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -6351,6 +7740,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -6368,6 +7758,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -6507,7 +7898,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -6523,6 +7914,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -6538,13 +7930,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -6560,6 +7953,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -6577,6 +7971,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -6726,6 +8121,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -6741,6 +8137,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: @@ -6762,6 +8159,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -6779,6 +8177,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - name: filter[search] @@ -6954,7 +8353,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -6970,6 +8369,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -6985,13 +8385,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -7007,6 +8408,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -7024,6 +8426,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -7128,7 +8531,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -7144,6 +8547,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -7159,13 +8563,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -7181,6 +8586,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -7198,6 +8604,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -7326,7 +8733,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -7342,6 +8749,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -7357,13 +8765,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -7379,6 +8788,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -7396,6 +8806,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -8165,7 +9576,7 @@ paths: name: filter[provider] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -8181,6 +9592,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -8196,13 +9608,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -8218,6 +9631,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -8235,13 +9649,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -8257,6 +9672,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -8272,13 +9688,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -8294,6 +9711,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -8311,6 +9729,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - name: filter[search] @@ -8964,7 +10383,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -8980,6 +10399,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -8995,13 +10415,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -9017,6 +10438,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -9034,6 +10456,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -9511,7 +10934,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -9527,6 +10950,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -9542,13 +10966,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -9564,6 +10989,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -9581,6 +11007,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -9871,7 +11298,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -9887,6 +11314,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -9902,13 +11330,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -9924,6 +11353,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -9941,6 +11371,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -10237,7 +11668,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -10253,6 +11684,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -10268,13 +11700,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -10290,6 +11723,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -10307,6 +11741,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -11049,6 +12484,21 @@ paths: schema: type: string format: date + - in: query + name: filter[id] + schema: + type: string + format: uuid + - in: query + name: filter[id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[inserted_at] schema: @@ -11113,7 +12563,7 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -11129,6 +12579,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- * `aws` - AWS * `azure` - Azure @@ -11144,13 +12595,14 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 enum: - alibabacloud - aws @@ -11166,6 +12618,7 @@ paths: - mongodbatlas - openstack - oraclecloud + - vercel description: |- Multiple values may be separated by commas. @@ -11183,6 +12636,7 @@ paths: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel explode: false style: form - in: query @@ -11477,6 +12931,73 @@ paths: schema: $ref: '#/components/schemas/ScanUpdateResponse' description: '' + /api/v1/scans/{id}/cis: + get: + operationId: scans_cis_retrieve + description: Download the CIS Benchmark compliance report as a PDF file. When + a provider ships multiple CIS versions, the report is generated for the highest + available version. + summary: Retrieve CIS Benchmark compliance report + parameters: + - in: query + name: fields[scans] + schema: + type: array + items: + type: string + enum: + - name + - trigger + - state + - unique_resource_count + - progress + - duration + - provider + - task + - inserted_at + - started_at + - completed_at + - scheduled_at + - next_scan_at + - processor + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this scan. + required: true + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - provider + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + tags: + - Scan + security: + - JWT or API Key: [] + responses: + '200': + description: PDF file containing the CIS compliance report + '202': + description: The task is in progress + '401': + description: API key missing or user not Authenticated + '403': + description: There is a problem with credentials + '404': + description: The scan has no CIS reports, or the CIS report generation task + has not started yet /api/v1/scans/{id}/compliance/{name}: get: operationId: scans_compliance_retrieve @@ -12291,12 +13812,53 @@ paths: description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false + - in: query + name: filter[date_joined] + schema: + type: string + format: date + - in: query + name: filter[date_joined__date] + schema: + type: string + format: date + - in: query + name: filter[date_joined__gte] + schema: + type: string + format: date-time + - in: query + name: filter[date_joined__lte] + schema: + type: string + format: date-time + - in: query + name: filter[role] + schema: + type: string + x-spec-enum-id: 12c359ee5dc51001 + enum: + - member + - owner + description: |- + * `owner` - Owner + * `member` - Member - name: filter[search] required: false in: query description: A search term. schema: type: string + - in: query + name: filter[tenant] + schema: + type: string + format: uuid + - in: query + name: filter[user] + schema: + type: string + format: uuid - name: page[number] required: false in: query @@ -12350,9 +13912,12 @@ paths: /api/v1/tenants/{tenant_pk}/memberships/{id}: delete: operationId: tenants_memberships_destroy - description: Delete the membership details of users in a tenant. You need to - be one of the owners to delete a membership that is not yours. If you are - the last owner of a tenant, you cannot delete your own membership. + description: 'Delete a user''s membership from a tenant. This action: (1) removes + the membership, (2) revokes all refresh tokens for the expelled user, (3) + removes their role grants for this tenant, (4) cleans up orphaned roles, and + (5) deletes the user account if this was their last membership. You must be + a tenant owner to delete another user''s membership. The last owner of a tenant + cannot delete their own membership.' summary: Delete tenant memberships parameters: - in: path @@ -13175,6 +14740,11 @@ paths: schema: type: string format: uuid + - in: query + name: filter[user] + schema: + type: string + format: uuid - name: page[number] required: false in: query @@ -13370,6 +14940,7 @@ components: query: type: string minLength: 1 + maxLength: 10000 required: - query required: @@ -14239,6 +15810,8 @@ components: type: string status: type: string + muted: + type: boolean impacted_providers: type: array items: @@ -14251,12 +15824,44 @@ components: type: integer fail_count: type: integer + manual_count: + type: integer + pass_muted_count: + type: integer + fail_muted_count: + type: integer + manual_muted_count: + type: integer muted_count: type: integer new_count: type: integer changed_count: type: integer + new_fail_count: + type: integer + new_fail_muted_count: + type: integer + new_pass_count: + type: integer + new_pass_muted_count: + type: integer + new_manual_count: + type: integer + new_manual_muted_count: + type: integer + changed_fail_count: + type: integer + changed_fail_muted_count: + type: integer + changed_pass_count: + type: integer + changed_pass_muted_count: + type: integer + changed_manual_count: + type: integer + changed_manual_muted_count: + type: integer first_seen_at: type: string format: date-time @@ -14274,13 +15879,30 @@ components: - check_id - severity - status + - muted - resources_fail - resources_total - pass_count - fail_count + - manual_count + - pass_muted_count + - fail_muted_count + - manual_muted_count - muted_count - new_count - changed_count + - new_fail_count + - new_fail_muted_count + - new_pass_count + - new_pass_muted_count + - new_manual_count + - new_manual_muted_count + - changed_fail_count + - changed_fail_muted_count + - changed_pass_count + - changed_pass_muted_count + - changed_manual_count + - changed_manual_muted_count FindingGroupResponse: type: object properties: @@ -14882,16 +16504,46 @@ components: type: string minLength: 1 issue_type: - enum: - - Task type: string - description: '* `Task` - Task' - x-spec-enum-id: b527b0cec62087c1 + minLength: 1 required: - project_key - issue_type required: - data + IntegrationJiraIssueTypes: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - jira-issue-types + id: {} + attributes: + type: object + properties: + project_key: + type: string + readOnly: true + issue_types: + type: array + items: + type: string + readOnly: true + IntegrationJiraIssueTypesResponse: + type: object + properties: + data: + $ref: '#/components/schemas/IntegrationJiraIssueTypes' + required: + - data IntegrationResponse: type: object properties: @@ -18463,6 +20115,15 @@ components: required: - clouds_yaml_content - clouds_yaml_cloud + - type: object + title: Vercel API Token + properties: + api_token: + type: string + description: Vercel API token for authentication. Can be scoped + to a specific team. + required: + - api_token writeOnly: true required: - secret @@ -19465,6 +21126,7 @@ components: - openstack - image - googleworkspace + - vercel type: string description: |- * `aws` - AWS @@ -19481,7 +21143,8 @@ components: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace - x-spec-enum-id: c0d56cad8ab9abe5 + * `vercel` - Vercel + x-spec-enum-id: 91f917e0c3ab97e8 uid: type: string title: Unique identifier for the provider, set by the provider @@ -19601,8 +21264,9 @@ components: - openstack - image - googleworkspace + - vercel type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 description: |- Type of provider to create. @@ -19620,6 +21284,7 @@ components: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel uid: type: string title: Unique identifier for the provider, set by the provider @@ -19671,8 +21336,9 @@ components: - openstack - image - googleworkspace + - vercel type: string - x-spec-enum-id: c0d56cad8ab9abe5 + x-spec-enum-id: 91f917e0c3ab97e8 description: |- Type of provider to create. @@ -19690,6 +21356,7 @@ components: * `openstack` - OpenStack * `image` - Image * `googleworkspace` - Google Workspace + * `vercel` - Vercel uid: type: string minLength: 3 @@ -20539,6 +22206,15 @@ components: required: - clouds_yaml_content - clouds_yaml_cloud + - type: object + title: Vercel API Token + properties: + api_token: + type: string + description: Vercel API token for authentication. Can be scoped + to a specific team. + required: + - api_token writeOnly: true required: - secret_type @@ -20955,6 +22631,15 @@ components: required: - clouds_yaml_content - clouds_yaml_cloud + - type: object + title: Vercel API Token + properties: + api_token: + type: string + description: Vercel API token for authentication. Can be scoped + to a specific team. + required: + - api_token writeOnly: true required: - secret_type @@ -21381,6 +23066,15 @@ components: required: - clouds_yaml_content - clouds_yaml_cloud + - type: object + title: Vercel API Token + properties: + api_token: + type: string + description: Vercel API token for authentication. Can be scoped + to a specific team. + required: + - api_token writeOnly: true required: - secret diff --git a/api/src/backend/api/tests/integration/test_authentication.py b/api/src/backend/api/tests/integration/test_authentication.py index 2abf725124..061c2efac0 100644 --- a/api/src/backend/api/tests/integration/test_authentication.py +++ b/api/src/backend/api/tests/integration/test_authentication.py @@ -215,6 +215,21 @@ class TestTokenSwitchTenant: tenant_id = tenants_fixture[0].id user_instance = User.objects.get(email=test_user) Membership.objects.create(user=user_instance, tenant_id=tenant_id) + # Assign an admin role in the target tenant so the user can access resources + target_role = Role.objects.create( + name="admin", + tenant_id=tenant_id, + manage_users=True, + manage_account=True, + manage_billing=True, + manage_providers=True, + manage_integrations=True, + manage_scans=True, + unlimited_visibility=True, + ) + UserRoleRelationship.objects.create( + user=user_instance, role=target_role, tenant_id=tenant_id + ) # Check that using our new user's credentials we can authenticate and get the providers access_token, _ = get_api_tokens(client, test_user, test_password) @@ -301,7 +316,7 @@ class TestTokenSwitchTenant: assert invalid_tenant_response.status_code == 400 assert invalid_tenant_response.json()["errors"][0]["code"] == "invalid" assert invalid_tenant_response.json()["errors"][0]["detail"] == ( - "Tenant does not exist or user is not a " "member." + "Tenant does not exist or user is not a member." ) @@ -912,10 +927,9 @@ class TestAPIKeyLifecycle: auth_response = client.get(reverse("provider-list"), headers=api_key_headers) # Must return 401 Unauthorized, not 500 Internal Server Error - assert auth_response.status_code == 401, ( - f"Expected 401 but got {auth_response.status_code}: " - f"{auth_response.json()}" - ) + assert ( + auth_response.status_code == 401 + ), f"Expected 401 but got {auth_response.status_code}: {auth_response.json()}" # Verify error message is present response_json = auth_response.json() diff --git a/api/src/backend/api/tests/test_apps.py b/api/src/backend/api/tests/test_apps.py index 712bc33882..5889b4e2cb 100644 --- a/api/src/backend/api/tests/test_apps.py +++ b/api/src/backend/api/tests/test_apps.py @@ -10,11 +10,11 @@ from django.conf import settings import api import api.apps as api_apps_module from api.apps import ( - ApiConfig, PRIVATE_KEY_FILE, PUBLIC_KEY_FILE, SIGNING_KEY_ENV, VERIFYING_KEY_ENV, + ApiConfig, ) @@ -187,9 +187,10 @@ def test_ready_initializes_driver_for_api_process(monkeypatch): _set_argv(monkeypatch, ["gunicorn"]) _set_testing(monkeypatch, False) - with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch( - "api.attack_paths.database.init_driver" - ) as init_driver: + with ( + patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), + patch("api.attack_paths.database.init_driver") as init_driver, + ): config.ready() init_driver.assert_called_once() @@ -200,9 +201,10 @@ def test_ready_skips_driver_for_celery(monkeypatch): _set_argv(monkeypatch, ["celery", "-A", "api"]) _set_testing(monkeypatch, False) - with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch( - "api.attack_paths.database.init_driver" - ) as init_driver: + with ( + patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), + patch("api.attack_paths.database.init_driver") as init_driver, + ): config.ready() init_driver.assert_not_called() @@ -213,9 +215,10 @@ def test_ready_skips_driver_for_manage_py_skip_command(monkeypatch): _set_argv(monkeypatch, ["manage.py", "migrate"]) _set_testing(monkeypatch, False) - with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch( - "api.attack_paths.database.init_driver" - ) as init_driver: + with ( + patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), + patch("api.attack_paths.database.init_driver") as init_driver, + ): config.ready() init_driver.assert_not_called() @@ -226,9 +229,10 @@ def test_ready_skips_driver_when_testing(monkeypatch): _set_argv(monkeypatch, ["gunicorn"]) _set_testing(monkeypatch, True) - with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch( - "api.attack_paths.database.init_driver" - ) as init_driver: + with ( + patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), + patch("api.attack_paths.database.init_driver") as init_driver, + ): config.ready() init_driver.assert_not_called() diff --git a/api/src/backend/api/tests/test_attack_paths.py b/api/src/backend/api/tests/test_attack_paths.py index 3d164bf825..019b6aa1f2 100644 --- a/api/src/backend/api/tests/test_attack_paths.py +++ b/api/src/backend/api/tests/test_attack_paths.py @@ -9,6 +9,10 @@ from rest_framework.exceptions import APIException, PermissionDenied, Validation from api.attack_paths import database as graph_database from api.attack_paths import views_helpers +from tasks.jobs.attack_paths.config import ( + PROVIDER_ELEMENT_ID_PROPERTY, + get_provider_label, +) def _make_neo4j_error(message, code): @@ -49,7 +53,7 @@ def test_prepare_parameters_includes_provider_and_casts( ) assert result["provider_uid"] == "123456789012" - assert result["provider_id"] == "test-provider-id" + assert "provider_id" not in result assert result["limit"] == 5 @@ -103,12 +107,12 @@ def test_execute_query_serializes_graph( parameters = {"provider_uid": "123"} provider_id = "test-provider-123" + plabel = get_provider_label(provider_id) node = attack_paths_graph_stub_classes.Node( element_id="node-1", - labels=["AWSAccount"], + labels=["AWSAccount", plabel], properties={ "name": "account", - "provider_id": provider_id, "complex": { "items": [ attack_paths_graph_stub_classes.NativeValue("value"), @@ -117,15 +121,13 @@ def test_execute_query_serializes_graph( }, }, ) - node_2 = attack_paths_graph_stub_classes.Node( - "node-2", ["RDSInstance"], {"provider_id": provider_id} - ) + node_2 = attack_paths_graph_stub_classes.Node("node-2", ["RDSInstance", plabel], {}) relationship = attack_paths_graph_stub_classes.Relationship( element_id="rel-1", rel_type="OWNS", start_node=node, end_node=node_2, - properties={"weight": 1, "provider_id": provider_id}, + properties={"weight": 1}, ) graph = SimpleNamespace(nodes=[node, node_2], relationships=[relationship]) @@ -209,29 +211,27 @@ def test_execute_query_raises_permission_denied_on_read_only( ) -def test_serialize_graph_filters_by_provider_id(attack_paths_graph_stub_classes): +def test_serialize_graph_filters_by_provider_label(attack_paths_graph_stub_classes): provider_id = "provider-keep" + plabel = get_provider_label(provider_id) + other_label = get_provider_label("provider-other") - node_keep = attack_paths_graph_stub_classes.Node( - "n1", ["AWSAccount"], {"provider_id": provider_id} - ) + node_keep = attack_paths_graph_stub_classes.Node("n1", ["AWSAccount", plabel], {}) node_drop = attack_paths_graph_stub_classes.Node( - "n2", ["AWSAccount"], {"provider_id": "provider-other"} + "n2", ["AWSAccount", other_label], {} ) rel_keep = attack_paths_graph_stub_classes.Relationship( - "r1", "OWNS", node_keep, node_keep, {"provider_id": provider_id} - ) - rel_drop_by_provider = attack_paths_graph_stub_classes.Relationship( - "r2", "OWNS", node_keep, node_drop, {"provider_id": "provider-other"} + "r1", "OWNS", node_keep, node_keep, {} ) + # Relationship connecting a kept node to a dropped node — filtered by endpoint check rel_drop_orphaned = attack_paths_graph_stub_classes.Relationship( - "r3", "OWNS", node_keep, node_drop, {"provider_id": provider_id} + "r2", "OWNS", node_keep, node_drop, {} ) graph = SimpleNamespace( nodes=[node_keep, node_drop], - relationships=[rel_keep, rel_drop_by_provider, rel_drop_orphaned], + relationships=[rel_keep, rel_drop_orphaned], ) result = views_helpers._serialize_graph(graph, provider_id) @@ -350,10 +350,7 @@ def test_serialize_properties_filters_internal_fields(): "_module_name": "cartography:aws", "_module_version": "0.98.0", # Provider isolation - "_provider_id": "42", - "_provider_element_id": "42:abc123", - "provider_id": "42", - "provider_element_id": "42:abc123", + PROVIDER_ELEMENT_ID_PROPERTY: "42:abc123", } result = views_helpers._serialize_properties(properties) @@ -361,6 +358,14 @@ def test_serialize_properties_filters_internal_fields(): assert result == {"name": "prod"} +def test_filter_labels_strips_dynamic_isolation_labels(): + labels = ["AWSRole", "_Tenant_abc123", "_Provider_def456", "_ProviderResource"] + + result = views_helpers._filter_labels(labels) + + assert result == ["AWSRole"] + + def test_serialize_graph_as_text_node_without_properties(): graph = { "nodes": [{"id": "n1", "labels": ["AWSAccount"], "properties": {}}], @@ -439,14 +444,11 @@ def test_execute_custom_query_serializes_graph( attack_paths_graph_stub_classes, ): provider_id = "test-provider-123" - node_1 = attack_paths_graph_stub_classes.Node( - "node-1", ["AWSAccount"], {"provider_id": provider_id} - ) - node_2 = attack_paths_graph_stub_classes.Node( - "node-2", ["RDSInstance"], {"provider_id": provider_id} - ) + plabel = get_provider_label(provider_id) + node_1 = attack_paths_graph_stub_classes.Node("node-1", ["AWSAccount", plabel], {}) + node_2 = attack_paths_graph_stub_classes.Node("node-2", ["RDSInstance", plabel], {}) relationship = attack_paths_graph_stub_classes.Relationship( - "rel-1", "OWNS", node_1, node_2, {"provider_id": provider_id} + "rel-1", "OWNS", node_1, node_2, {} ) graph_result = MagicMock() @@ -461,10 +463,11 @@ def test_execute_custom_query_serializes_graph( "db-tenant-test", "MATCH (n) RETURN n", provider_id ) - mock_execute.assert_called_once_with( - database="db-tenant-test", - cypher="MATCH (n) RETURN n", - ) + mock_execute.assert_called_once() + call_kwargs = mock_execute.call_args[1] + assert call_kwargs["database"] == "db-tenant-test" + # The cypher is rewritten with the provider label injection + assert plabel in call_kwargs["cypher"] assert len(result["nodes"]) == 2 assert result["relationships"][0]["label"] == "OWNS" assert result["truncated"] is False @@ -501,72 +504,6 @@ def test_execute_custom_query_wraps_graph_errors(): mock_logger.error.assert_called_once() -# -- validate_custom_query ------------------------------------------------ - - -@pytest.mark.parametrize( - "cypher", - [ - "LOAD CSV FROM 'http://169.254.169.254/' AS x RETURN x", - "load csv from 'http://evil.com' as row return row", - "CALL apoc.load.json('http://evil.com/') YIELD value RETURN value", - "CALL apoc.load.csvParams('http://evil.com/', {}, null) YIELD list RETURN list", - "CALL apoc.import.csv([{fileName: 'f'}], [], {}) YIELD node RETURN node", - "CALL apoc.export.csv.all('file.csv', {})", - "CALL apoc.cypher.run('CREATE (n)', {}) YIELD value RETURN value", - "CALL apoc.systemdb.graph() YIELD nodes RETURN nodes", - "CALL apoc.config.list() YIELD key, value RETURN key, value", - "CALL apoc.periodic.iterate('MATCH (n) RETURN n', 'DELETE n', {batchSize: 100})", - "CALL apoc.do.when(true, 'CREATE (n) RETURN n', '', {}) YIELD value RETURN value", - "CALL apoc.trigger.add('t', 'RETURN 1', {phase: 'before'})", - "CALL apoc.custom.asProcedure('myProc', 'RETURN 1')", - ], - ids=[ - "LOAD_CSV", - "LOAD_CSV_lowercase", - "apoc.load.json", - "apoc.load.csvParams", - "apoc.import.csv", - "apoc.export.csv", - "apoc.cypher.run", - "apoc.systemdb.graph", - "apoc.config.list", - "apoc.periodic.iterate", - "apoc.do.when", - "apoc.trigger.add", - "apoc.custom.asProcedure", - ], -) -def test_validate_custom_query_rejects_blocked_patterns(cypher): - with pytest.raises(ValidationError) as exc: - views_helpers.validate_custom_query(cypher) - - assert "blocked operation" in str(exc.value.detail) - - -@pytest.mark.parametrize( - "cypher", - [ - "MATCH (n:AWSAccount) RETURN n LIMIT 10", - "MATCH (a)-[r]->(b) RETURN a, r, b", - "MATCH (n) WHERE n.name CONTAINS 'load' RETURN n", - "CALL apoc.create.vNode(['Label'], {}) YIELD node RETURN node", - "MATCH (n) WHERE n.name = 'apoc.load.json' RETURN n", - 'MATCH (n) WHERE n.description = "LOAD CSV is cool" RETURN n', - ], - ids=[ - "simple_match", - "traversal", - "contains_load_substring", - "apoc_virtual_node", - "apoc_load_inside_single_quotes", - "load_csv_inside_double_quotes", - ], -) -def test_validate_custom_query_allows_clean_queries(cypher): - views_helpers.validate_custom_query(cypher) - - # -- _truncate_graph ---------------------------------------------------------- diff --git a/api/src/backend/api/tests/test_attack_paths_database.py b/api/src/backend/api/tests/test_attack_paths_database.py index 8b458cb7b7..8828d23911 100644 --- a/api/src/backend/api/tests/test_attack_paths_database.py +++ b/api/src/backend/api/tests/test_attack_paths_database.py @@ -12,6 +12,8 @@ from unittest.mock import MagicMock, patch import neo4j import pytest +import api.attack_paths.database as db_module + class TestLazyInitialization: """Test that Neo4j driver is initialized lazily on first use.""" @@ -19,8 +21,6 @@ class TestLazyInitialization: @pytest.fixture(autouse=True) def reset_module_state(self): """Reset module-level singleton state before each test.""" - import api.attack_paths.database as db_module - original_driver = db_module._driver db_module._driver = None @@ -31,8 +31,6 @@ class TestLazyInitialization: def test_driver_not_initialized_at_import(self): """Driver should be None after module import (no eager connection).""" - import api.attack_paths.database as db_module - assert db_module._driver is None @patch("api.attack_paths.database.settings") @@ -41,8 +39,6 @@ class TestLazyInitialization: self, mock_driver_factory, mock_settings ): """init_driver() should create connection only when called.""" - import api.attack_paths.database as db_module - mock_driver = MagicMock() mock_driver_factory.return_value = mock_driver mock_settings.DATABASES = { @@ -69,8 +65,6 @@ class TestLazyInitialization: self, mock_driver_factory, mock_settings ): """Subsequent calls should return cached driver without reconnecting.""" - import api.attack_paths.database as db_module - mock_driver = MagicMock() mock_driver_factory.return_value = mock_driver mock_settings.DATABASES = { @@ -99,8 +93,6 @@ class TestLazyInitialization: self, mock_driver_factory, mock_settings ): """get_driver() should use init_driver() for lazy initialization.""" - import api.attack_paths.database as db_module - mock_driver = MagicMock() mock_driver_factory.return_value = mock_driver mock_settings.DATABASES = { @@ -118,14 +110,50 @@ class TestLazyInitialization: mock_driver_factory.assert_called_once() +class TestConnectionAcquisitionTimeout: + """Test that the connection acquisition timeout is configurable.""" + + @pytest.fixture(autouse=True) + def reset_module_state(self): + original_driver = db_module._driver + original_timeout = db_module.CONN_ACQUISITION_TIMEOUT + + db_module._driver = None + + yield + + db_module._driver = original_driver + db_module.CONN_ACQUISITION_TIMEOUT = original_timeout + + @patch("api.attack_paths.database.settings") + @patch("api.attack_paths.database.neo4j.GraphDatabase.driver") + def test_driver_receives_configured_timeout( + self, mock_driver_factory, mock_settings + ): + """init_driver() should pass CONN_ACQUISITION_TIMEOUT to the neo4j driver.""" + mock_driver_factory.return_value = MagicMock() + mock_settings.DATABASES = { + "neo4j": { + "HOST": "localhost", + "PORT": 7687, + "USER": "neo4j", + "PASSWORD": "password", + } + } + db_module.CONN_ACQUISITION_TIMEOUT = 42 + + db_module.init_driver() + + _, kwargs = mock_driver_factory.call_args + assert kwargs["connection_acquisition_timeout"] == 42 + + class TestAtexitRegistration: """Test that atexit cleanup handler is registered correctly.""" @pytest.fixture(autouse=True) def reset_module_state(self): """Reset module-level singleton state before each test.""" - import api.attack_paths.database as db_module - original_driver = db_module._driver db_module._driver = None @@ -141,8 +169,6 @@ class TestAtexitRegistration: self, mock_driver_factory, mock_atexit_register, mock_settings ): """atexit.register should be called on first initialization.""" - import api.attack_paths.database as db_module - mock_driver_factory.return_value = MagicMock() mock_settings.DATABASES = { "neo4j": { @@ -168,8 +194,6 @@ class TestAtexitRegistration: The double-checked locking on _driver ensures the atexit registration block only executes once (when _driver is first created). """ - import api.attack_paths.database as db_module - mock_driver_factory.return_value = MagicMock() mock_settings.DATABASES = { "neo4j": { @@ -194,8 +218,6 @@ class TestCloseDriver: @pytest.fixture(autouse=True) def reset_module_state(self): """Reset module-level singleton state before each test.""" - import api.attack_paths.database as db_module - original_driver = db_module._driver db_module._driver = None @@ -206,8 +228,6 @@ class TestCloseDriver: def test_close_driver_closes_and_clears_driver(self): """close_driver() should close the driver and set it to None.""" - import api.attack_paths.database as db_module - mock_driver = MagicMock() db_module._driver = mock_driver @@ -218,8 +238,6 @@ class TestCloseDriver: def test_close_driver_handles_none_driver(self): """close_driver() should handle case where driver is None.""" - import api.attack_paths.database as db_module - db_module._driver = None # Should not raise @@ -229,8 +247,6 @@ class TestCloseDriver: def test_close_driver_clears_driver_even_on_close_error(self): """Driver should be cleared even if close() raises an exception.""" - import api.attack_paths.database as db_module - mock_driver = MagicMock() mock_driver.close.side_effect = Exception("Connection error") db_module._driver = mock_driver @@ -246,8 +262,6 @@ class TestExecuteReadQuery: """Test read query execution helper.""" def test_execute_read_query_calls_read_session_and_returns_result(self): - import api.attack_paths.database as db_module - tx = MagicMock() expected_graph = MagicMock() run_result = MagicMock() @@ -289,8 +303,6 @@ class TestExecuteReadQuery: assert result is expected_graph def test_execute_read_query_defaults_parameters_to_empty_dict(self): - import api.attack_paths.database as db_module - tx = MagicMock() run_result = MagicMock() run_result.graph.return_value = MagicMock() @@ -325,8 +337,6 @@ class TestGetSessionReadOnly: @pytest.fixture(autouse=True) def reset_module_state(self): - import api.attack_paths.database as db_module - original_driver = db_module._driver db_module._driver = None yield @@ -341,8 +351,6 @@ class TestGetSessionReadOnly: ) def test_get_session_raises_write_query_not_allowed(self, neo4j_code): """Read-mode Neo4j errors should raise `WriteQueryNotAllowedException`.""" - import api.attack_paths.database as db_module - mock_session = MagicMock() neo4j_error = neo4j.exceptions.Neo4jError._hydrate_neo4j( code=neo4j_code, @@ -362,8 +370,6 @@ class TestGetSessionReadOnly: def test_get_session_raises_generic_exception_for_other_errors(self): """Non-read-mode Neo4j errors should raise GraphDatabaseQueryException.""" - import api.attack_paths.database as db_module - mock_session = MagicMock() neo4j_error = neo4j.exceptions.Neo4jError._hydrate_neo4j( code="Neo.ClientError.Statement.SyntaxError", @@ -388,8 +394,6 @@ class TestThreadSafety: @pytest.fixture(autouse=True) def reset_module_state(self): """Reset module-level singleton state before each test.""" - import api.attack_paths.database as db_module - original_driver = db_module._driver db_module._driver = None @@ -404,8 +408,6 @@ class TestThreadSafety: self, mock_driver_factory, mock_settings ): """Multiple threads calling init_driver() should create only one driver.""" - import api.attack_paths.database as db_module - mock_driver = MagicMock() mock_driver_factory.return_value = mock_driver mock_settings.DATABASES = { @@ -442,3 +444,70 @@ class TestThreadSafety: # All threads got the same driver instance assert all(r is mock_driver for r in results) assert len(results) == 10 + + +class TestHasProviderData: + """Test has_provider_data helper for checking provider nodes in Neo4j.""" + + def test_returns_true_when_nodes_exist(self): + mock_session = MagicMock() + mock_result = MagicMock() + mock_result.single.return_value = MagicMock() # non-None record + mock_session.run.return_value = mock_result + + session_ctx = MagicMock() + session_ctx.__enter__.return_value = mock_session + session_ctx.__exit__.return_value = False + + with patch( + "api.attack_paths.database.get_session", + return_value=session_ctx, + ): + assert db_module.has_provider_data("db-tenant-abc", "provider-123") is True + + mock_session.run.assert_called_once() + + def test_returns_false_when_no_nodes(self): + mock_session = MagicMock() + mock_result = MagicMock() + mock_result.single.return_value = None + mock_session.run.return_value = mock_result + + session_ctx = MagicMock() + session_ctx.__enter__.return_value = mock_session + session_ctx.__exit__.return_value = False + + with patch( + "api.attack_paths.database.get_session", + return_value=session_ctx, + ): + assert db_module.has_provider_data("db-tenant-abc", "provider-123") is False + + def test_returns_false_when_database_not_found(self): + session_ctx = MagicMock() + session_ctx.__enter__.side_effect = db_module.GraphDatabaseQueryException( + message="Database does not exist", + code="Neo.ClientError.Database.DatabaseNotFound", + ) + + with patch( + "api.attack_paths.database.get_session", + return_value=session_ctx, + ): + assert ( + db_module.has_provider_data("db-tenant-gone", "provider-123") is False + ) + + def test_raises_on_other_errors(self): + session_ctx = MagicMock() + session_ctx.__enter__.side_effect = db_module.GraphDatabaseQueryException( + message="Connection refused", + code="Neo.TransientError.General.UnknownError", + ) + + with patch( + "api.attack_paths.database.get_session", + return_value=session_ctx, + ): + with pytest.raises(db_module.GraphDatabaseQueryException): + db_module.has_provider_data("db-tenant-abc", "provider-123") diff --git a/api/src/backend/api/tests/test_celery_settings.py b/api/src/backend/api/tests/test_celery_settings.py new file mode 100644 index 0000000000..d4010796ee --- /dev/null +++ b/api/src/backend/api/tests/test_celery_settings.py @@ -0,0 +1,43 @@ +import pytest +from config.settings.celery import _build_celery_broker_url + + +class TestBuildCeleryBrokerUrl: + def test_without_credentials(self): + broker_url = _build_celery_broker_url("redis", "", "", "valkey", "6379", "0") + + assert broker_url == "redis://valkey:6379/0" + + def test_with_password_only(self): + broker_url = _build_celery_broker_url( + "rediss", "", "secret", "cache.example.com", "6379", "0" + ) + + assert broker_url == "rediss://:secret@cache.example.com:6379/0" + + def test_with_username_and_password(self): + broker_url = _build_celery_broker_url( + "rediss", "default", "secret", "cache.example.com", "6379", "0" + ) + + assert broker_url == "rediss://default:secret@cache.example.com:6379/0" + + def test_with_username_only(self): + broker_url = _build_celery_broker_url( + "redis", "admin", "", "valkey", "6379", "0" + ) + + assert broker_url == "redis://admin@valkey:6379/0" + + def test_url_encodes_credentials(self): + broker_url = _build_celery_broker_url( + "rediss", "user@name", "p@ss:word", "cache.example.com", "6379", "0" + ) + + assert ( + broker_url == "rediss://user%40name:p%40ss%3Aword@cache.example.com:6379/0" + ) + + def test_invalid_scheme_raises_error(self): + with pytest.raises(ValueError, match="Invalid VALKEY_SCHEME 'http'"): + _build_celery_broker_url("http", "", "", "valkey", "6379", "0") diff --git a/api/src/backend/api/tests/test_compliance.py b/api/src/backend/api/tests/test_compliance.py index 8774f33787..ce30a3cc52 100644 --- a/api/src/backend/api/tests/test_compliance.py +++ b/api/src/backend/api/tests/test_compliance.py @@ -1,13 +1,18 @@ from unittest.mock import MagicMock, patch +import pytest + +from api import compliance as compliance_module from api.compliance import ( generate_compliance_overview_template, generate_scan_compliance, + get_compliance_frameworks, get_prowler_provider_checks, get_prowler_provider_compliance, load_prowler_checks, ) from api.models import Provider +from prowler.lib.check.compliance_models import Compliance class TestCompliance: @@ -250,3 +255,58 @@ class TestCompliance: } assert template == expected_template + + +@pytest.fixture +def reset_compliance_cache(): + """Reset the module-level cache so each test starts cold.""" + previous = dict(compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS) + compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS.clear() + try: + yield + finally: + compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS.clear() + compliance_module.AVAILABLE_COMPLIANCE_FRAMEWORKS.update(previous) + + +class TestGetComplianceFrameworks: + def test_returns_keys_from_compliance_get_bulk(self, reset_compliance_cache): + with patch("api.compliance.Compliance") as mock_compliance: + mock_compliance.get_bulk.return_value = { + "cis_1.4_aws": MagicMock(), + "mitre_attack_aws": MagicMock(), + } + result = get_compliance_frameworks(Provider.ProviderChoices.AWS) + + assert sorted(result) == ["cis_1.4_aws", "mitre_attack_aws"] + mock_compliance.get_bulk.assert_called_once_with(Provider.ProviderChoices.AWS) + + def test_caches_result_per_provider(self, reset_compliance_cache): + with patch("api.compliance.Compliance") as mock_compliance: + mock_compliance.get_bulk.return_value = {"cis_1.4_aws": MagicMock()} + get_compliance_frameworks(Provider.ProviderChoices.AWS) + get_compliance_frameworks(Provider.ProviderChoices.AWS) + + # Cached after first call. + assert mock_compliance.get_bulk.call_count == 1 + + @pytest.mark.parametrize( + "provider_type", + [choice.value for choice in Provider.ProviderChoices], + ) + def test_listing_is_subset_of_bulk(self, reset_compliance_cache, provider_type): + """Regression for CLOUD-API-40S: every name returned by + ``get_compliance_frameworks`` must be loadable via ``Compliance.get_bulk``. + + A divergence here is what produced ``KeyError: 'csa_ccm_4.0'`` in + ``generate_outputs_task`` after universal/multi-provider compliance + JSONs were introduced at the top-level ``prowler/compliance/`` path. + """ + bulk_keys = set(Compliance.get_bulk(provider_type).keys()) + listed = set(get_compliance_frameworks(provider_type)) + + missing = listed - bulk_keys + assert not missing, ( + f"get_compliance_frameworks({provider_type!r}) returned names not " + f"loadable by Compliance.get_bulk: {sorted(missing)}" + ) diff --git a/api/src/backend/api/tests/test_cypher_sanitizer.py b/api/src/backend/api/tests/test_cypher_sanitizer.py new file mode 100644 index 0000000000..a54afcd8bb --- /dev/null +++ b/api/src/backend/api/tests/test_cypher_sanitizer.py @@ -0,0 +1,429 @@ +"""Unit tests for the Cypher sanitizer (validation + provider-label injection).""" + +from unittest.mock import patch + +import pytest + +from rest_framework.exceptions import ValidationError + +from api.attack_paths.cypher_sanitizer import ( + inject_provider_label, + validate_custom_query, +) + +PROVIDER_ID = "019c41ee-7df3-7dec-a684-d839f95619f8" +LABEL = "_Provider_019c41ee7df37deca684d839f95619f8" + + +def _inject(cypher: str) -> str: + """Shortcut that patches `get_provider_label` to avoid config imports.""" + with patch( + "api.attack_paths.cypher_sanitizer.get_provider_label", return_value=LABEL + ): + return inject_provider_label(cypher, PROVIDER_ID) + + +# --------------------------------------------------------------------------- +# Pass A - Labeled node patterns (all clauses) +# --------------------------------------------------------------------------- + + +class TestLabeledNodes: + def test_single_label(self): + result = _inject("MATCH (n:AWSRole) RETURN n") + assert f"(n:AWSRole:{LABEL})" in result + + def test_label_with_properties(self): + result = _inject("MATCH (n:AWSRole {name: 'admin'}) RETURN n") + assert f"(n:AWSRole:{LABEL} {{name: 'admin'}})" in result + + def test_multiple_labels(self): + result = _inject("MATCH (n:AWSRole:AWSPrincipal) RETURN n") + assert f"(n:AWSRole:AWSPrincipal:{LABEL})" in result + + def test_anonymous_labeled(self): + result = _inject( + "MATCH (:AWSPrincipal {arn: 'ecs-tasks.amazonaws.com'}) RETURN 1" + ) + assert f"(:AWSPrincipal:{LABEL} {{arn: 'ecs-tasks.amazonaws.com'}})" in result + + def test_backtick_label(self): + result = _inject("MATCH (n:`My Label`) RETURN n") + assert f"(n:`My Label`:{LABEL})" in result + + def test_labeled_in_where_clause(self): + """Labeled nodes in WHERE (pattern existence) still get the label.""" + result = _inject( + "MATCH (n:AWSRole) WHERE EXISTS((n)-[:REL]->(:Target)) RETURN n" + ) + assert f"(n:AWSRole:{LABEL})" in result + assert f"(:Target:{LABEL})" in result + + def test_labeled_in_return_clause(self): + """Labeled nodes in RETURN still get the label (they're always node patterns).""" + result = _inject("MATCH (n:AWSRole) RETURN (n:AWSRole)") + assert result.count(f":AWSRole:{LABEL}") == 2 + + def test_labeled_in_optional_match(self): + result = _inject( + "OPTIONAL MATCH (pf:ProwlerFinding {status: 'FAIL'}) RETURN pf" + ) + assert f"(pf:ProwlerFinding:{LABEL} {{status: 'FAIL'}})" in result + + +# --------------------------------------------------------------------------- +# Pass B - Bare node patterns (MATCH/OPTIONAL MATCH only) +# --------------------------------------------------------------------------- + + +class TestBareNodes: + def test_bare_in_match(self): + result = _inject("MATCH (a)-[:HAS_POLICY]->(b) RETURN a, b") + assert f"(a:{LABEL})" in result + assert f"(b:{LABEL})" in result + + def test_bare_with_properties_in_match(self): + result = _inject("MATCH (n {name: 'x'}) RETURN n") + assert f"(n:{LABEL} {{name: 'x'}})" in result + + def test_bare_in_optional_match(self): + result = _inject("OPTIONAL MATCH (n)-[r]-(m) RETURN n") + assert f"(n:{LABEL})" in result + assert f"(m:{LABEL})" in result + + def test_bare_not_injected_in_return(self): + """Bare (identifier) in RETURN could be expression grouping.""" + cypher = "MATCH (n:AWSRole) RETURN (n)" + result = _inject(cypher) + # The labeled (n:AWSRole) gets the label, but the bare (n) in RETURN should not + assert f"(n:AWSRole:{LABEL})" in result + # Count how many times the label appears - should be 1 (from MATCH only) + assert result.count(LABEL) == 1 + + def test_bare_not_injected_in_where(self): + cypher = "MATCH (n:AWSRole) WHERE (n.x > 1) RETURN n" + result = _inject(cypher) + # (n.x > 1) is an expression group, not a node pattern - should be untouched + assert "(n.x > 1)" in result + + def test_bare_not_injected_in_with(self): + cypher = "MATCH (n:AWSRole) WITH (n) RETURN n" + result = _inject(cypher) + assert result.count(LABEL) == 1 + + def test_bare_not_injected_in_unwind(self): + cypher = "UNWIND nodes(path) as n OPTIONAL MATCH (n)-[r]-(m) RETURN n" + result = _inject(cypher) + # (n) and (m) in OPTIONAL MATCH get injected, but nodes(path) in UNWIND does not + assert f"(n:{LABEL})" in result + assert f"(m:{LABEL})" in result + + +# --------------------------------------------------------------------------- +# Function call exclusion +# --------------------------------------------------------------------------- + + +class TestFunctionCallExclusion: + @pytest.mark.parametrize( + "func_call", + [ + "collect(DISTINCT pf)", + "any(x IN stmt.action WHERE toLower(x) = 'iam:*')", + "toLower(action)", + "nodes(path)", + "count(n)", + "apoc.create.vNode(labels)", + "EXISTS(n.prop)", + "size(n.list)", + ], + ) + def test_function_calls_not_injected(self, func_call): + cypher = f"MATCH (n:AWSRole) WHERE {func_call} RETURN n" + result = _inject(cypher) + # The function call should remain unchanged + assert func_call in result + # Only the MATCH labeled node should get the label + assert result.count(LABEL) == 1 + + +# --------------------------------------------------------------------------- +# String and comment protection +# --------------------------------------------------------------------------- + + +class TestProtection: + def test_string_with_fake_node_pattern(self): + cypher = "MATCH (n:AWSRole) WHERE n.name = '(fake:Label)' RETURN n" + result = _inject(cypher) + assert "'(fake:Label)'" in result + assert result.count(LABEL) == 1 + + def test_double_quoted_string(self): + cypher = 'MATCH (n:AWSRole) WHERE n.name = "(fake:Label)" RETURN n' + result = _inject(cypher) + assert '"(fake:Label)"' in result + assert result.count(LABEL) == 1 + + def test_line_comment_with_node_pattern(self): + cypher = "// (n:Fake)\nMATCH (n:AWSRole) RETURN n" + result = _inject(cypher) + assert "// (n:Fake)" in result + assert result.count(LABEL) == 1 + + def test_string_containing_double_slash(self): + """Strings with // inside should be consumed as strings, not comments.""" + cypher = "MATCH (n:AWSRole {url: 'https://example.com'}) RETURN n" + result = _inject(cypher) + assert "'https://example.com'" in result + assert f"(n:AWSRole:{LABEL}" in result + + def test_escaped_quotes_in_string(self): + cypher = r"MATCH (n:AWSRole) WHERE n.name = 'it\'s a test' RETURN n" + result = _inject(cypher) + assert result.count(LABEL) == 1 + + +# --------------------------------------------------------------------------- +# Clause splitting +# --------------------------------------------------------------------------- + + +class TestClauseSplitting: + def test_case_insensitive_keywords(self): + cypher = "match (n:AWSRole) where n.x = 1 return n" + result = _inject(cypher) + assert f"(n:AWSRole:{LABEL})" in result + + def test_optional_match_with_extra_whitespace(self): + cypher = "OPTIONAL MATCH (n:AWSRole) RETURN n" + result = _inject(cypher) + assert f"(n:AWSRole:{LABEL})" in result + + def test_multiple_match_clauses(self): + cypher = ( + "MATCH (a:AWSAccount)--(b:AWSRole) " + "MATCH (b)--(c:AWSPolicy) " + "RETURN a, b, c" + ) + result = _inject(cypher) + assert f"(a:AWSAccount:{LABEL})" in result + assert f"(b:AWSRole:{LABEL})" in result + assert f"(c:AWSPolicy:{LABEL})" in result + # (b) in second MATCH is bare and gets injected + assert result.count(LABEL) == 4 # a, b (labeled), b (bare in 2nd MATCH), c + + +# --------------------------------------------------------------------------- +# Real-world query patterns from aws.py +# --------------------------------------------------------------------------- + + +class TestRealWorldQueries: + def test_basic_resource_query(self): + cypher = ( + "MATCH path = (aws:AWSAccount {id: $provider_uid})--(rds:RDSInstance)\n" + "UNWIND nodes(path) as n\n" + "OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding {status: 'FAIL'})\n" + "RETURN path, collect(DISTINCT pf) as dpf" + ) + result = _inject(cypher) + assert f"(aws:AWSAccount:{LABEL} {{id: $provider_uid}})" in result + assert f"(rds:RDSInstance:{LABEL})" in result + assert f"(n:{LABEL})" in result + assert f"(pf:ProwlerFinding:{LABEL} {{status: 'FAIL'}})" in result + assert "nodes(path)" in result # function call untouched + assert "collect(DISTINCT pf)" in result # function call untouched + + def test_privilege_escalation_query(self): + cypher = ( + "MATCH path_principal = (aws:AWSAccount {id: $uid})" + "--(principal:AWSPrincipal)--(pol:AWSPolicy)\n" + "WHERE pol.effect = 'Allow'\n" + "MATCH (principal)--(cfn_policy:AWSPolicy)" + "--(stmt_cfn:AWSPolicyStatement)\n" + "WHERE any(action IN stmt_cfn.action WHERE toLower(action) = 'iam:passrole')\n" + "MATCH path_target = (aws)--(target_role:AWSRole)" + "-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {arn: 'cloudformation.amazonaws.com'})\n" + "RETURN path_principal, path_target" + ) + result = _inject(cypher) + assert f"(aws:AWSAccount:{LABEL} {{id: $uid}})" in result + assert f"(principal:AWSPrincipal:{LABEL})" in result + assert f"(pol:AWSPolicy:{LABEL})" in result + assert f"(principal:{LABEL})" in result # bare in 2nd MATCH + assert f"(cfn_policy:AWSPolicy:{LABEL})" in result + assert f"(stmt_cfn:AWSPolicyStatement:{LABEL})" in result + assert f"(aws:{LABEL})" in result # bare in 3rd MATCH + assert f"(target_role:AWSRole:{LABEL})" in result + assert ( + f"(:AWSPrincipal:{LABEL} {{arn: 'cloudformation.amazonaws.com'}})" in result + ) + # Function calls in WHERE untouched + assert "any(action IN" in result + assert "toLower(action)" in result + + def test_custom_bare_query(self): + cypher = ( + "MATCH (a)-[:HAS_POLICY]->(b)\n" + "WHERE a.name CONTAINS 'admin'\n" + "RETURN a, b" + ) + result = _inject(cypher) + assert f"(a:{LABEL})" in result + assert f"(b:{LABEL})" in result + assert result.count(LABEL) == 2 + + def test_internet_via_path_connectivity(self): + """Post-refactor pattern: Internet reached via CAN_ACCESS, not standalone.""" + cypher = ( + "MATCH path = (aws:AWSAccount {id: $provider_uid})--(ec2:EC2Instance)\n" + "WHERE ec2.exposed_internet = true\n" + "OPTIONAL MATCH (internet:Internet)-[can_access:CAN_ACCESS]->(ec2)\n" + "RETURN path, internet, can_access" + ) + result = _inject(cypher) + assert f"(aws:AWSAccount:{LABEL}" in result + assert f"(ec2:EC2Instance:{LABEL})" in result + assert f"(internet:Internet:{LABEL})" in result + # ec2 in OPTIONAL MATCH is bare, but already labeled via Pass A won't match it + # because it has no label. It IS bare, so Pass B injects. + assert f"(ec2:{LABEL})" in result + + +# --------------------------------------------------------------------------- +# Edge cases +# --------------------------------------------------------------------------- + + +class TestEdgeCases: + def test_empty_query(self): + assert _inject("") == "" + + def test_no_node_patterns(self): + cypher = "RETURN 1 + 2" + assert _inject(cypher) == cypher + + def test_anonymous_empty_parens_not_injected(self): + """Empty () in MATCH is extremely rare but should not be injected.""" + cypher = "MATCH ()--(m:AWSRole) RETURN m" + result = _inject(cypher) + assert "()" in result # empty parens untouched + assert f"(m:AWSRole:{LABEL})" in result + + def test_fully_anonymous_query_bypasses_injection(self): + """All-anonymous patterns bypass injection entirely. + + MATCH ()--()--() has no labels and no variables, so neither Pass A + (labeled) nor Pass B (bare identifier) can inject the provider label. + This is safe because _serialize_graph() (Layer 3) filters every + returned node by provider label, dropping cross-provider data before + it reaches the user. + """ + cypher = "MATCH ()--()--() RETURN *" + result = _inject(cypher) + assert result == cypher # completely unmodified + assert LABEL not in result + + def test_relationship_patterns_untouched(self): + cypher = "MATCH (a:X)-[r:REL_TYPE {x: 1}]->(b:Y) RETURN a" + result = _inject(cypher) + assert "[r:REL_TYPE {x: 1}]" in result # relationship untouched + assert f"(a:X:{LABEL})" in result + assert f"(b:Y:{LABEL})" in result + + def test_call_subquery(self): + cypher = ( + "CALL {\n" + " MATCH (inner:AWSRole) RETURN inner\n" + "}\n" + "MATCH (outer:AWSAccount) RETURN outer, inner" + ) + result = _inject(cypher) + assert f"(inner:AWSRole:{LABEL})" in result + assert f"(outer:AWSAccount:{LABEL})" in result + + def test_multiple_protected_regions(self): + cypher = ( + "MATCH (n:X {a: 'hello'}) " 'WHERE n.b = "world" ' "// comment\n" "RETURN n" + ) + result = _inject(cypher) + assert "'hello'" in result + assert '"world"' in result + assert "// comment" in result + assert f"(n:X:{LABEL}" in result + + def test_idempotent_on_already_injected(self): + """Running injection twice should add the label twice (not ideal, but predictable).""" + first = _inject("MATCH (n:AWSRole) RETURN n") + second = _inject(first) + # The label appears twice (stacked) + assert second.count(LABEL) == 2 + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +class TestValidation: + @pytest.mark.parametrize( + "cypher", + [ + "LOAD CSV FROM 'http://169.254.169.254/' AS x RETURN x", + "load csv from 'http://evil.com' as row return row", + "CALL apoc.load.json('http://evil.com/') YIELD value RETURN value", + "CALL apoc.load.csvParams('http://evil.com/', {}, null) YIELD list RETURN list", + "CALL apoc.import.csv([{fileName: 'f'}], [], {}) YIELD node RETURN node", + "CALL apoc.export.csv.all('file.csv', {})", + "CALL apoc.cypher.run('CREATE (n)', {}) YIELD value RETURN value", + "CALL apoc.systemdb.graph() YIELD nodes RETURN nodes", + "CALL apoc.config.list() YIELD key, value RETURN key, value", + "CALL apoc.periodic.iterate('MATCH (n) RETURN n', 'DELETE n', {batchSize: 100})", + "CALL apoc.do.when(true, 'CREATE (n) RETURN n', '', {}) YIELD value RETURN value", + "CALL apoc.trigger.add('t', 'RETURN 1', {phase: 'before'})", + "CALL apoc.custom.asProcedure('myProc', 'RETURN 1')", + ], + ids=[ + "LOAD_CSV", + "LOAD_CSV_lowercase", + "apoc.load.json", + "apoc.load.csvParams", + "apoc.import.csv", + "apoc.export.csv", + "apoc.cypher.run", + "apoc.systemdb.graph", + "apoc.config.list", + "apoc.periodic.iterate", + "apoc.do.when", + "apoc.trigger.add", + "apoc.custom.asProcedure", + ], + ) + def test_rejects_blocked_patterns(self, cypher): + with pytest.raises(ValidationError) as exc: + validate_custom_query(cypher) + + assert "blocked operation" in str(exc.value.detail) + + @pytest.mark.parametrize( + "cypher", + [ + "MATCH (n:AWSAccount) RETURN n LIMIT 10", + "MATCH (a)-[r]->(b) RETURN a, r, b", + "MATCH (n) WHERE n.name CONTAINS 'load' RETURN n", + "CALL apoc.create.vNode(['Label'], {}) YIELD node RETURN node", + "MATCH (n) WHERE n.name = 'apoc.load.json' RETURN n", + 'MATCH (n) WHERE n.description = "LOAD CSV is cool" RETURN n', + ], + ids=[ + "simple_match", + "traversal", + "contains_load_substring", + "apoc_virtual_node", + "apoc_load_inside_single_quotes", + "load_csv_inside_double_quotes", + ], + ) + def test_allows_clean_queries(self, cypher): + validate_custom_query(cypher) diff --git a/api/src/backend/api/tests/test_db_utils.py b/api/src/backend/api/tests/test_db_utils.py index f52bb349aa..18935b9a3e 100644 --- a/api/src/backend/api/tests/test_db_utils.py +++ b/api/src/backend/api/tests/test_db_utils.py @@ -6,10 +6,12 @@ import pytest from django.conf import settings from django.db import DEFAULT_DB_ALIAS, OperationalError from freezegun import freeze_time +from psycopg2 import sql as psycopg2_sql from rest_framework_json_api.serializers import ValidationError from api.db_utils import ( POSTGRES_TENANT_VAR, + PostgresEnumMigration, _should_create_index_on_partition, batch_delete, create_objects_in_batches, @@ -910,3 +912,61 @@ class TestRlsTransaction: cursor.execute("SELECT 1") result = cursor.fetchone() assert result[0] == 1 + + +class TestPostgresEnumMigration: + """ + Verify that PostgresEnumMigration builds DDL statements via psycopg2.sql + so that enum type names and values are always properly quoted — preventing + SQL injection through f-string interpolation. + """ + + def _make_mock_schema_editor(self): + mock_cursor = MagicMock() + mock_conn = MagicMock() + mock_conn.cursor.return_value.__enter__ = MagicMock(return_value=mock_cursor) + mock_conn.cursor.return_value.__exit__ = MagicMock(return_value=False) + mock_schema_editor = MagicMock() + mock_schema_editor.connection = mock_conn + return mock_schema_editor, mock_cursor + + def test_create_enum_type_generates_correct_sql(self): + """create_enum_type builds a proper CREATE TYPE … AS ENUM via psycopg2.sql.""" + migration = PostgresEnumMigration("my_enum", ("val_a", "val_b")) + schema_editor, mock_cursor = self._make_mock_schema_editor() + + migration.create_enum_type(apps=None, schema_editor=schema_editor) + + mock_cursor.execute.assert_called_once() + query_arg = mock_cursor.execute.call_args[0][0] + assert isinstance( + query_arg, psycopg2_sql.Composable + ), "create_enum_type must pass a psycopg2.sql.Composable, not a raw string." + # Verify the composed SQL structure: CREATE TYPE AS ENUM () + parts = query_arg.seq + assert parts[0] == psycopg2_sql.SQL("CREATE TYPE ") + assert isinstance(parts[1], psycopg2_sql.Identifier) + assert parts[1].strings == ("my_enum",) + assert parts[2] == psycopg2_sql.SQL(" AS ENUM (") + # The enum values are a Composed of Literal items joined by ", " + enum_literals = [p for p in parts[3].seq if isinstance(p, psycopg2_sql.Literal)] + assert [lit._wrapped for lit in enum_literals] == ["val_a", "val_b"] + assert parts[4] == psycopg2_sql.SQL(")") + + def test_drop_enum_type_generates_correct_sql(self): + """drop_enum_type builds a proper DROP TYPE via psycopg2.sql.""" + migration = PostgresEnumMigration("my_enum", ("val_a",)) + schema_editor, mock_cursor = self._make_mock_schema_editor() + + migration.drop_enum_type(apps=None, schema_editor=schema_editor) + + mock_cursor.execute.assert_called_once() + query_arg = mock_cursor.execute.call_args[0][0] + assert isinstance( + query_arg, psycopg2_sql.Composable + ), "drop_enum_type must pass a psycopg2.sql.Composable, not a raw string." + # Verify the composed SQL structure: DROP TYPE + parts = query_arg.seq + assert parts[0] == psycopg2_sql.SQL("DROP TYPE ") + assert isinstance(parts[1], psycopg2_sql.Identifier) + assert parts[1].strings == ("my_enum",) diff --git a/api/src/backend/api/tests/test_models.py b/api/src/backend/api/tests/test_models.py index 13878794b1..b8b7f61dd1 100644 --- a/api/src/backend/api/tests/test_models.py +++ b/api/src/backend/api/tests/test_models.py @@ -243,6 +243,39 @@ class TestSAMLConfigurationModel: assert "Invalid XML" in errors["metadata_xml"][0] assert "not well-formed" in errors["metadata_xml"][0] + def test_xml_bomb_rejected(self, tenants_fixture): + """ + Regression test: a 'billion laughs' XML bomb in the SAML metadata field + must be rejected and not allowed to exhaust server memory / CPU. + + Before the fix, xml.etree.ElementTree was used directly, which does not + protect against entity-expansion attacks. The fix switches to defusedxml + which raises an exception for any XML containing entity definitions. + """ + tenant = tenants_fixture[0] + xml_bomb = ( + "" + "" + " " + " " + " " + "]>" + "" + ) + config = SAMLConfiguration( + email_domain="xmlbomb.com", + metadata_xml=xml_bomb, + tenant=tenant, + ) + + with pytest.raises(ValidationError) as exc_info: + config._parse_metadata() + + errors = exc_info.value.message_dict + assert "metadata_xml" in errors + def test_metadata_missing_sso_fails(self, tenants_fixture): tenant = tenants_fixture[0] xml = """ diff --git a/api/src/backend/api/tests/test_rbac.py b/api/src/backend/api/tests/test_rbac.py index addd8b4ec6..684a9e44b9 100644 --- a/api/src/backend/api/tests/test_rbac.py +++ b/api/src/backend/api/tests/test_rbac.py @@ -2,7 +2,7 @@ import json from unittest.mock import ANY, Mock, patch import pytest -from conftest import TODAY +from conftest import TEST_PASSWORD, TODAY from django.urls import reverse from rest_framework import status @@ -830,3 +830,66 @@ class TestUserRoleLinkPermissions: ) assert response.status_code == status.HTTP_403_FORBIDDEN + + +@pytest.mark.django_db +class TestCrossTenantRoleLeak: + """Regression tests for get_role() cross-tenant privilege leak. + + get_role() must query admin_db (bypassing RLS) so that a user with a role + in tenant A cannot accidentally pass role checks when authenticated against + tenant B where they have no role. + """ + + def test_user_with_role_in_tenant_a_denied_in_tenant_b(self, tenants_fixture): + """User has admin role in tenant A, membership in tenant B but no role. + Hitting an RBAC-protected endpoint with a tenant-B token must return 403.""" + from rest_framework.test import APIClient + + tenant_a = tenants_fixture[0] + tenant_b = tenants_fixture[1] + + user = User.objects.create_user( + name="cross_tenant_user", + email="cross_tenant@test.com", + password=TEST_PASSWORD, + ) + Membership.objects.create( + user=user, tenant=tenant_a, role=Membership.RoleChoices.OWNER + ) + Membership.objects.create( + user=user, tenant=tenant_b, role=Membership.RoleChoices.OWNER + ) + + # Role only in tenant A + role = Role.objects.create( + name="admin", + tenant_id=tenant_a.id, + manage_users=True, + manage_account=True, + manage_billing=True, + manage_providers=True, + manage_integrations=True, + manage_scans=True, + unlimited_visibility=True, + ) + UserRoleRelationship.objects.create(user=user, role=role, tenant_id=tenant_a.id) + + # Mint token scoped to tenant B (where user has NO role) + serializer = TokenSerializer( + data={ + "type": "tokens", + "email": "cross_tenant@test.com", + "password": TEST_PASSWORD, + "tenant_id": tenant_b.id, + } + ) + serializer.is_valid(raise_exception=True) + access_token = serializer.validated_data["access"] + + client = APIClient() + client.defaults["HTTP_AUTHORIZATION"] = f"Bearer {access_token}" + + # user-list requires manage_users permission via HasPermissions + response = client.get(reverse("user-list")) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/api/src/backend/api/tests/test_sentry.py b/api/src/backend/api/tests/test_sentry.py index cf71593469..fb7abaffb4 100644 --- a/api/src/backend/api/tests/test_sentry.py +++ b/api/src/backend/api/tests/test_sentry.py @@ -4,14 +4,25 @@ from unittest.mock import MagicMock from config.settings.sentry import before_send +def _make_log_record(msg, level=logging.ERROR, name="test", args=None): + """Build a real LogRecord so getMessage() works like in production.""" + record = logging.LogRecord( + name=name, + level=level, + pathname="", + lineno=0, + msg=msg, + args=args, + exc_info=None, + ) + return record + + def test_before_send_ignores_log_with_ignored_exception(): """Test that before_send ignores logs containing ignored exceptions.""" - log_record = MagicMock() - log_record.msg = "Provider kubernetes is not connected" - log_record.levelno = logging.ERROR # 40 + log_record = _make_log_record("Provider kubernetes is not connected") hint = {"log_record": log_record} - event = MagicMock() result = before_send(event, hint) @@ -36,12 +47,9 @@ def test_before_send_ignores_exception_with_ignored_exception(): def test_before_send_passes_through_non_ignored_log(): """Test that before_send passes through logs that don't contain ignored exceptions.""" - log_record = MagicMock() - log_record.msg = "Some other error message" - log_record.levelno = logging.ERROR # 40 + log_record = _make_log_record("Some other error message") hint = {"log_record": log_record} - event = MagicMock() result = before_send(event, hint) @@ -66,15 +74,53 @@ def test_before_send_passes_through_non_ignored_exception(): def test_before_send_handles_warning_level(): """Test that before_send handles warning level logs.""" - log_record = MagicMock() - log_record.msg = "Provider kubernetes is not connected" - log_record.levelno = logging.WARNING # 30 + log_record = _make_log_record( + "Provider kubernetes is not connected", level=logging.WARNING + ) hint = {"log_record": log_record} - event = MagicMock() result = before_send(event, hint) # Assert that the event was dropped (None returned) assert result is None + + +def test_before_send_ignores_neo4j_defunct_connection(): + """Test that before_send drops neo4j.io defunct connection logs. + + The Neo4j driver logs transient connection errors at ERROR level + before RetryableSession retries them. These are noise. + + The driver uses %s formatting, so "defunct" is in the args, not + in the template. This test mirrors the real LogRecord structure. + """ + log_record = _make_log_record( + msg="[#%04X] _: error: %s: %r", + name="neo4j.io", + args=( + 0xE5CC, + "Failed to read from defunct connection " + "IPv4Address(('cloud-neo4j.prowler.com', 7687))", + ConnectionResetError(104, "Connection reset by peer"), + ), + ) + + hint = {"log_record": log_record} + event = MagicMock() + + assert before_send(event, hint) is None + + +def test_before_send_passes_non_defunct_neo4j_log(): + """Test that before_send passes through neo4j.io logs that are not about defunct connections.""" + log_record = _make_log_record( + msg="Some other neo4j transport error", + name="neo4j.io", + ) + + hint = {"log_record": log_record} + event = MagicMock() + + assert before_send(event, hint) == event diff --git a/api/src/backend/api/tests/test_utils.py b/api/src/backend/api/tests/test_utils.py index a3b1bf5736..86da5567da 100644 --- a/api/src/backend/api/tests/test_utils.py +++ b/api/src/backend/api/tests/test_utils.py @@ -33,6 +33,7 @@ from prowler.providers.m365.m365_provider import M365Provider from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider from prowler.providers.openstack.openstack_provider import OpenstackProvider from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider +from prowler.providers.vercel.vercel_provider import VercelProvider class TestMergeDicts: @@ -128,6 +129,7 @@ class TestReturnProwlerProvider: (Provider.ProviderChoices.CLOUDFLARE.value, CloudflareProvider), (Provider.ProviderChoices.OPENSTACK.value, OpenstackProvider), (Provider.ProviderChoices.IMAGE.value, ImageProvider), + (Provider.ProviderChoices.VERCEL.value, VercelProvider), ], ) def test_return_prowler_provider(self, provider_type, expected_provider): @@ -218,6 +220,24 @@ class TestProwlerProviderConnectionTest: registry_token="tok123", ) + @patch("api.utils.return_prowler_provider") + def test_prowler_provider_connection_test_vercel_provider( + self, mock_return_prowler_provider + ): + """Test connection test for Vercel provider passes team_id.""" + provider = MagicMock() + provider.uid = "team_abcdef1234567890" + provider.provider = Provider.ProviderChoices.VERCEL.value + provider.secret.secret = {"api_token": "vercel_token_123"} + mock_return_prowler_provider.return_value = MagicMock() + + prowler_provider_connection_test(provider) + mock_return_prowler_provider.return_value.test_connection.assert_called_once_with( + api_token="vercel_token_123", + team_id="team_abcdef1234567890", + raise_on_exception=False, + ) + @patch("api.utils.return_prowler_provider") def test_prowler_provider_connection_test_image_provider_no_creds( self, mock_return_prowler_provider @@ -284,6 +304,10 @@ class TestGetProwlerProviderKwargs: Provider.ProviderChoices.OPENSTACK.value, {}, ), + ( + Provider.ProviderChoices.VERCEL.value, + {"team_id": "provider_uid"}, + ), ], ) def test_get_prowler_provider_kwargs(self, provider_type, expected_extra_kwargs): @@ -782,11 +806,15 @@ class TestProwlerIntegrationConnectionTest: } integration.configuration = {} - # Mock successful JIRA connection with projects + # Mock successful JIRA connection with projects and issue types mock_connection = MagicMock() mock_connection.is_connected = True mock_connection.error = None mock_connection.projects = {"PROJ1": "Project 1", "PROJ2": "Project 2"} + mock_connection.issue_types = { + "PROJ1": ["Task", "Bug"], + "PROJ2": ["Task", "Story"], + } mock_jira_class.test_connection.return_value = mock_connection # Mock rls_transaction context manager @@ -815,6 +843,12 @@ class TestProwlerIntegrationConnectionTest: "PROJ2": "Project 2", } + # Verify issue types were saved to integration configuration + assert integration.configuration["issue_types"] == { + "PROJ1": ["Task", "Bug"], + "PROJ2": ["Task", "Story"], + } + # Verify integration.save() was called integration.save.assert_called_once() @@ -838,6 +872,7 @@ class TestProwlerIntegrationConnectionTest: mock_connection.is_connected = False mock_connection.error = Exception("Authentication failed: Invalid credentials") mock_connection.projects = {} # Empty projects when connection fails + mock_connection.issue_types = {} # Empty issue types when connection fails mock_jira_class.test_connection.return_value = mock_connection # Mock rls_transaction context manager @@ -863,6 +898,9 @@ class TestProwlerIntegrationConnectionTest: # Verify empty projects dict was saved to integration configuration assert integration.configuration["projects"] == {} + # Verify empty issue types dict was saved to integration configuration + assert integration.configuration["issue_types"] == {} + # Verify integration.save() was called even on connection failure integration.save.assert_called_once() @@ -881,11 +919,11 @@ class TestProwlerIntegrationConnectionTest: "domain": "example.atlassian.net", } integration.configuration = { - "issue_types": ["Task"], # Existing configuration + "issue_types": {"OLD_PROJ": ["Task"]}, # Existing configuration "projects": {"OLD_PROJ": "Old Project"}, # Will be overwritten } - # Mock successful JIRA connection with new projects + # Mock successful JIRA connection with new projects and issue types mock_connection = MagicMock() mock_connection.is_connected = True mock_connection.error = None @@ -893,6 +931,10 @@ class TestProwlerIntegrationConnectionTest: "NEW_PROJ1": "New Project 1", "NEW_PROJ2": "New Project 2", } + mock_connection.issue_types = { + "NEW_PROJ1": ["Task", "Bug"], + "NEW_PROJ2": ["Story"], + } mock_jira_class.test_connection.return_value = mock_connection # Mock rls_transaction context manager @@ -910,8 +952,11 @@ class TestProwlerIntegrationConnectionTest: "NEW_PROJ2": "New Project 2", } - # Verify other configuration fields were preserved - assert integration.configuration["issue_types"] == ["Task"] + # Verify issue types were also updated + assert integration.configuration["issue_types"] == { + "NEW_PROJ1": ["Task", "Bug"], + "NEW_PROJ2": ["Story"], + } # Verify integration.save() was called integration.save.assert_called_once() diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 5aa7ae143d..97796c97ca 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -32,6 +32,11 @@ from django_celery_results.models import TaskResult from rest_framework import status from rest_framework.exceptions import PermissionDenied from rest_framework.response import Response +from rest_framework_simplejwt.token_blacklist.models import ( + BlacklistedToken, + OutstandingToken, +) +from rest_framework_simplejwt.tokens import RefreshToken from api.attack_paths import ( AttackPathsQueryDefinition, @@ -47,6 +52,7 @@ from api.models import ( Finding, Integration, Invitation, + InvitationRoleRelationship, LighthouseProviderConfiguration, LighthouseProviderModels, LighthouseTenantConfiguration, @@ -57,6 +63,7 @@ from api.models import ( ProviderGroupMembership, ProviderSecret, Resource, + ResourceFindingMapping, Role, RoleProviderGroupRelationship, SAMLConfiguration, @@ -516,6 +523,13 @@ class TestTenantViewSet: response.json()["data"]["attributes"]["name"] == valid_tenant_payload["name"] ) + new_tenant_id = response.json()["data"]["id"] + user = authenticated_client.user + assert UserRoleRelationship.objects.filter( + user=user, + tenant_id=new_tenant_id, + role__name="admin", + ).exists() def test_tenants_invalid_create(self, authenticated_client, invalid_tenant_payload): response = authenticated_client.post( @@ -575,22 +589,66 @@ class TestTenantViewSet: Tenant.objects.filter(pk=kwargs.get("tenant_id")).delete() delete_tenant_mock.side_effect = _delete_tenant + # Use tenant2 where the user is OWNER + _, tenant2, _ = tenants_fixture + response = authenticated_client.delete( + reverse("tenant-detail", kwargs={"pk": tenant2.id}) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + assert Membership.objects.filter(tenant_id=tenant2.id).count() == 0 + # User is not deleted because it has another membership (tenant1) + assert User.objects.count() == 1 + + @patch("api.v1.views.delete_tenant_task.apply_async") + def test_tenants_delete_as_member_forbidden( + self, delete_tenant_mock, authenticated_client, tenants_fixture + ): + # tenant1: user is MEMBER, not OWNER -> should be forbidden tenant1, *_ = tenants_fixture response = authenticated_client.delete( reverse("tenant-detail", kwargs={"pk": tenant1.id}) ) + assert response.status_code == status.HTTP_403_FORBIDDEN + delete_tenant_mock.assert_not_called() + + @patch("api.v1.views.delete_tenant_task.apply_async") + def test_tenants_delete_cross_tenant( + self, delete_tenant_mock, authenticated_client, tenants_fixture + ): + # tenant3: user has no membership -> should be 404 + _, _, tenant3 = tenants_fixture + response = authenticated_client.delete( + reverse("tenant-detail", kwargs={"pk": tenant3.id}) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + delete_tenant_mock.assert_not_called() + + @patch("api.v1.views.delete_tenant_task.apply_async") + def test_tenants_delete_only_removes_exclusive_users( + self, delete_tenant_mock, authenticated_client, tenants_fixture, extra_users + ): + def _delete_tenant(kwargs): + Tenant.objects.filter(pk=kwargs.get("tenant_id")).delete() + + delete_tenant_mock.side_effect = _delete_tenant + _, tenant2, _ = tenants_fixture + # extra_users adds user2 (OWNER in tenant2) and user3 (MEMBER in tenant2) + # user2 and user3 are ONLY in tenant2, so they should be deleted + # The test user is in tenant1 + tenant2, so should NOT be deleted + initial_user_count = User.objects.count() # test_user + user2 + user3 = 3 + assert initial_user_count == 3 + + response = authenticated_client.delete( + reverse("tenant-detail", kwargs={"pk": tenant2.id}) + ) assert response.status_code == status.HTTP_204_NO_CONTENT - assert Tenant.objects.count() == len(tenants_fixture) - 1 - assert Membership.objects.filter(tenant_id=tenant1.id).count() == 0 - # User is not deleted because it has another membership + # user2 and user3 are deleted (no other memberships), test_user remains assert User.objects.count() == 1 def test_tenants_delete_invalid(self, authenticated_client): response = authenticated_client.delete( reverse("tenant-detail", kwargs={"pk": "random_id"}) ) - # To change if we implement RBAC - # (user might not have permissions to see if the tenant exists or not -> 200 empty) assert response.status_code == status.HTTP_404_NOT_FOUND def test_tenants_list_filter_search(self, authenticated_client, tenants_fixture): @@ -694,7 +752,39 @@ class TestTenantViewSet: # Test user + 2 extra users for tenant 2 assert len(response.json()["data"]) == 3 - @patch("api.v1.views.TenantMembersViewSet.required_permissions", []) + def test_tenants_list_memberships_filter_by_user( + self, authenticated_client, tenants_fixture, extra_users + ): + _, tenant2, _ = tenants_fixture + _, user3_membership = extra_users + user3, membership3 = user3_membership + + response = authenticated_client.get( + reverse("tenant-membership-list", kwargs={"tenant_pk": tenant2.id}), + {"filter[user]": str(user3.id)}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["id"] == str(membership3.id) + + def test_tenants_list_memberships_filter_by_user_no_match( + self, authenticated_client, tenants_fixture, extra_users + ): + _, tenant2, _ = tenants_fixture + unrelated_user = User.objects.create_user( + name="unrelated", + password=TEST_PASSWORD, + email="unrelated@gmail.com", + ) + + response = authenticated_client.get( + reverse("tenant-membership-list", kwargs={"tenant_pk": tenant2.id}), + {"filter[user]": str(unrelated_user.id)}, + ) + assert response.status_code == status.HTTP_200_OK + assert response.json()["data"] == [] + def test_tenants_list_memberships_as_member( self, authenticated_client, tenants_fixture, extra_users ): @@ -752,6 +842,7 @@ class TestTenantViewSet: ): _, tenant2, _ = tenants_fixture user_membership = Membership.objects.get(tenant=tenant2, user__email=TEST_USER) + user_id = user_membership.user_id response = authenticated_client.delete( reverse( "tenant-membership-detail", @@ -760,6 +851,127 @@ class TestTenantViewSet: ) assert response.status_code == status.HTTP_403_FORBIDDEN assert Membership.objects.filter(id=user_membership.id).exists() + assert User.objects.filter(id=user_id).exists() + + def test_expel_user_deletes_account_if_last_membership( + self, authenticated_client, tenants_fixture, extra_users + ): + # TEST_USER is OWNER of tenant2; user3 is MEMBER only in tenant2 + _, tenant2, _ = tenants_fixture + _, user3_membership = extra_users + user3, membership3 = user3_membership + + assert Membership.objects.filter(user=user3).count() == 1 + + response = authenticated_client.delete( + reverse( + "tenant-membership-detail", + kwargs={"tenant_pk": tenant2.id, "pk": membership3.id}, + ) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + assert not Membership.objects.filter(id=membership3.id).exists() + assert not User.objects.filter(id=user3.id).exists() + + def test_expel_user_blacklists_refresh_tokens( + self, authenticated_client, tenants_fixture, extra_users + ): + _, tenant2, _ = tenants_fixture + _, user3_membership = extra_users + user3, membership3 = user3_membership + + # Issue two refresh tokens to simulate active sessions + RefreshToken.for_user(user3) + RefreshToken.for_user(user3) + outstanding_ids = list( + OutstandingToken.objects.filter(user=user3).values_list("id", flat=True) + ) + assert len(outstanding_ids) == 2 + assert not BlacklistedToken.objects.filter( + token_id__in=outstanding_ids + ).exists() + + response = authenticated_client.delete( + reverse( + "tenant-membership-detail", + kwargs={"tenant_pk": tenant2.id, "pk": membership3.id}, + ) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + assert ( + BlacklistedToken.objects.filter(token_id__in=outstanding_ids).count() == 2 + ) + + def test_expel_user_blacklists_refresh_tokens_is_idempotent( + self, authenticated_client, tenants_fixture, extra_users + ): + # Regression test for the bulk blacklisting path: if one of the + # user's refresh tokens is already blacklisted when the expel + # endpoint runs, the remaining tokens must still be blacklisted + # and the already-blacklisted one must not be duplicated. + tenant1, tenant2, _ = tenants_fixture + _, user3_membership = extra_users + user3, membership3 = user3_membership + + # Keep the user alive after the expel so the assertions below can + # still query OutstandingToken by user_id. + Membership.objects.create( + user=user3, + tenant=tenant1, + role=Membership.RoleChoices.MEMBER, + ) + + RefreshToken.for_user(user3) + RefreshToken.for_user(user3) + outstanding_ids = list( + OutstandingToken.objects.filter(user=user3).values_list("id", flat=True) + ) + assert len(outstanding_ids) == 2 + + # Pre-blacklist one of the two tokens to simulate a prior revocation. + BlacklistedToken.objects.create(token_id=outstanding_ids[0]) + assert ( + BlacklistedToken.objects.filter(token_id__in=outstanding_ids).count() == 1 + ) + + response = authenticated_client.delete( + reverse( + "tenant-membership-detail", + kwargs={"tenant_pk": tenant2.id, "pk": membership3.id}, + ) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + + blacklisted = BlacklistedToken.objects.filter(token_id__in=outstanding_ids) + assert blacklisted.count() == 2 + assert set(blacklisted.values_list("token_id", flat=True)) == set( + outstanding_ids + ) + + def test_expel_user_keeps_account_if_has_other_memberships( + self, authenticated_client, tenants_fixture, extra_users + ): + tenant1, tenant2, _ = tenants_fixture + _, user3_membership = extra_users + user3, membership3 = user3_membership + + # Give user3 an additional membership in tenant1 so they are not orphaned + other_membership = Membership.objects.create( + user=user3, + tenant=tenant1, + role=Membership.RoleChoices.MEMBER, + ) + + response = authenticated_client.delete( + reverse( + "tenant-membership-detail", + kwargs={"tenant_pk": tenant2.id, "pk": membership3.id}, + ) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + assert not Membership.objects.filter(id=membership3.id).exists() + assert User.objects.filter(id=user3.id).exists() + assert Membership.objects.filter(id=other_membership.id).exists() def test_tenants_delete_another_membership_as_owner( self, authenticated_client, tenants_fixture, extra_users @@ -807,6 +1019,209 @@ class TestTenantViewSet: ) assert response.status_code == status.HTTP_404_NOT_FOUND + def test_tenants_delete_membership_cross_tenant( + self, authenticated_client, tenants_fixture + ): + # Create a tenant with a different user's membership + other_tenant = Tenant.objects.create(name="Other Tenant") + other_user = User.objects.create_user( + name="other", password=TEST_PASSWORD, email="other@test.com" + ) + other_membership = Membership.objects.create( + user=other_user, + tenant=other_tenant, + role=Membership.RoleChoices.OWNER, + ) + + # Authenticated user is NOT a member of other_tenant -> 404 + response = authenticated_client.delete( + reverse( + "tenant-membership-detail", + kwargs={"tenant_pk": other_tenant.id, "pk": other_membership.id}, + ) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + assert Membership.objects.filter(id=other_membership.id).exists() + + def test_delete_membership_cleans_up_orphaned_role_grants( + self, authenticated_client, tenants_fixture + ): + """Test that deleting a membership removes UserRoleRelationship records + for that tenant while preserving grants in other tenants.""" + tenant1, tenant2, _ = tenants_fixture + + # Create a user with memberships in both tenants + user = User.objects.create_user( + name="Multi-tenant User", + password=TEST_PASSWORD, + email="multitenant@test.com", + ) + + # Create memberships in both tenants + Membership.objects.create( + user=user, tenant=tenant1, role=Membership.RoleChoices.MEMBER + ) + membership2 = Membership.objects.create( + user=user, tenant=tenant2, role=Membership.RoleChoices.MEMBER + ) + + # Create roles in both tenants + role1 = Role.objects.create( + name="Test Role 1", tenant=tenant1, manage_providers=True + ) + role2 = Role.objects.create( + name="Test Role 2", tenant=tenant2, manage_scans=True + ) + + # Create user role relationships for both tenants + UserRoleRelationship.objects.create(user=user, role=role1, tenant=tenant1) + UserRoleRelationship.objects.create(user=user, role=role2, tenant=tenant2) + + # Verify initial state + assert UserRoleRelationship.objects.filter(user=user, tenant=tenant1).exists() + assert UserRoleRelationship.objects.filter(user=user, tenant=tenant2).exists() + assert Role.objects.filter(id=role1.id).exists() + assert Role.objects.filter(id=role2.id).exists() + + # Delete membership from tenant2 (authenticated user is owner of tenant2) + response = authenticated_client.delete( + reverse( + "tenant-membership-detail", + kwargs={"tenant_pk": tenant2.id, "pk": membership2.id}, + ) + ) + + assert response.status_code == status.HTTP_204_NO_CONTENT + + # Verify the membership was deleted + assert not Membership.objects.filter(id=membership2.id).exists() + + # Verify UserRoleRelationship for tenant2 was deleted + assert not UserRoleRelationship.objects.filter( + user=user, tenant=tenant2 + ).exists() + + # Verify UserRoleRelationship for tenant1 is preserved + assert UserRoleRelationship.objects.filter(user=user, tenant=tenant1).exists() + + # Verify orphaned role2 was deleted (no more user or invitation relationships) + assert not Role.objects.filter(id=role2.id).exists() + + # Verify role1 is preserved (still has user relationship) + assert Role.objects.filter(id=role1.id).exists() + + # Verify the user still exists (has other memberships) + assert User.objects.filter(id=user.id).exists() + + def test_delete_membership_preserves_role_with_invitation_relationship( + self, authenticated_client, tenants_fixture + ): + """Test that roles are not deleted if they have invitation relationships.""" + _, tenant2, _ = tenants_fixture + + # Create a user with membership + user = User.objects.create_user( + name="Test User", password=TEST_PASSWORD, email="testuser@test.com" + ) + membership = Membership.objects.create( + user=user, tenant=tenant2, role=Membership.RoleChoices.MEMBER + ) + + # Create a role and user relationship + role = Role.objects.create( + name="Shared Role", tenant=tenant2, manage_providers=True + ) + UserRoleRelationship.objects.create(user=user, role=role, tenant=tenant2) + + # Create an invitation with the same role + invitation = Invitation.objects.create(email="pending@test.com", tenant=tenant2) + InvitationRoleRelationship.objects.create( + invitation=invitation, role=role, tenant=tenant2 + ) + + # Verify initial state + assert UserRoleRelationship.objects.filter(user=user, role=role).exists() + assert InvitationRoleRelationship.objects.filter( + invitation=invitation, role=role + ).exists() + assert Role.objects.filter(id=role.id).exists() + + # Delete the membership + response = authenticated_client.delete( + reverse( + "tenant-membership-detail", + kwargs={"tenant_pk": tenant2.id, "pk": membership.id}, + ) + ) + + assert response.status_code == status.HTTP_204_NO_CONTENT + + # Verify UserRoleRelationship was deleted + assert not UserRoleRelationship.objects.filter(user=user, role=role).exists() + + # Verify role is preserved because invitation relationship exists + assert Role.objects.filter(id=role.id).exists() + assert InvitationRoleRelationship.objects.filter( + invitation=invitation, role=role + ).exists() + + def test_tenants_list_no_permissions( + self, authenticated_client_no_permissions_rbac, tenants_fixture + ): + response = authenticated_client_no_permissions_rbac.get(reverse("tenant-list")) + assert response.status_code == status.HTTP_200_OK + + def test_tenants_retrieve_no_permissions( + self, authenticated_client_no_permissions_rbac, tenants_fixture + ): + tenant1, *_ = tenants_fixture + response = authenticated_client_no_permissions_rbac.get( + reverse("tenant-detail", kwargs={"pk": tenant1.id}) + ) + assert response.status_code == status.HTTP_200_OK + + def test_tenants_create_no_permissions( + self, authenticated_client_no_permissions_rbac, valid_tenant_payload + ): + response = authenticated_client_no_permissions_rbac.post( + reverse("tenant-list"), + data=valid_tenant_payload, + format="json", + ) + assert response.status_code == status.HTTP_201_CREATED + + def test_tenants_partial_update_no_permissions( + self, authenticated_client_no_permissions_rbac, tenants_fixture + ): + tenant1, *_ = tenants_fixture + payload = { + "data": { + "type": "tenants", + "id": str(tenant1.id), + "attributes": {"name": "Unauthorized update"}, + }, + } + response = authenticated_client_no_permissions_rbac.patch( + reverse("tenant-detail", kwargs={"pk": tenant1.id}), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + + @patch("api.v1.views.delete_tenant_task.apply_async") + def test_tenants_delete_no_permissions( + self, + delete_tenant_mock, + authenticated_client_no_permissions_rbac, + tenants_fixture, + ): + tenant1, *_ = tenants_fixture + response = authenticated_client_no_permissions_rbac.delete( + reverse("tenant-detail", kwargs={"pk": tenant1.id}) + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + delete_tenant_mock.assert_not_called() + @pytest.mark.django_db class TestMembershipViewSet: @@ -1658,6 +2073,46 @@ class TestProviderViewSet: "min_length", "uid", ), + # Vercel UID validation - missing team_ prefix + ( + { + "provider": "vercel", + "uid": "abcdef1234567890abcdef12", + "alias": "test", + }, + "vercel-uid", + "uid", + ), + # Vercel UID validation - too short after prefix + ( + { + "provider": "vercel", + "uid": "team_abc123", + "alias": "test", + }, + "vercel-uid", + "uid", + ), + # Vercel UID validation - contains special characters + ( + { + "provider": "vercel", + "uid": "team_abcdef-1234567890ab", + "alias": "test", + }, + "vercel-uid", + "uid", + ), + # Vercel UID validation - too long (33 chars after prefix) + ( + { + "provider": "vercel", + "uid": "team_abcdefghijklmnopqrstuvwxyz1234567", + "alias": "test", + }, + "vercel-uid", + "uid", + ), # Google Workspace UID validation - missing 'C' prefix ( { @@ -1861,21 +2316,21 @@ class TestProviderViewSet: ( "uid.icontains", "1", - 11, + 12, ), ("alias", "aws_testing_1", 1), ("alias.icontains", "aws", 2), - ("inserted_at", TODAY, 12), + ("inserted_at", TODAY, 13), ( "inserted_at.gte", "2024-01-01", - 12, + 13, ), ("inserted_at.lte", "2024-01-01", 0), ( "updated_at.gte", "2024-01-01", - 12, + 13, ), ("updated_at.lte", "2024-01-01", 0), ] @@ -2500,6 +2955,14 @@ class TestProviderSecretViewSet: "delegated_user": "admin@example.com", }, ), + # Vercel with API Token + ( + Provider.ProviderChoices.VERCEL.value, + ProviderSecret.TypeChoices.STATIC, + { + "api_token": "fake-vercel-api-token-for-testing", + }, + ), ], ) def test_provider_secrets_create_valid( @@ -3178,6 +3641,29 @@ class TestScanViewSet: assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == 3 + def test_scan_filter_by_id_exact(self, authenticated_client, scans_fixture): + scan1, *_ = scans_fixture + response = authenticated_client.get( + reverse("scan-list"), + {"filter[id]": str(scan1.id)}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["id"] == str(scan1.id) + + def test_scan_filter_by_id_in(self, authenticated_client, scans_fixture): + scan1, scan2, *_ = scans_fixture + response = authenticated_client.get( + reverse("scan-list"), + {"filter[id.in]": f"{scan1.id},{scan2.id}"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + returned_ids = {item["id"] for item in data} + assert returned_ids == {str(scan1.id), str(scan2.id)} + def test_scans_filter_state_failed(self, authenticated_client, scans_fixture): """Ensure state filter matches only FAILED scans.""" scan1, *_ = scans_fixture @@ -3355,9 +3841,14 @@ class TestScanViewSet: "prowler-output-123_threatscore_report.pdf", ) + presigned_url = ( + "https://test-bucket.s3.amazonaws.com/" + "tenant-id/scan-id/threatscore/prowler-output-123_threatscore_report.pdf" + "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=300" + ) mock_s3_client = Mock() mock_s3_client.list_objects_v2.return_value = {"Contents": [{"Key": pdf_key}]} - mock_s3_client.get_object.return_value = {"Body": io.BytesIO(b"pdf-bytes")} + mock_s3_client.generate_presigned_url.return_value = presigned_url mock_env_str.return_value = bucket mock_get_s3_client.return_value = mock_s3_client @@ -3366,19 +3857,26 @@ class TestScanViewSet: url = reverse("scan-threatscore", kwargs={"pk": scan.id}) response = authenticated_client.get(url) - assert response.status_code == status.HTTP_200_OK - assert response["Content-Type"] == "application/pdf" - assert response["Content-Disposition"].endswith( - '"prowler-output-123_threatscore_report.pdf"' - ) - assert response.content == b"pdf-bytes" + assert response.status_code == status.HTTP_302_FOUND + assert response["Location"] == presigned_url mock_s3_client.list_objects_v2.assert_called_once() - mock_s3_client.get_object.assert_called_once_with(Bucket=bucket, Key=pdf_key) + mock_s3_client.generate_presigned_url.assert_called_once_with( + "get_object", + Params={ + "Bucket": bucket, + "Key": pdf_key, + "ResponseContentDisposition": ( + 'attachment; filename="prowler-output-123_threatscore_report.pdf"' + ), + "ResponseContentType": "application/pdf", + }, + ExpiresIn=300, + ) def test_report_s3_success(self, authenticated_client, scans_fixture, monkeypatch): """ - When output_location is an S3 URL and the S3 client returns the file successfully, - the view should return the ZIP file with HTTP 200 and proper headers. + When output_location is an S3 URL and the object exists, + the view should return a 302 redirect to a presigned S3 URL. """ scan = scans_fixture[0] bucket = "test-bucket" @@ -3392,22 +3890,33 @@ class TestScanViewSet: type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(), ) + presigned_url = ( + "https://test-bucket.s3.amazonaws.com/report.zip" + "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=300" + ) + class FakeS3Client: - def get_object(self, Bucket, Key): + def head_object(self, Bucket, Key): assert Bucket == bucket assert Key == key - return {"Body": io.BytesIO(b"s3 zip content")} + return {} + + def generate_presigned_url(self, ClientMethod, Params, ExpiresIn): + assert ClientMethod == "get_object" + assert Params["Bucket"] == bucket + assert Params["Key"] == key + assert Params["ResponseContentDisposition"] == ( + 'attachment; filename="report.zip"' + ) + assert ExpiresIn == 300 + return presigned_url monkeypatch.setattr("api.v1.views.get_s3_client", lambda: FakeS3Client()) url = reverse("scan-report", kwargs={"pk": scan.id}) response = authenticated_client.get(url) - assert response.status_code == 200 - expected_filename = os.path.basename("report.zip") - content_disposition = response.get("Content-Disposition") - assert content_disposition.startswith('attachment; filename="') - assert f'filename="{expected_filename}"' in content_disposition - assert response.content == b"s3 zip content" + assert response.status_code == status.HTTP_302_FOUND + assert response["Location"] == presigned_url def test_report_s3_success_no_local_files( self, authenticated_client, scans_fixture, monkeypatch @@ -3546,23 +4055,31 @@ class TestScanViewSet: ) match_key = "path/compliance/mitre_attack_aws.csv" + presigned_url = ( + "https://test-bucket.s3.amazonaws.com/path/compliance/mitre_attack_aws.csv" + "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=300" + ) class FakeS3Client: def list_objects_v2(self, Bucket, Prefix): return {"Contents": [{"Key": match_key}]} - def get_object(self, Bucket, Key): - return {"Body": io.BytesIO(b"ignored")} + def generate_presigned_url(self, ClientMethod, Params, ExpiresIn): + assert ClientMethod == "get_object" + assert Params["Key"] == match_key + assert Params["ResponseContentDisposition"] == ( + 'attachment; filename="mitre_attack_aws.csv"' + ) + assert ExpiresIn == 300 + return presigned_url monkeypatch.setattr("api.v1.views.get_s3_client", lambda: FakeS3Client()) framework = match_key.split("/")[-1].split(".")[0] url = reverse("scan-compliance", kwargs={"pk": scan.id, "name": framework}) resp = authenticated_client.get(url) - assert resp.status_code == status.HTTP_200_OK - cd = resp["Content-Disposition"] - assert cd.startswith('attachment; filename="') - assert cd.endswith('filename="mitre_attack_aws.csv"') + assert resp.status_code == status.HTTP_302_FOUND + assert resp["Location"] == presigned_url def test_compliance_s3_not_found( self, authenticated_client, scans_fixture, monkeypatch @@ -3627,6 +4144,51 @@ class TestScanViewSet: assert cd.startswith('attachment; filename="') assert cd.endswith(f'filename="{fname.name}"') + def test_cis_no_output(self, authenticated_client, scans_fixture): + """CIS PDF endpoint must 404 when the scan has no output_location.""" + scan = scans_fixture[0] + scan.state = StateChoices.COMPLETED + scan.output_location = "" + scan.save() + + url = reverse("scan-cis", kwargs={"pk": scan.id}) + resp = authenticated_client.get(url) + assert resp.status_code == status.HTTP_404_NOT_FOUND + assert ( + resp.json()["errors"]["detail"] + == "The scan has no reports, or the CIS report generation task has not started yet." + ) + + def test_cis_local_file(self, authenticated_client, scans_fixture, monkeypatch): + """CIS PDF endpoint must serve the latest generated PDF.""" + scan = scans_fixture[0] + scan.state = StateChoices.COMPLETED + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + base = tmp_path / "reports" + cis_dir = base / "cis" + cis_dir.mkdir(parents=True, exist_ok=True) + fname = cis_dir / "prowler-output-aws-20260101000000_cis_report.pdf" + fname.write_bytes(b"%PDF-1.4 fake pdf") + + scan.output_location = str(base / "scan.zip") + scan.save() + + monkeypatch.setattr( + glob, + "glob", + lambda p: [str(fname)] if p.endswith("*_cis_report.pdf") else [], + ) + + url = reverse("scan-cis", kwargs={"pk": scan.id}) + resp = authenticated_client.get(url) + assert resp.status_code == status.HTTP_200_OK + assert resp["Content-Type"] == "application/pdf" + cd = resp["Content-Disposition"] + assert cd.startswith('attachment; filename="') + assert cd.endswith(f'filename="{fname.name}"') + @patch("api.v1.views.Task.objects.get") @patch("api.v1.views.TaskSerializer") def test__get_task_status_returns_none_if_task_not_executing( @@ -3720,8 +4282,8 @@ class TestScanViewSet: scan.save() fake_client = MagicMock() - fake_client.get_object.side_effect = ClientError( - {"Error": {"Code": "NoSuchKey"}}, "GetObject" + fake_client.head_object.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey"}}, "HeadObject" ) mock_get_s3_client.return_value = fake_client @@ -3744,8 +4306,8 @@ class TestScanViewSet: scan.save() fake_client = MagicMock() - fake_client.get_object.side_effect = ClientError( - {"Error": {"Code": "AccessDenied"}}, "GetObject" + fake_client.head_object.side_effect = ClientError( + {"Error": {"Code": "AccessDenied"}}, "HeadObject" ) mock_get_s3_client.return_value = fake_client @@ -4085,7 +4647,6 @@ class TestAttackPathsScanViewSet: "api.v1.views.attack_paths_views_helpers.execute_query", return_value=graph_payload, ) as mock_execute, - patch("api.v1.views.graph_database.clear_cache") as mock_clear_cache, ): response = authenticated_client.post( reverse( @@ -4112,7 +4673,6 @@ class TestAttackPathsScanViewSet: prepared_parameters, provider_id, ) - mock_clear_cache.assert_called_once_with(expected_db_name) result = response.json()["data"] attributes = result["attributes"] assert attributes["nodes"] == graph_payload["nodes"] @@ -4167,7 +4727,6 @@ class TestAttackPathsScanViewSet: "api.v1.views.attack_paths_views_helpers.execute_query", return_value=graph_payload, ), - patch("api.v1.views.graph_database.clear_cache"), ): response = authenticated_client.post( reverse( @@ -4251,7 +4810,6 @@ class TestAttackPathsScanViewSet: "truncated": False, }, ), - patch("api.v1.views.graph_database.clear_cache"), patch( "api.v1.views.graph_database.get_database_name", return_value="db-test" ), @@ -4306,7 +4864,6 @@ class TestAttackPathsScanViewSet: "truncated": False, }, ), - patch("api.v1.views.graph_database.clear_cache"), patch( "api.v1.views.graph_database.get_database_name", return_value="db-test" ), @@ -4386,7 +4943,6 @@ class TestAttackPathsScanViewSet: "truncated": False, }, ), - patch("api.v1.views.graph_database.clear_cache"), ): response = authenticated_client.post( reverse( @@ -4452,7 +5008,6 @@ class TestAttackPathsScanViewSet: "api.v1.views.graph_database.get_database_name", return_value="db-test", ), - patch("api.v1.views.graph_database.clear_cache"), ): response = authenticated_client.post( reverse( @@ -4509,7 +5064,6 @@ class TestAttackPathsScanViewSet: "api.v1.views.graph_database.get_database_name", return_value="db-test", ), - patch("api.v1.views.graph_database.clear_cache"), ): response = authenticated_client.post( reverse( @@ -4556,7 +5110,6 @@ class TestAttackPathsScanViewSet: "api.v1.views.graph_database.get_database_name", return_value="db-test", ), - patch("api.v1.views.graph_database.clear_cache"), ): response = authenticated_client.post( reverse( @@ -4907,9 +5460,6 @@ class TestAttackPathsScanViewSet: "api.v1.views.graph_database.get_database_name", return_value="db-test", ), - patch( - "api.v1.views.graph_database.clear_cache", - ), ): for i in range(11): response = authenticated_client.post( @@ -7858,8 +8408,12 @@ class TestUserRoleRelationshipViewSet: assert response.status_code == status.HTTP_204_NO_CONTENT relationships = UserRoleRelationship.objects.filter(user=create_test_user.id) assert relationships.count() == 4 - for relationship in relationships[2:]: # Skip admin role - assert relationship.role.id in [r.id for r in roles_fixture[:2]] + # Use set membership instead of positional slicing — QuerySet ordering is + # non-deterministic without an explicit order_by, which makes slice-based + # checks intermittently fail. + added_role_ids = {r.id for r in roles_fixture[:2]} + relationship_role_ids = {rel.role.id for rel in relationships} + assert added_role_ids.issubset(relationship_role_ids) def test_create_relationship_already_exists( self, authenticated_client, roles_fixture, create_test_user @@ -8036,6 +8590,8 @@ class TestUserRoleRelationshipViewSet: manage_scans=False, unlimited_visibility=False, ) + # Assign the role to the user + UserRoleRelationship.objects.create(user=user, role=only_role, tenant=tenant) # Switch token to this tenant serializer = TokenSerializer( @@ -14685,10 +15241,16 @@ class TestMuteRuleViewSet: assert len(data) == 2 assert data[0]["id"] == str(mute_rules_fixture[first_index].id) - @patch("tasks.tasks.mute_historical_findings_task.apply_async") + @patch("api.v1.views.chain") + @patch("api.v1.views.reaggregate_all_finding_group_summaries_task.si") + @patch("api.v1.views.mute_historical_findings_task.si") + @patch("api.v1.views.transaction.on_commit", side_effect=lambda fn: fn()) def test_mute_rules_create_valid( self, - mock_task, + _mock_on_commit, + mock_mute_signature, + mock_reaggregate_signature, + mock_chain, authenticated_client, findings_fixture, create_test_user, @@ -14726,8 +15288,14 @@ class TestMuteRuleViewSet: assert finding.muted_at is not None assert finding.muted_reason == "Security exception approved" - # Verify background task was called - mock_task.assert_called_once() + # Verify background task chain was called: mute → reaggregate all + mock_mute_signature.assert_called_once() + mock_reaggregate_signature.assert_called_once() + mock_chain.assert_called_once_with( + mock_mute_signature.return_value, + mock_reaggregate_signature.return_value, + ) + mock_chain.return_value.apply_async.assert_called_once() @patch("tasks.tasks.mute_historical_findings_task.apply_async") def test_mute_rules_create_converts_finding_ids_to_uids( @@ -15200,6 +15768,29 @@ class TestFindingGroupViewSet: # ec2_instance_public_ip has 1 PASS and 1 FAIL, should aggregate to FAIL assert data[0]["attributes"]["status"] == "FAIL" + def test_finding_groups_region_filter_reaggregates_metrics( + self, authenticated_client, finding_groups_fixture + ): + """Test finding-level filters recompute group metrics from matching findings.""" + response = authenticated_client.get( + reverse("finding-group-list"), + { + "filter[inserted_at]": TODAY, + "filter[check_id]": "ec2_instance_public_ip", + "filter[region]": "us-east-1", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + + attrs = data[0]["attributes"] + assert attrs["status"] == "PASS" + assert attrs["pass_count"] == 1 + assert attrs["fail_count"] == 0 + assert attrs["resources_total"] == 1 + assert attrs["resources_fail"] == 0 + def test_finding_groups_status_pass_when_no_fail( self, authenticated_client, finding_groups_fixture ): @@ -15214,10 +15805,16 @@ class TestFindingGroupViewSet: # iam_password_policy has only PASS findings assert data[0]["attributes"]["status"] == "PASS" - def test_finding_groups_status_muted_all( + def test_finding_groups_fully_muted_group_is_pass( self, authenticated_client, finding_groups_fixture ): - """Test that MUTED status returned when all findings are muted.""" + """A fully-muted group reports status=PASS and muted=True. + + rds_encryption has 2 muted FAIL findings. Muted findings are treated + as resolved/accepted, so the group is no longer actionable and its + status must be PASS. The `muted` flag is True because every finding + in the group is muted. + """ response = authenticated_client.get( reverse("finding-group-list"), {"filter[inserted_at]": TODAY, "filter[check_id]": "rds_encryption"}, @@ -15225,8 +15822,274 @@ class TestFindingGroupViewSet: assert response.status_code == status.HTTP_200_OK data = response.json()["data"] assert len(data) == 1 - # rds_encryption has all muted findings - assert data[0]["attributes"]["status"] == "MUTED" + attrs = data[0]["attributes"] + assert attrs["status"] == "PASS" + assert attrs["muted"] is True + assert attrs["fail_count"] == 0 + assert attrs["fail_muted_count"] == 2 + assert attrs["pass_muted_count"] == 0 + assert attrs["manual_muted_count"] == 0 + assert attrs["muted_count"] == 2 + # Sanity: the per-status muted counts must add up to muted_count. + assert ( + attrs["pass_muted_count"] + + attrs["fail_muted_count"] + + attrs["manual_muted_count"] + == attrs["muted_count"] + ) + + def test_finding_groups_status_ignores_muted_failures( + self, + authenticated_client, + tenants_fixture, + scans_fixture, + resources_fixture, + ): + """Muted FAIL findings must not drive the aggregated status. + + When a group mixes one non-muted PASS with one muted FAIL, the + actionable outcome is PASS: there are no unmuted failures left. The + aggregated `status` must reflect that (not FAIL), while `muted` + stays False because the group still has a non-muted finding. + """ + tenant = tenants_fixture[0] + scan1, *_ = scans_fixture + resource1, *_ = resources_fixture + + pass_finding = Finding.objects.create( + tenant_id=tenant.id, + uid="fg_mixed_muted_pass", + scan=scan1, + delta=None, + status=Status.PASS, + severity=Severity.low, + impact=Severity.low, + check_id="mixed_muted_check", + check_metadata={ + "CheckId": "mixed_muted_check", + "checktitle": "Mixed muted check", + "Description": "Fixture for muted status aggregation.", + }, + first_seen_at="2024-01-11T00:00:00Z", + muted=False, + ) + pass_finding.add_resources([resource1]) + + fail_muted_finding = Finding.objects.create( + tenant_id=tenant.id, + uid="fg_mixed_muted_fail", + scan=scan1, + delta=None, + status=Status.FAIL, + severity=Severity.high, + impact=Severity.high, + check_id="mixed_muted_check", + check_metadata={ + "CheckId": "mixed_muted_check", + "checktitle": "Mixed muted check", + "Description": "Fixture for muted status aggregation.", + }, + first_seen_at="2024-01-12T00:00:00Z", + muted=True, + ) + fail_muted_finding.add_resources([resource1]) + + # filter[region] forces finding-level aggregation so we exercise the + # raw-findings path without touching the daily summary fixture. + response = authenticated_client.get( + reverse("finding-group-list"), + { + "filter[inserted_at]": TODAY, + "filter[check_id]": "mixed_muted_check", + "filter[region]": "us-east-1", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + attrs = data[0]["attributes"] + assert attrs["status"] == "PASS" + assert attrs["muted"] is False + assert attrs["pass_count"] == 1 + assert attrs["fail_count"] == 0 + assert attrs["fail_muted_count"] == 1 + assert attrs["muted_count"] == 1 + + def test_finding_groups_status_filter( + self, authenticated_client, finding_groups_fixture + ): + """Test finding groups can be filtered by aggregated status.""" + response = authenticated_client.get( + reverse("finding-group-list"), + {"filter[inserted_at]": TODAY, "filter[status]": "FAIL"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + assert all(item["attributes"]["status"] == "FAIL" for item in data) + + def test_finding_groups_status_in_filter( + self, authenticated_client, finding_groups_fixture + ): + """Test finding groups support status__in filter on aggregated status.""" + response = authenticated_client.get( + reverse("finding-group-list"), + {"filter[inserted_at]": TODAY, "filter[status__in]": "FAIL,PASS"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + assert all(item["attributes"]["status"] in {"FAIL", "PASS"} for item in data) + + def test_finding_groups_severity_filter( + self, authenticated_client, finding_groups_fixture + ): + """Test finding groups can be filtered by aggregated severity.""" + response = authenticated_client.get( + reverse("finding-group-list"), + {"filter[inserted_at]": TODAY, "filter[severity]": "critical"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + assert all(item["attributes"]["severity"] == "critical" for item in data) + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + def test_finding_groups_combined_region_and_status_filters( + self, authenticated_client, finding_groups_fixture, endpoint_name + ): + """Test combined region + aggregated status filters.""" + params = {"filter[region]": "us-east-1", "filter[status]": "FAIL"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + check_ids = {item["id"] for item in data} + assert check_ids == {"s3_bucket_public_access", "cloudtrail_enabled"} + assert all(item["attributes"]["status"] == "FAIL" for item in data) + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + def test_finding_groups_combined_delta_and_severity_filters( + self, authenticated_client, finding_groups_fixture, endpoint_name + ): + """Test combined delta + aggregated severity filters.""" + params = {"filter[delta]": "new", "filter[severity]": "critical"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + check_ids = {item["id"] for item in data} + assert check_ids == {"s3_bucket_public_access", "cloudtrail_enabled"} + assert all(item["attributes"]["severity"] == "critical" for item in data) + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + @pytest.mark.parametrize( + "filter_key,filter_value", + [ + ("status", "INVALID_STATUS"), + ("severity", "INVALID_SEVERITY"), + ], + ) + def test_finding_groups_invalid_status_or_severity_returns_400( + self, + authenticated_client, + finding_groups_fixture, + endpoint_name, + filter_key, + filter_value, + ): + """Test invalid aggregated status/severity values are rejected.""" + params = {f"filter[{filter_key}]": filter_value} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json()["errors"][0]["code"] == "invalid" + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + @pytest.mark.parametrize( + "filter_key,filter_value,expected_detail", + [ + ("status__in", "FAIL,INVALID_STATUS", "invalid status filter"), + ("severity__in", "critical,INVALID_SEVERITY", "invalid severity filter"), + ], + ) + def test_finding_groups_invalid_in_filters_return_400( + self, + authenticated_client, + finding_groups_fixture, + endpoint_name, + filter_key, + filter_value, + expected_detail, + ): + """Test invalid values in status__in/severity__in are rejected.""" + params = {f"filter[{filter_key}]": filter_value} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json()["errors"] + assert errors[0]["code"] == "invalid" + assert expected_detail in errors[0]["detail"] + + @pytest.mark.parametrize( + "filter_name,filter_value", + [ + ("region", "__region_does_not_exist__"), + ("service", "__service_does_not_exist__"), + ("category", "__category_does_not_exist__"), + ("resource_groups", "__group_does_not_exist__"), + ("resource_type", "__type_does_not_exist__"), + ("scan", "00000000-0000-7000-8000-000000000001"), + ], + ) + def test_finding_groups_finding_level_filters_are_applied( + self, + authenticated_client, + finding_groups_fixture, + filter_name, + filter_value, + ): + """Test finding-level filters are applied in /finding-groups aggregation.""" + response = authenticated_client.get( + reverse("finding-group-list"), + {"filter[inserted_at]": TODAY, f"filter[{filter_name}]": filter_value}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 0 + + def test_finding_groups_delta_filter_is_applied( + self, authenticated_client, finding_groups_fixture + ): + """Test delta filter is applied in /finding-groups aggregation.""" + response = authenticated_client.get( + reverse("finding-group-list"), + {"filter[inserted_at]": TODAY, "filter[delta]": "new"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + assert all(item["attributes"]["new_count"] > 0 for item in data) def test_finding_groups_provider_aggregation( self, authenticated_client, finding_groups_fixture @@ -15522,6 +16385,67 @@ class TestFindingGroupViewSet: assert len(response.json()["data"]) == 1 assert "bucket" in response.json()["data"][0]["id"].lower() + def test_finding_groups_check_title_icontains( + self, authenticated_client, finding_groups_fixture + ): + """Test searching check titles with icontains.""" + response = authenticated_client.get( + reverse("finding-group-list"), + { + "filter[inserted_at]": TODAY, + "filter[check_title.icontains]": "public access", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["id"] == "s3_bucket_public_access" + + @pytest.mark.parametrize( + "extra_filters", + [ + {}, + {"filter[delta]": "new"}, + ], + ids=["summary_path", "finding_level_path"], + ) + def test_check_title_icontains_includes_all_title_variants( + self, + authenticated_client, + finding_groups_title_variants_fixture, + extra_filters, + ): + """ + Regression: two providers report the same check_id with different + checktitle values (e.g. after a Prowler version upgrade). Filtering + by check_title__icontains with a term that matches only ONE variant + must still return the finding group with counts from BOTH providers. + + Parametrized to cover both aggregation paths: + - summary_path: default, uses _CheckTitleToCheckIdMixin on summaries + - finding_level_path: filter[delta]=new forces _aggregate_findings via + CommonFindingFilters (delta is finding-level, not summary-level) + """ + params = { + "filter[inserted_at]": TODAY, + "filter[check_title.icontains]": "Ensure repository", + **extra_filters, + } + response = authenticated_client.get( + reverse("finding-group-list"), + params, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["id"] == "github_secret_scanning_enabled" + attrs = data[0]["attributes"] + # Both providers' findings must be counted + assert attrs["fail_count"] == 2, ( + "fail_count must include findings from both providers, " + "regardless of which title variant matches the search" + ) + def test_resources_not_found(self, authenticated_client): """Test 404 returned for nonexistent check_id.""" response = authenticated_client.get( @@ -15543,6 +16467,36 @@ class TestFindingGroupViewSet: # s3_bucket_public_access has 2 findings with 2 different resources assert len(data) == 2 + def test_resources_id_matches_resource_id_for_mapped_findings( + self, authenticated_client, finding_groups_fixture + ): + """Findings with a resource expose the resource id as row id (hot path contract).""" + response = authenticated_client.get( + reverse( + "finding-group-resources", kwargs={"pk": "s3_bucket_public_access"} + ), + {"filter[inserted_at]": TODAY}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data, "expected resources in response" + + resource_ids = set( + ResourceFindingMapping.objects.filter( + finding__check_id="s3_bucket_public_access", + ).values_list("resource_id", flat=True) + ) + finding_ids = set( + Finding.objects.filter( + check_id="s3_bucket_public_access", + ).values_list("id", flat=True) + ) + + returned_ids = {item["id"] for item in data} + assert returned_ids <= {str(rid) for rid in resource_ids} + assert returned_ids.isdisjoint({str(fid) for fid in finding_ids}) + def test_resources_fields(self, authenticated_client, finding_groups_fixture): """Test resource fields (uid, name, service, region, type) have valid values.""" response = authenticated_client.get( @@ -15563,6 +16517,44 @@ class TestFindingGroupViewSet: assert resource.get("region"), "resource.region must not be empty" assert resource.get("type"), "resource.type must not be empty" + def test_resources_resource_group( + self, authenticated_client, finding_groups_fixture + ): + """Test resource_group is extracted from check_metadata.resourcegroup.""" + response = authenticated_client.get( + reverse( + "finding-group-resources", kwargs={"pk": "s3_bucket_public_access"} + ), + {"filter[inserted_at]": TODAY}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + for item in data: + resource = item["attributes"]["resource"] + assert ( + resource["resource_group"] == "storage" + ), "resource_group must be 'storage'" + + def test_resources_name_icontains( + self, authenticated_client, finding_groups_fixture + ): + """Test resource_name__icontains filters resources by name substring.""" + # s3_bucket_public_access has "My Instance 1" and "My Instance 2" + response = authenticated_client.get( + reverse( + "finding-group-resources", kwargs={"pk": "s3_bucket_public_access"} + ), + { + "filter[inserted_at]": TODAY, + "filter[resource_name.icontains]": "Instance 1", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert "Instance 1" in data[0]["attributes"]["resource"]["name"] + def test_resources_provider_info( self, authenticated_client, finding_groups_fixture ): @@ -15641,6 +16633,191 @@ class TestFindingGroupViewSet: # Should still return the 2 resources within the date range assert len(response.json()["data"]) == 2 + def test_resources_status_filter_returns_empty_not_404( + self, authenticated_client, finding_groups_fixture + ): + """Test that filtering by status on a valid check returns empty list, not 404.""" + # s3_bucket_public_access has only FAIL findings, filtering by PASS should return [] + response = authenticated_client.get( + reverse( + "finding-group-resources", kwargs={"pk": "s3_bucket_public_access"} + ), + {"filter[inserted_at]": TODAY, "filter[status]": "PASS"}, + ) + assert response.status_code == status.HTTP_200_OK + assert response.json()["data"] == [] + + def test_resources_nonexistent_check_still_404( + self, authenticated_client, finding_groups_fixture + ): + """Test that a truly nonexistent check_id still returns 404.""" + response = authenticated_client.get( + reverse("finding-group-resources", kwargs={"pk": "totally_fake_check"}), + {"filter[inserted_at]": TODAY}, + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_resources_sort_by_status_ascending( + self, authenticated_client, finding_groups_fixture + ): + """Test sort=status returns PASS before FAIL.""" + # ec2_instance_public_ip has 1 PASS (resource1) and 1 FAIL (resource2) + response = authenticated_client.get( + reverse( + "finding-group-resources", + kwargs={"pk": "ec2_instance_public_ip"}, + ), + {"filter[inserted_at]": TODAY, "sort": "status"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + assert data[0]["attributes"]["status"] == "PASS" + assert data[1]["attributes"]["status"] == "FAIL" + + def test_resources_sort_by_status_descending( + self, authenticated_client, finding_groups_fixture + ): + """Test sort=-status returns FAIL before PASS.""" + response = authenticated_client.get( + reverse( + "finding-group-resources", + kwargs={"pk": "ec2_instance_public_ip"}, + ), + {"filter[inserted_at]": TODAY, "sort": "-status"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + assert data[0]["attributes"]["status"] == "FAIL" + assert data[1]["attributes"]["status"] == "PASS" + + def test_resources_sort_invalid_field_returns_400( + self, authenticated_client, finding_groups_fixture + ): + """Test that an invalid sort field returns 400.""" + response = authenticated_client.get( + reverse( + "finding-group-resources", kwargs={"pk": "s3_bucket_public_access"} + ), + {"filter[inserted_at]": TODAY, "sort": "invalid_field"}, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_latest_resources_status_filter_returns_empty_not_404( + self, authenticated_client, finding_groups_fixture + ): + """Test latest resources with status filter on valid check returns empty, not 404.""" + response = authenticated_client.get( + reverse( + "finding-group-latest_resources", + kwargs={"check_id": "s3_bucket_public_access"}, + ), + {"filter[status]": "PASS"}, + ) + assert response.status_code == status.HTTP_200_OK + assert response.json()["data"] == [] + + def test_latest_resources_sort_by_status( + self, authenticated_client, finding_groups_fixture + ): + """Test latest resources sort=status returns PASS before FAIL.""" + response = authenticated_client.get( + reverse( + "finding-group-latest_resources", + kwargs={"check_id": "ec2_instance_public_ip"}, + ), + {"sort": "status"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + assert data[0]["attributes"]["status"] == "PASS" + assert data[1]["attributes"]["status"] == "FAIL" + + def test_resources_nonexistent_check_missing_date_returns_400( + self, authenticated_client, finding_groups_fixture + ): + """Nonexistent check_id with missing required date filter returns 400, not 404.""" + response = authenticated_client.get( + reverse("finding-group-resources", kwargs={"pk": "totally_fake_check"}), + ) + # FindingGroupFilter requires inserted_at — validation fires before existence check + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_resources_nonexistent_check_invalid_sort_returns_400( + self, authenticated_client, finding_groups_fixture + ): + """Nonexistent check_id with invalid sort returns 400, not 404.""" + response = authenticated_client.get( + reverse("finding-group-resources", kwargs={"pk": "totally_fake_check"}), + {"filter[inserted_at]": TODAY, "sort": "invalid_field"}, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_resources_empty_sort_falls_back_to_default_order( + self, authenticated_client, finding_groups_fixture + ): + """Degenerate sort values should behave like no sort, not raise 500.""" + all_ids = set() + for page_num in (1, 2): + response = authenticated_client.get( + reverse( + "finding-group-resources", + kwargs={"pk": "s3_bucket_public_access"}, + ), + { + "filter[inserted_at]": TODAY, + "sort": ",", + "page[size]": 1, + "page[number]": page_num, + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + all_ids.add(data[0]["id"]) + assert len(all_ids) == 2 + + def test_resources_sort_pagination_stability( + self, authenticated_client, finding_groups_fixture + ): + """Sort with small page size returns all resources without duplicates or gaps.""" + # s3_bucket_public_access has 2 resources, both FAIL — they tie on status + all_ids = set() + for page_num in (1, 2): + response = authenticated_client.get( + reverse( + "finding-group-resources", + kwargs={"pk": "s3_bucket_public_access"}, + ), + { + "filter[inserted_at]": TODAY, + "sort": "status", + "page[size]": 1, + "page[number]": page_num, + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + all_ids.add(data[0]["id"]) + # Both pages should return different resources (no duplicates) + assert len(all_ids) == 2 + + def test_latest_resources_nonexistent_check_invalid_sort_returns_400( + self, authenticated_client, finding_groups_fixture + ): + """Nonexistent check_id with invalid sort on latest returns 400, not 404.""" + response = authenticated_client.get( + reverse( + "finding-group-latest_resources", + kwargs={"check_id": "totally_fake_check"}, + ), + {"sort": "invalid_field"}, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + # Test provider_id filter actually filters data def test_finding_groups_provider_id_filter_actually_filters( self, authenticated_client, finding_groups_fixture, providers_fixture @@ -15820,6 +16997,258 @@ class TestFindingGroupViewSet: assert len(data) == 1 assert data[0]["id"] == "cloudtrail_enabled" + def test_finding_groups_latest_status_filter( + self, authenticated_client, finding_groups_fixture + ): + """Test /latest supports status filter on aggregated status.""" + response = authenticated_client.get( + reverse("finding-group-latest"), + {"filter[status]": "FAIL"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + assert all(item["attributes"]["status"] == "FAIL" for item in data) + + def test_finding_groups_latest_region_filter_reaggregates_metrics( + self, authenticated_client, finding_groups_fixture + ): + """Test /latest recomputes metrics from findings matching region filter.""" + response = authenticated_client.get( + reverse("finding-group-latest"), + { + "filter[check_id]": "ec2_instance_public_ip", + "filter[region]": "us-east-1", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + + attrs = data[0]["attributes"] + assert attrs["status"] == "PASS" + assert attrs["pass_count"] == 1 + assert attrs["fail_count"] == 0 + assert attrs["resources_total"] == 1 + assert attrs["resources_fail"] == 0 + + def test_finding_groups_latest_status_in_filter( + self, authenticated_client, finding_groups_fixture + ): + """Test /latest supports status__in filter on aggregated status.""" + response = authenticated_client.get( + reverse("finding-group-latest"), + {"filter[status__in]": "FAIL,PASS"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + assert all(item["attributes"]["status"] in {"FAIL", "PASS"} for item in data) + + def test_finding_groups_latest_severity_filter( + self, authenticated_client, finding_groups_fixture + ): + """Test /latest supports severity filter on aggregated severity.""" + response = authenticated_client.get( + reverse("finding-group-latest"), + {"filter[severity]": "critical"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + assert all(item["attributes"]["severity"] == "critical" for item in data) + + @pytest.mark.parametrize( + "filter_name,filter_value", + [ + ("region", "__region_does_not_exist__"), + ("service", "__service_does_not_exist__"), + ("category", "__category_does_not_exist__"), + ("resource_groups", "__group_does_not_exist__"), + ("resource_type", "__type_does_not_exist__"), + ("scan", "00000000-0000-7000-8000-000000000001"), + ], + ) + def test_finding_groups_latest_finding_level_filters_are_applied( + self, + authenticated_client, + finding_groups_fixture, + filter_name, + filter_value, + ): + """Test finding-level filters are applied in /finding-groups/latest aggregation.""" + response = authenticated_client.get( + reverse("finding-group-latest"), + {f"filter[{filter_name}]": filter_value}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 0 + + def test_finding_groups_check_title_filter_applies_with_delta( + self, authenticated_client, finding_groups_fixture + ): + """Test check_title filter is honored when finding-level path is used.""" + response = authenticated_client.get( + reverse("finding-group-list"), + { + "filter[inserted_at]": TODAY, + "filter[delta]": "new", + "filter[check_title.icontains]": "__missing_check_title__", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 0 + + def test_finding_groups_latest_check_title_filter_applies_with_delta( + self, authenticated_client, finding_groups_fixture + ): + """Test /latest check_title filter is honored on finding-level path.""" + response = authenticated_client.get( + reverse("finding-group-latest"), + { + "filter[delta]": "new", + "filter[check_title.icontains]": "__missing_check_title__", + }, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 0 + + def test_finding_groups_latest_delta_filter_is_applied( + self, authenticated_client, finding_groups_fixture + ): + """Test delta filter is applied in /finding-groups/latest aggregation.""" + response = authenticated_client.get( + reverse("finding-group-latest"), + {"filter[delta]": "new"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + assert all(item["attributes"]["new_count"] > 0 for item in data) + + def test_finding_groups_latest_aggregates_latest_per_provider( + self, + authenticated_client, + providers_fixture, + resources_fixture, + ): + """Test /latest keeps all findings from the latest scan per provider. + + Verifies that when the latest scan produces multiple findings for the + same check_id (e.g. one per resource), all of them are included in the + aggregation — not just one. + """ + provider1 = providers_fixture[0] + provider2 = providers_fixture[1] + resource1 = resources_fixture[0] + resource2 = resources_fixture[1] + resource3 = resources_fixture[2] + check_id = "cross_provider_latest_resources_total" + + latest_scan_provider1 = Scan.objects.create( + tenant_id=provider1.tenant_id, + provider=provider1, + state=StateChoices.COMPLETED, + trigger=Scan.TriggerChoices.MANUAL, + completed_at=datetime.now(timezone.utc), + ) + + latest_scan_provider2 = Scan.objects.create( + tenant_id=provider2.tenant_id, + provider=provider2, + state=StateChoices.COMPLETED, + trigger=Scan.TriggerChoices.MANUAL, + completed_at=datetime.now(timezone.utc), + ) + + older_scan_provider1 = Scan.objects.create( + tenant_id=provider1.tenant_id, + provider=provider1, + state=StateChoices.COMPLETED, + trigger=Scan.TriggerChoices.MANUAL, + completed_at=datetime.now(timezone.utc) - timedelta(days=1), + ) + + # Older scan — these should be excluded from /latest + Finding.objects.create( + tenant_id=provider1.tenant_id, + uid="old_cross_provider_1", + scan=older_scan_provider1, + delta="new", + status="FAIL", + severity="high", + impact="high", + check_id=check_id, + check_metadata={"CheckId": check_id, "checktitle": "Cross provider check"}, + first_seen_at=datetime.now(timezone.utc) - timedelta(days=2), + muted=False, + ) + + # Latest scan provider1: TWO findings (PASS + FAIL) for the same check + latest_p1_pass = Finding.objects.create( + tenant_id=provider1.tenant_id, + uid="latest_cross_provider_1_pass", + scan=latest_scan_provider1, + delta="new", + status="PASS", + severity="high", + impact="high", + check_id=check_id, + check_metadata={"CheckId": check_id, "checktitle": "Cross provider check"}, + first_seen_at=datetime.now(timezone.utc) - timedelta(hours=1), + muted=False, + ) + latest_p1_pass.add_resources([resource1]) + + latest_p1_fail = Finding.objects.create( + tenant_id=provider1.tenant_id, + uid="latest_cross_provider_1_fail", + scan=latest_scan_provider1, + delta="new", + status="FAIL", + severity="high", + impact="high", + check_id=check_id, + check_metadata={"CheckId": check_id, "checktitle": "Cross provider check"}, + first_seen_at=datetime.now(timezone.utc) - timedelta(hours=1), + muted=False, + ) + latest_p1_fail.add_resources([resource2]) + + # Latest scan provider2: one finding + latest_p2 = Finding.objects.create( + tenant_id=provider2.tenant_id, + uid="latest_cross_provider_2", + scan=latest_scan_provider2, + delta="new", + status="FAIL", + severity="high", + impact="high", + check_id=check_id, + check_metadata={"CheckId": check_id, "checktitle": "Cross provider check"}, + first_seen_at=datetime.now(timezone.utc) - timedelta(hours=1), + muted=False, + ) + latest_p2.add_resources([resource3]) + + response = authenticated_client.get( + reverse("finding-group-latest"), + {"filter[check_id]": check_id, "filter[delta]": "new"}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + attrs = data[0]["attributes"] + # 3 findings total: 2 from provider1 latest + 1 from provider2 latest + assert attrs["pass_count"] == 1 + assert attrs["fail_count"] == 2 + assert attrs["resources_total"] == 3 + assert attrs["resources_fail"] == 2 + def test_finding_groups_latest_provider_type_filter( self, authenticated_client, finding_groups_fixture ): @@ -15859,6 +17288,77 @@ class TestFindingGroupViewSet: check_ids = [item["id"] for item in data] assert check_ids == sorted(check_ids) + def test_finding_groups_latest_sort_by_check_title( + self, authenticated_client, finding_groups_fixture + ): + """Test /latest supports sorting by check_title.""" + response = authenticated_client.get( + reverse("finding-group-latest"), + {"sort": "check_title"}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + check_titles = [item["attributes"]["check_title"] for item in data] + assert check_titles == sorted(check_titles) + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + @pytest.mark.parametrize( + "sort_field", + ["first_seen_at", "-first_seen_at", "last_seen_at", "failing_since"], + ) + def test_finding_groups_sort_by_time_fields( + self, + authenticated_client, + finding_groups_fixture, + endpoint_name, + sort_field, + ): + """Test sorting by aggregated time fields (first_seen_at, last_seen_at, failing_since).""" + params = {"sort": sort_field} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + def test_finding_groups_sort_by_delta( + self, + authenticated_client, + finding_groups_fixture, + endpoint_name, + ): + """Sort by delta orders by new_count then changed_count (lexicographic).""" + params = {"sort": "-delta"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + + def delta_key(item): + attrs = item["attributes"] + return (attrs.get("new_count", 0), attrs.get("changed_count", 0)) + + desc_keys = [delta_key(item) for item in data] + assert desc_keys == sorted(desc_keys, reverse=True) + + # Ascending order produces the inverse arrangement + params["sort"] = "delta" + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + asc_keys = [delta_key(item) for item in response.json()["data"]] + assert asc_keys == sorted(asc_keys) + def test_finding_groups_latest_ignores_date_filters( self, authenticated_client, finding_groups_fixture ): @@ -15872,3 +17372,446 @@ class TestFindingGroupViewSet: data = response.json()["data"] # Should still return data, not filtered by the old date assert len(data) == 5 + + def test_finding_groups_status_choices_no_muted( + self, authenticated_client, finding_groups_fixture + ): + """Every returned group must have status ∈ {FAIL, PASS, MANUAL}.""" + response = authenticated_client.get( + reverse("finding-group-list"), + {"filter[inserted_at]": TODAY}, + ) + assert response.status_code == status.HTTP_200_OK + statuses = {item["attributes"]["status"] for item in response.json()["data"]} + assert statuses, "fixture should produce at least one group" + assert statuses <= {"FAIL", "PASS", "MANUAL"} + assert "MUTED" not in statuses + + def test_finding_groups_serializer_exposes_muted_and_manual_count( + self, authenticated_client, finding_groups_fixture + ): + """The /finding-groups payload must expose `muted`, `manual_count` and + the per-status muted siblings (`pass_muted_count`/`fail_muted_count`/ + `manual_muted_count`).""" + response = authenticated_client.get( + reverse("finding-group-list"), + {"filter[inserted_at]": TODAY, "filter[check_id]": "iam_password_policy"}, + ) + assert response.status_code == status.HTTP_200_OK + attrs = response.json()["data"][0]["attributes"] + assert "muted" in attrs and isinstance(attrs["muted"], bool) + assert "manual_count" in attrs and isinstance(attrs["manual_count"], int) + assert attrs["muted"] is False # iam_password_policy has only non-muted PASS + assert attrs["manual_count"] == 0 + assert attrs["pass_muted_count"] == 0 + assert attrs["fail_muted_count"] == 0 + assert attrs["manual_muted_count"] == 0 + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + def test_finding_groups_filter_status_muted_is_rejected( + self, authenticated_client, finding_groups_fixture, endpoint_name + ): + """`filter[status]=MUTED` is no longer a valid status value.""" + params = {"filter[status]": "MUTED"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + def test_finding_groups_filter_muted_true( + self, authenticated_client, finding_groups_fixture, endpoint_name + ): + """`filter[muted]=true` returns only fully-muted groups.""" + params = {"filter[muted]": "true"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + check_ids = {item["id"] for item in data} + # Only rds_encryption is fully muted in the fixture + assert check_ids == {"rds_encryption"} + assert all(item["attributes"]["muted"] is True for item in data) + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + def test_finding_groups_filter_muted_false( + self, authenticated_client, finding_groups_fixture, endpoint_name + ): + """`filter[muted]=false` returns only groups with actionable findings.""" + params = {"filter[muted]": "false"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + check_ids = {item["id"] for item in data} + assert "rds_encryption" not in check_ids + assert check_ids == { + "s3_bucket_public_access", + "ec2_instance_public_ip", + "iam_password_policy", + "cloudtrail_enabled", + } + assert all(item["attributes"]["muted"] is False for item in data) + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + def test_finding_groups_sort_by_status( + self, authenticated_client, finding_groups_fixture, endpoint_name + ): + """sort=status orders by aggregated status (FAIL > PASS > MANUAL).""" + priority = {"FAIL": 3, "PASS": 2, "MANUAL": 1} + params = {"sort": "-status"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data, "fixture should produce groups" + + desc_keys = [priority[item["attributes"]["status"]] for item in data] + assert desc_keys == sorted(desc_keys, reverse=True) + + params["sort"] = "status" + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + asc_keys = [ + priority[item["attributes"]["status"]] for item in response.json()["data"] + ] + assert asc_keys == sorted(asc_keys) + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + def test_finding_groups_sort_by_muted( + self, authenticated_client, finding_groups_fixture, endpoint_name + ): + """sort=muted orders by the boolean muted attribute.""" + # Need include_muted=true so the fully-muted group is part of the result + params = {"sort": "-muted", "filter[include_muted]": "true"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data, "fixture should produce groups" + + muted_values = [item["attributes"]["muted"] for item in data] + # Descending boolean: True (1) before False (0) + assert muted_values == sorted(muted_values, reverse=True) + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + @pytest.mark.parametrize( + "sort_field", + [ + "pass_muted_count", + "fail_muted_count", + "manual_muted_count", + "new_fail_count", + "new_fail_muted_count", + "new_pass_count", + "new_pass_muted_count", + "new_manual_count", + "new_manual_muted_count", + "changed_fail_count", + "changed_fail_muted_count", + "changed_pass_count", + "changed_pass_muted_count", + "changed_manual_count", + "changed_manual_muted_count", + ], + ) + def test_finding_groups_sort_by_counter_fields( + self, + authenticated_client, + finding_groups_fixture, + endpoint_name, + sort_field, + ): + """All counter fields are accepted as sort parameters (asc and desc).""" + params = {"sort": f"-{sort_field}"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) > 0 + + desc_values = [item["attributes"][sort_field] for item in data] + assert desc_values == sorted(desc_values, reverse=True) + + params["sort"] = sort_field + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + asc_values = [ + item["attributes"][sort_field] for item in response.json()["data"] + ] + assert asc_values == sorted(asc_values) + + @pytest.mark.parametrize( + "endpoint_name", ["finding-group-list", "finding-group-latest"] + ) + def test_finding_groups_delta_status_breakdown( + self, authenticated_client, finding_groups_fixture, endpoint_name + ): + """`new_*` and `changed_*` counters split by status and mute state. + + s3_bucket_public_access has 1 new FAIL and 1 changed FAIL (both + non-muted) so the breakdown must reflect exactly that and the totals + must equal the sum of the buckets. + """ + params = {"filter[check_id]": "s3_bucket_public_access"} + if endpoint_name == "finding-group-list": + params["filter[inserted_at]"] = TODAY + + response = authenticated_client.get(reverse(endpoint_name), params) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + attrs = data[0]["attributes"] + + assert attrs["new_fail_count"] == 1 + assert attrs["new_fail_muted_count"] == 0 + assert attrs["new_pass_count"] == 0 + assert attrs["new_pass_muted_count"] == 0 + assert attrs["new_manual_count"] == 0 + assert attrs["new_manual_muted_count"] == 0 + assert attrs["changed_fail_count"] == 1 + assert attrs["changed_fail_muted_count"] == 0 + assert attrs["changed_pass_count"] == 0 + assert attrs["changed_pass_muted_count"] == 0 + assert attrs["changed_manual_count"] == 0 + assert attrs["changed_manual_muted_count"] == 0 + + new_total = ( + attrs["new_fail_count"] + + attrs["new_fail_muted_count"] + + attrs["new_pass_count"] + + attrs["new_pass_muted_count"] + + attrs["new_manual_count"] + + attrs["new_manual_muted_count"] + ) + changed_total = ( + attrs["changed_fail_count"] + + attrs["changed_fail_muted_count"] + + attrs["changed_pass_count"] + + attrs["changed_pass_muted_count"] + + attrs["changed_manual_count"] + + attrs["changed_manual_muted_count"] + ) + # The non-muted variants of the breakdown must sum to the legacy + # totals (new_count/changed_count are stored as non-muted). + assert ( + attrs["new_fail_count"] + + attrs["new_pass_count"] + + attrs["new_manual_count"] + == attrs["new_count"] + ) + assert ( + attrs["changed_fail_count"] + + attrs["changed_pass_count"] + + attrs["changed_manual_count"] + == attrs["changed_count"] + ) + # And the *full* breakdown (including the muted halves) is exposed + # so clients can also count muted-only deltas without losing data. + assert new_total >= attrs["new_count"] + assert changed_total >= attrs["changed_count"] + + def test_finding_groups_resources_serializer_exposes_muted( + self, authenticated_client, finding_groups_fixture + ): + """The /finding-groups//resources payload must expose `muted`.""" + response = authenticated_client.get( + reverse( + "finding-group-resources", + kwargs={"pk": "rds_encryption"}, + ), + {"filter[inserted_at]": TODAY}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data, "rds_encryption should expose its resources" + for item in data: + attrs = item["attributes"] + assert "muted" in attrs and isinstance(attrs["muted"], bool) + # rds_encryption has all muted findings + assert attrs["muted"] is True + # Status reflects the underlying check outcome (FAIL), not MUTED + assert attrs["status"] == "FAIL" + + def test_finding_groups_resources_exposes_finding_id( + self, authenticated_client, finding_groups_fixture + ): + """The /resources payload exposes the most recent matching finding_id. + + rds_encryption has 2 findings, one per resource. Each resource row must + report the UUID of its corresponding Finding (UUIDv7 ordering means + Max(finding__id) resolves to the latest snapshot in time). + """ + response = authenticated_client.get( + reverse( + "finding-group-resources", + kwargs={"pk": "rds_encryption"}, + ), + {"filter[inserted_at]": TODAY}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data, "rds_encryption should expose its resources" + + rds_finding_ids = { + str(f.id) for f in finding_groups_fixture if f.check_id == "rds_encryption" + } + assert rds_finding_ids, "fixture sanity" + + for item in data: + attrs = item["attributes"] + assert "finding_id" in attrs + assert attrs["finding_id"] in rds_finding_ids + + def test_finding_groups_latest_resources_exposes_finding_id( + self, authenticated_client, finding_groups_fixture + ): + """The /latest/.../resources payload also exposes finding_id.""" + response = authenticated_client.get( + reverse( + "finding-group-latest_resources", + kwargs={"check_id": "rds_encryption"}, + ), + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data, "rds_encryption should expose its resources via /latest" + + rds_finding_ids = { + str(f.id) for f in finding_groups_fixture if f.check_id == "rds_encryption" + } + for item in data: + attrs = item["attributes"] + assert "finding_id" in attrs + assert attrs["finding_id"] in rds_finding_ids + + def test_latest_resources_picks_scan_by_completed_at_when_overlap( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + resources_fixture, + ): + """Overlapping scans on the same provider must resolve to the scan + with the latest completed_at, matching the /latest summary path and + the daily-summary upsert (keyed on midnight(completed_at)). Picking + by inserted_at here produced /resources and /latest reading from + different scans and reporting diverging delta/new counts. + """ + tenant = tenants_fixture[0] + provider = providers_fixture[0] + resource = resources_fixture[0] + check_id = "overlap_regression_check" + + t0 = datetime.now(timezone.utc) - timedelta(hours=5) + t1 = t0 + timedelta(hours=1) + t1_end = t1 + timedelta(minutes=30) + t2 = t0 + timedelta(hours=4) + + scan_long = Scan.objects.create( + name="long overlap scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant_id=tenant.id, + started_at=t0, + completed_at=t2, + ) + scan_short = Scan.objects.create( + name="short overlap scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant_id=tenant.id, + started_at=t1, + completed_at=t1_end, + ) + # inserted_at is auto_now_add so override with .update() to recreate + # the overlap shape: short scan inserted later but completed earlier. + Scan.all_objects.filter(pk=scan_long.pk).update(inserted_at=t0) + Scan.all_objects.filter(pk=scan_short.pk).update(inserted_at=t1) + scan_long.refresh_from_db() + scan_short.refresh_from_db() + + assert scan_short.inserted_at > scan_long.inserted_at + assert scan_long.completed_at > scan_short.completed_at + + long_finding = Finding.objects.create( + tenant_id=tenant.id, + uid=f"{check_id}_long", + scan=scan_long, + delta=None, + status=Status.FAIL, + status_extended="long scan finding", + impact=Severity.high, + impact_extended="high", + severity=Severity.high, + raw_result={"status": Status.FAIL, "severity": Severity.high}, + check_id=check_id, + check_metadata={ + "CheckId": check_id, + "checktitle": "Overlap regression", + "Description": "Overlapping scan regression.", + }, + first_seen_at=t0, + muted=False, + ) + long_finding.add_resources([resource]) + + short_finding = Finding.objects.create( + tenant_id=tenant.id, + uid=f"{check_id}_short", + scan=scan_short, + delta="new", + status=Status.FAIL, + status_extended="short scan finding", + impact=Severity.high, + impact_extended="high", + severity=Severity.high, + raw_result={"status": Status.FAIL, "severity": Severity.high}, + check_id=check_id, + check_metadata={ + "CheckId": check_id, + "checktitle": "Overlap regression", + "Description": "Overlapping scan regression.", + }, + first_seen_at=t1, + muted=False, + ) + short_finding.add_resources([resource]) + + response = authenticated_client.get( + reverse( + "finding-group-latest_resources", + kwargs={"check_id": check_id}, + ), + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + attrs = data[0]["attributes"] + assert attrs["finding_id"] == str(long_finding.id) + assert attrs["delta"] is None diff --git a/api/src/backend/api/utils.py b/api/src/backend/api/utils.py index 6d4102af55..b80d54b08a 100644 --- a/api/src/backend/api/utils.py +++ b/api/src/backend/api/utils.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: ) from prowler.providers.openstack.openstack_provider import OpenstackProvider from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider + from prowler.providers.vercel.vercel_provider import VercelProvider class CustomOAuth2Client(OAuth2Client): @@ -94,6 +95,7 @@ def return_prowler_provider( | MongodbatlasProvider | OpenstackProvider | OraclecloudProvider + | VercelProvider ): """Return the Prowler provider class based on the given provider type. @@ -175,6 +177,10 @@ def return_prowler_provider( from prowler.providers.image.image_provider import ImageProvider prowler_provider = ImageProvider + case Provider.ProviderChoices.VERCEL.value: + from prowler.providers.vercel.vercel_provider import VercelProvider + + prowler_provider = VercelProvider case _: raise ValueError(f"Provider type {provider.provider} not supported") return prowler_provider @@ -235,6 +241,11 @@ def get_prowler_provider_kwargs( # clouds_yaml_content, clouds_yaml_cloud and provider_id are validated # in the provider itself, so it's not needed here. pass + elif provider.provider == Provider.ProviderChoices.VERCEL.value: + prowler_provider_kwargs = { + **prowler_provider_kwargs, + "team_id": provider.uid, + } elif provider.provider == Provider.ProviderChoices.IMAGE.value: # Detect whether uid is a registry URL (e.g. "docker.io/andoniaf") or # a concrete image reference (e.g. "docker.io/andoniaf/myimage:latest"). @@ -281,6 +292,7 @@ def initialize_prowler_provider( | MongodbatlasProvider | OpenstackProvider | OraclecloudProvider + | VercelProvider ): """Initialize a Prowler provider instance based on the given provider type. @@ -332,6 +344,13 @@ def prowler_provider_connection_test(provider: Provider) -> Connection: "raise_on_exception": False, } return prowler_provider.test_connection(**openstack_kwargs) + elif provider.provider == Provider.ProviderChoices.VERCEL.value: + vercel_kwargs = { + **prowler_provider_kwargs, + "team_id": provider.uid, + "raise_on_exception": False, + } + return prowler_provider.test_connection(**vercel_kwargs) elif provider.provider == Provider.ProviderChoices.IMAGE.value: image_kwargs = { "image": provider.uid, @@ -415,8 +434,12 @@ def prowler_integration_connection_test(integration: Integration) -> Connection: raise_on_exception=False, ) project_keys = jira_connection.projects if jira_connection.is_connected else {} + issue_types = ( + jira_connection.issue_types if jira_connection.is_connected else {} + ) with rls_transaction(str(integration.tenant_id)): integration.configuration["projects"] = project_keys + integration.configuration["issue_types"] = issue_types integration.save() return jira_connection elif integration.integration_type == Integration.IntegrationChoices.SLACK: diff --git a/api/src/backend/api/v1/serializer_utils/integrations.py b/api/src/backend/api/v1/serializer_utils/integrations.py index b389085886..aaa0f4aa31 100644 --- a/api/src/backend/api/v1/serializer_utils/integrations.py +++ b/api/src/backend/api/v1/serializer_utils/integrations.py @@ -69,8 +69,10 @@ class SecurityHubConfigSerializer(BaseValidateSerializer): class JiraConfigSerializer(BaseValidateSerializer): domain = serializers.CharField(read_only=True) - issue_types = serializers.ListField( - read_only=True, child=serializers.CharField(), default=["Task"] + issue_types = serializers.DictField( + read_only=True, + child=serializers.ListField(child=serializers.CharField()), + default={}, ) projects = serializers.DictField(read_only=True) diff --git a/api/src/backend/api/v1/serializer_utils/providers.py b/api/src/backend/api/v1/serializer_utils/providers.py index 034bc79cb2..f8d67f0e3b 100644 --- a/api/src/backend/api/v1/serializer_utils/providers.py +++ b/api/src/backend/api/v1/serializer_utils/providers.py @@ -404,6 +404,17 @@ from rest_framework_json_api import serializers }, "required": ["clouds_yaml_content", "clouds_yaml_cloud"], }, + { + "type": "object", + "title": "Vercel API Token", + "properties": { + "api_token": { + "type": "string", + "description": "Vercel API token for authentication. Can be scoped to a specific team.", + }, + }, + "required": ["api_token"], + }, ] } ) diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 63e60cd88f..51cb95fbff 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -1573,6 +1573,8 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer): serializer = OpenStackCloudsYamlProviderSecret(data=secret) elif provider_type == Provider.ProviderChoices.IMAGE.value: serializer = ImageProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.VERCEL.value: + serializer = VercelProviderSecret(data=secret) else: raise serializers.ValidationError( {"provider": f"Provider type not supported {provider_type}"} @@ -1779,6 +1781,13 @@ class ImageProviderSecret(serializers.Serializer): return attrs +class VercelProviderSecret(serializers.Serializer): + api_token = serializers.CharField() + + class Meta: + resource_name = "provider-secrets" + + class AlibabaCloudProviderSecret(serializers.Serializer): access_key_id = serializers.CharField() access_key_secret = serializers.CharField() @@ -2713,11 +2722,11 @@ class BaseWriteIntegrationSerializer(BaseWriteSerializer): ) config_serializer = JiraConfigSerializer # Create non-editable configuration for JIRA integration - default_jira_issue_types = ["Task"] + # issue_types will be populated per project when connection is tested configuration.update( { "projects": {}, - "issue_types": default_jira_issue_types, + "issue_types": {}, "domain": credentials.get("domain"), } ) @@ -2932,13 +2941,25 @@ class IntegrationUpdateSerializer(BaseWriteIntegrationSerializer): return representation +class IntegrationJiraIssueTypesSerializer(BaseSerializerV1): + """ + Serializer for Jira issue types response. + """ + + project_key = serializers.CharField(read_only=True) + issue_types = serializers.ListField(child=serializers.CharField(), read_only=True) + + class JSONAPIMeta: + resource_name = "jira-issue-types" + + class IntegrationJiraDispatchSerializer(BaseSerializerV1): """ Serializer for dispatching findings to JIRA integration. """ project_key = serializers.CharField(required=True) - issue_type = serializers.ChoiceField(required=True, choices=["Task"]) + issue_type = serializers.CharField(required=True) class JSONAPIMeta: resource_name = "integrations-jira-dispatches" @@ -2967,6 +2988,23 @@ class IntegrationJiraDispatchSerializer(BaseSerializerV1): } ) + issue_type = attrs.get("issue_type") + available_issue_types = integration_instance.configuration.get( + "issue_types", {} + ) + # Handle old format where issue_types was a flat list (e.g., ["Task"]) + if not isinstance(available_issue_types, dict): + available_issue_types = {} + project_issue_types = available_issue_types.get(project_key, []) + if project_issue_types and issue_type not in project_issue_types: + raise ValidationError( + { + "issue_type": f"The issue type '{issue_type}' is not available for project '{project_key}'. " + f"Available types: {', '.join(project_issue_types)}. " + "Refresh the connection if this is an error." + } + ) + return validated_attrs @@ -4147,6 +4185,7 @@ class FindingGroupSerializer(BaseSerializerV1): check_description = serializers.CharField(required=False, allow_null=True) severity = serializers.CharField() status = serializers.CharField() + muted = serializers.BooleanField() impacted_providers = serializers.ListField( child=serializers.CharField(), required=False ) @@ -4154,9 +4193,25 @@ class FindingGroupSerializer(BaseSerializerV1): resources_total = serializers.IntegerField() pass_count = serializers.IntegerField() fail_count = serializers.IntegerField() + manual_count = serializers.IntegerField() + pass_muted_count = serializers.IntegerField() + fail_muted_count = serializers.IntegerField() + manual_muted_count = serializers.IntegerField() muted_count = serializers.IntegerField() new_count = serializers.IntegerField() changed_count = serializers.IntegerField() + new_fail_count = serializers.IntegerField() + new_fail_muted_count = serializers.IntegerField() + new_pass_count = serializers.IntegerField() + new_pass_muted_count = serializers.IntegerField() + new_manual_count = serializers.IntegerField() + new_manual_muted_count = serializers.IntegerField() + changed_fail_count = serializers.IntegerField() + changed_fail_muted_count = serializers.IntegerField() + changed_pass_count = serializers.IntegerField() + changed_pass_muted_count = serializers.IntegerField() + changed_manual_count = serializers.IntegerField() + changed_manual_muted_count = serializers.IntegerField() first_seen_at = serializers.DateTimeField(required=False, allow_null=True) last_seen_at = serializers.DateTimeField(required=False, allow_null=True) failing_since = serializers.DateTimeField(required=False, allow_null=True) @@ -4170,16 +4225,21 @@ class FindingGroupResourceSerializer(BaseSerializerV1): Serializer for Finding Group Resources - resources within a finding group. Returns individual resources with their current status, severity, - and timing information. + and timing information. Orphan findings (without any resource) expose the + finding id as `id` so the row stays identifiable in the UI. """ - id = serializers.UUIDField(source="resource_id") + id = serializers.UUIDField(source="row_id") resource = serializers.SerializerMethodField() provider = serializers.SerializerMethodField() + finding_id = serializers.UUIDField() status = serializers.CharField() severity = serializers.CharField() + muted = serializers.BooleanField() + delta = serializers.CharField(required=False, allow_null=True) first_seen_at = serializers.DateTimeField(required=False, allow_null=True) last_seen_at = serializers.DateTimeField(required=False, allow_null=True) + muted_reason = serializers.CharField(required=False, allow_null=True) class JSONAPIMeta: resource_name = "finding-group-resources" @@ -4193,6 +4253,7 @@ class FindingGroupResourceSerializer(BaseSerializerV1): "service": {"type": "string"}, "region": {"type": "string"}, "type": {"type": "string"}, + "resource_group": {"type": "string"}, }, } ) @@ -4204,6 +4265,7 @@ class FindingGroupResourceSerializer(BaseSerializerV1): "service": obj.get("resource_service", ""), "region": obj.get("resource_region", ""), "type": obj.get("resource_type", ""), + "resource_group": obj.get("resource_group", ""), } @extend_schema_field( diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 8c0518ce89..261b931297 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -4,7 +4,6 @@ import json import logging import os import time - from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta, timezone @@ -12,12 +11,12 @@ from decimal import ROUND_HALF_UP, Decimal, InvalidOperation from urllib.parse import urljoin import sentry_sdk - from allauth.socialaccount.models import SocialAccount, SocialApp from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter from allauth.socialaccount.providers.saml.views import FinishACSView, LoginView from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError +from celery import chain from celery.result import AsyncResult from config.custom_logging import BackendLogger from config.env import env @@ -27,18 +26,22 @@ from config.settings.social_login import ( ) from dj_rest_auth.registration.views import SocialLoginView from django.conf import settings as django_settings -from django.contrib.postgres.aggregates import ArrayAgg, StringAgg +from django.contrib.postgres.aggregates import ArrayAgg, BoolAnd, StringAgg from django.contrib.postgres.search import SearchQuery from django.db import transaction from django.db.models import ( + BooleanField, Case, + CharField, Count, DecimalField, + Exists, ExpressionWrapper, F, IntegerField, Max, Min, + OuterRef, Prefetch, Q, QuerySet, @@ -48,8 +51,9 @@ from django.db.models import ( When, Window, ) -from django.db.models.functions import Coalesce, RowNumber -from django.http import HttpResponse, QueryDict +from django.db.models.fields.json import KeyTextTransform +from django.db.models.functions import Cast, Coalesce, RowNumber +from django.http import HttpResponse, HttpResponseBase, HttpResponseRedirect, QueryDict from django.shortcuts import redirect from django.urls import reverse from django.utils.dateparse import parse_date @@ -76,8 +80,13 @@ from rest_framework.exceptions import ( ) from rest_framework.generics import GenericAPIView, get_object_or_404 from rest_framework.permissions import SAFE_METHODS +from rest_framework_json_api import filters as jsonapi_filters from rest_framework_json_api.views import RelationshipView, Response from rest_framework_simplejwt.exceptions import InvalidToken, TokenError +from rest_framework_simplejwt.token_blacklist.models import ( + BlacklistedToken, + OutstandingToken, +) from tasks.beat import schedule_provider_scan from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils from tasks.jobs.export import get_s3_client @@ -93,6 +102,7 @@ from tasks.tasks import ( jira_integration_task, mute_historical_findings_task, perform_scan_task, + reaggregate_all_finding_group_summaries_task, refresh_lighthouse_provider_models_task, ) @@ -100,7 +110,6 @@ from api.attack_paths import database as graph_database from api.attack_paths import get_queries_for_provider, get_query_by_id from api.attack_paths import views_helpers as attack_paths_views_helpers from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset -from api.renderers import APIJSONRenderer, PlainTextRenderer from api.compliance import ( PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE, get_compliance_frameworks, @@ -124,6 +133,7 @@ from api.filters import ( CustomDjangoFilterBackend, DailySeveritySummaryFilter, FindingFilter, + FindingGroupAggregatedComputedFilter, FindingGroupFilter, FindingGroupSummaryFilter, IntegrationFilter, @@ -163,6 +173,7 @@ from api.models import ( FindingGroupDailySummary, Integration, Invitation, + InvitationRoleRelationship, LighthouseConfiguration, LighthouseProviderConfiguration, LighthouseProviderModels, @@ -199,10 +210,12 @@ from api.models import ( ) from api.pagination import ComplianceOverviewPagination from api.rbac.permissions import Permissions, get_providers, get_role +from api.renderers import APIJSONRenderer, PlainTextRenderer from api.rls import Tenant from api.utils import ( CustomOAuth2Client, get_findings_metadata_no_aggregations, + initialize_prowler_integration, initialize_prowler_provider, validate_invitation, ) @@ -231,6 +244,7 @@ from api.v1.serializers import ( FindingsSeverityOverTimeSerializer, IntegrationCreateSerializer, IntegrationJiraDispatchSerializer, + IntegrationJiraIssueTypesSerializer, IntegrationSerializer, IntegrationUpdateSerializer, InvitationAcceptSerializer, @@ -408,7 +422,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.22.0" + spectacular_settings.VERSION = "1.27.0" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -943,7 +957,12 @@ class UserViewSet(BaseUserViewset): def get_serializer_context(self): context = super().get_serializer_context() if self.request.user.is_authenticated: - context["role"] = get_role(self.request.user) + tenant_id = getattr(self.request, "tenant_id", None) + if tenant_id: + try: + context["role"] = get_role(self.request.user, tenant_id) + except PermissionDenied: + context["role"] = None return context @action(detail=False, methods=["get"], url_name="me") @@ -1207,6 +1226,17 @@ class TenantViewSet(BaseTenantViewset): # RBAC required permissions required_permissions = [Permissions.MANAGE_ACCOUNT] + def set_required_permissions(self): + """ + Returns the required permissions based on the request method. + """ + if self.action in ("list", "retrieve", "create"): + # No permissions required for listing, retrieving or creating tenants + self.required_permissions = [] + else: + # Require MANAGE_ACCOUNT for update and delete + self.required_permissions = [Permissions.MANAGE_ACCOUNT] + def get_queryset(self): queryset = Tenant.objects.filter(membership__user=self.request.user) return queryset.prefetch_related("memberships") @@ -1214,28 +1244,44 @@ class TenantViewSet(BaseTenantViewset): def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) - tenant = serializer.save() - Membership.objects.create( + tenant = Tenant.objects.using(MainRouter.admin_db).create( + **serializer.validated_data + ) + Membership.objects.using(MainRouter.admin_db).create( user=self.request.user, tenant=tenant, role=Membership.RoleChoices.OWNER ) + serializer.instance = tenant return Response(data=serializer.data, status=status.HTTP_201_CREATED) def destroy(self, request, *args, **kwargs): - # This will perform validation and raise a 404 if the tenant does not exist - tenant_id = kwargs.get("pk") - get_object_or_404(Tenant, id=tenant_id) + tenant = self.get_object() + tenant_id = str(tenant.id) + + # Only owners can delete a tenant + membership = Membership.objects.filter(user=request.user, tenant=tenant).first() + if not membership or membership.role != Membership.RoleChoices.OWNER: + raise PermissionDenied("Only owners can delete a tenant.") with transaction.atomic(): - # Delete memberships + # Collect user IDs from this tenant's memberships before deleting them + tenant_user_ids = set( + Membership.objects.using(MainRouter.admin_db) + .filter(tenant_id=tenant_id) + .values_list("user_id", flat=True) + ) + + # Delete memberships for this tenant Membership.objects.using(MainRouter.admin_db).filter( tenant_id=tenant_id ).delete() - # Delete users without memberships - User.objects.using(MainRouter.admin_db).filter( - membership__isnull=True - ).delete() - # Delete tenant in batches + # Delete only users that were exclusively in this tenant + if tenant_user_ids: + User.objects.using(MainRouter.admin_db).filter( + id__in=tenant_user_ids, membership__isnull=True + ).delete() + + # Delete tenant data in background delete_tenant_task.apply_async(kwargs={"tenant_id": tenant_id}) return Response(status=status.HTTP_204_NO_CONTENT) @@ -1289,9 +1335,11 @@ class MembershipViewSet(BaseTenantViewset): ), destroy=extend_schema( summary="Delete tenant memberships", - description="Delete the membership details of users in a tenant. You need to be one of the owners to delete a " - "membership that is not yours. If you are the last owner of a tenant, you cannot delete your own " - "membership.", + description="Delete a user's membership from a tenant. This action: (1) removes the membership, " + "(2) revokes all refresh tokens for the expelled user, (3) removes their role grants for this tenant, " + "(4) cleans up orphaned roles, and (5) deletes the user account if this was their last membership. " + "You must be a tenant owner to delete another user's membership. The last owner of a tenant cannot " + "delete their own membership.", tags=["Tenant"], ), ) @@ -1300,8 +1348,13 @@ class TenantMembersViewSet(BaseTenantViewset): http_method_names = ["get", "delete"] serializer_class = MembershipSerializer queryset = Membership.objects.none() - # RBAC required permissions - required_permissions = [Permissions.MANAGE_ACCOUNT] + filterset_class = MembershipFilter + # Authorization is handled by get_requesting_membership (owner/member checks), + # not by RBAC, since the target tenant differs from the JWT tenant. + required_permissions = [] + + def set_required_permissions(self): + self.required_permissions = [] def get_queryset(self): tenant = self.get_tenant() @@ -1314,8 +1367,10 @@ class TenantMembersViewSet(BaseTenantViewset): def get_tenant(self): tenant_id = self.kwargs.get("tenant_pk") - tenant = get_object_or_404(Tenant, id=tenant_id) - return tenant + return get_object_or_404( + Tenant.objects.filter(membership__user=self.request.user), + id=tenant_id, + ) def get_requesting_membership(self, tenant): try: @@ -1351,7 +1406,84 @@ class TenantMembersViewSet(BaseTenantViewset): "You do not have permission to delete this membership." ) - membership_to_delete.delete() + user_to_check_id = membership_to_delete.user_id + tenant_id = membership_to_delete.tenant_id + # All writes run on the admin connection so that the uncommitted + # membership delete is visible to the subsequent "other memberships" + # check. Splitting the delete and the check across the default + # (prowler_user, RLS) and admin connections caused the admin side to + # miss the just-deleted row and leave the User row orphaned. + with transaction.atomic(using=MainRouter.admin_db): + Membership.objects.using(MainRouter.admin_db).filter( + id=membership_to_delete.id + ).delete() + + # Remove role grants for this user in this tenant to prevent + # orphaned permissions that could allow access after expulsion + deleted_role_relationships = UserRoleRelationship.objects.using( + MainRouter.admin_db + ).filter(user_id=user_to_check_id, tenant_id=tenant_id) + + # Collect role IDs that might become orphaned after deletion + role_ids_to_check = list( + deleted_role_relationships.values_list("role_id", flat=True) + ) + + # Delete the user role relationships for this tenant + deleted_role_relationships.delete() + + # Clean up orphaned roles that have no remaining user or invitation relationships + if role_ids_to_check: + for role_id in role_ids_to_check: + has_user_relationships = ( + UserRoleRelationship.objects.using(MainRouter.admin_db) + .filter(role_id=role_id) + .exists() + ) + + has_invitation_relationships = ( + InvitationRoleRelationship.objects.using(MainRouter.admin_db) + .filter(role_id=role_id) + .exists() + ) + + if not has_user_relationships and not has_invitation_relationships: + Role.objects.using(MainRouter.admin_db).filter( + id=role_id + ).delete() + + # Revoke any refresh tokens the expelled user still holds so they + # cannot mint fresh access tokens. This must happen before the + # User row is deleted, because OutstandingToken.user is + # on_delete=SET_NULL in djangorestframework-simplejwt 5.5.1 + # (see rest_framework_simplejwt/token_blacklist/models.py): once + # the user row is gone, user_id becomes NULL and we can no longer + # look up that user's outstanding tokens. Access tokens already + # issued remain valid until SIMPLE_JWT["ACCESS_TOKEN_LIFETIME"] + # expires. + outstanding_token_ids = list( + OutstandingToken.objects.using(MainRouter.admin_db) + .filter(user_id=user_to_check_id) + .values_list("id", flat=True) + ) + if outstanding_token_ids: + BlacklistedToken.objects.using(MainRouter.admin_db).bulk_create( + [ + BlacklistedToken(token_id=token_id) + for token_id in outstanding_token_ids + ], + ignore_conflicts=True, + ) + + has_other_memberships = ( + Membership.objects.using(MainRouter.admin_db) + .filter(user_id=user_to_check_id) + .exists() + ) + if not has_other_memberships: + User.objects.using(MainRouter.admin_db).filter( + id=user_to_check_id + ).delete() return Response(status=status.HTTP_204_NO_CONTENT) @@ -1402,7 +1534,7 @@ class ProviderGroupViewSet(BaseRLSViewSet): self.required_permissions = [Permissions.MANAGE_PROVIDERS] def get_queryset(self): - user_roles = get_role(self.request.user) + user_roles = get_role(self.request.user, self.request.tenant_id) # Check if any of the user's roles have UNLIMITED_VISIBILITY if user_roles.unlimited_visibility: # User has unlimited visibility, return all provider groups @@ -1571,7 +1703,7 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): self.required_permissions = [Permissions.MANAGE_PROVIDERS] def get_queryset(self): - user_roles = get_role(self.request.user) + user_roles = get_role(self.request.user, self.request.tenant_id) if user_roles.unlimited_visibility: # User has unlimited visibility, return all providers queryset = Provider.objects.filter(tenant_id=self.request.tenant_id) @@ -1794,6 +1926,27 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): ), }, ), + cis=extend_schema( + tags=["Scan"], + summary="Retrieve CIS Benchmark compliance report", + description="Download the CIS Benchmark compliance report as a PDF file. " + "When a provider ships multiple CIS versions, the report is generated " + "for the highest available version.", + request=None, + responses={ + 200: OpenApiResponse( + description="PDF file containing the CIS compliance report" + ), + 202: OpenApiResponse(description="The task is in progress"), + 401: OpenApiResponse( + description="API key missing or user not Authenticated" + ), + 403: OpenApiResponse(description="There is a problem with credentials"), + 404: OpenApiResponse( + description="The scan has no CIS reports, or the CIS report generation task has not started yet" + ), + }, + ), ) @method_decorator(CACHE_DECORATOR, name="list") @method_decorator(CACHE_DECORATOR, name="retrieve") @@ -1826,7 +1979,7 @@ class ScanViewSet(BaseRLSViewSet): self.required_permissions = [Permissions.MANAGE_SCANS] def get_queryset(self): - user_roles = get_role(self.request.user) + user_roles = get_role(self.request.user, self.request.tenant_id) if user_roles.unlimited_visibility: # User has unlimited visibility, return all scans queryset = Scan.objects.filter(tenant_id=self.request.tenant_id) @@ -1862,6 +2015,9 @@ class ScanViewSet(BaseRLSViewSet): elif self.action == "csa": if hasattr(self, "response_serializer_class"): return self.response_serializer_class + elif self.action == "cis": + if hasattr(self, "response_serializer_class"): + return self.response_serializer_class return super().get_serializer_class() def partial_update(self, request, *args, **kwargs): @@ -1924,24 +2080,38 @@ class ScanViewSet(BaseRLSViewSet): }, ) - def _load_file(self, path_pattern, s3=False, bucket=None, list_objects=False): + def _load_file( + self, + path_pattern, + s3=False, + bucket=None, + list_objects=False, + content_type=None, + ): """ - Loads a binary file (e.g., ZIP or CSV) and returns its content and filename. + Resolve a report file location and return the bytes (filesystem) or a redirect (S3). Depending on the input parameters, this method supports loading: - - From S3 using a direct key. - - From S3 by listing objects under a prefix and matching suffix. - - From the local filesystem using glob pattern matching. + - From S3 using a direct key, returns a 302 to a short-lived presigned URL. + - From S3 by listing objects under a prefix and matching suffix, returns a 302 to a short-lived presigned URL. + - From the local filesystem using glob pattern matching, returns the file bytes. + + The S3 branch never streams bytes through the worker; this prevents gunicorn + worker timeouts on large reports. Args: path_pattern (str): The key or glob pattern representing the file location. s3 (bool, optional): Whether the file is stored in S3. Defaults to False. bucket (str, optional): The name of the S3 bucket, required if `s3=True`. Defaults to None. list_objects (bool, optional): If True and `s3=True`, list objects by prefix to find the file. Defaults to False. + content_type (str, optional): On the S3 branch, forwarded as `ResponseContentType` + so the presigned download advertises the same Content-Type the API used to send. + Ignored on the filesystem branch. Returns: - tuple[bytes, str]: A tuple containing the file content as bytes and the filename if successful. - Response: A DRF `Response` object with an appropriate status and error detail if an error occurs. + tuple[bytes, str]: For the filesystem branch, the file content and filename. + HttpResponseRedirect: For the S3 branch on success, a 302 redirect to a presigned `GetObject` URL. + Response: For any error path, a DRF `Response` with an appropriate status and detail. """ if s3: try: @@ -1988,25 +2158,45 @@ class ScanViewSet(BaseRLSViewSet): # path_pattern here is prefix, but in compliance we build correct suffix check before key = keys[0] else: - # path_pattern is exact key + # path_pattern is exact key; HEAD before presigning to preserve the 404 contract. key = path_pattern - try: - s3_obj = client.get_object(Bucket=bucket, Key=key) - except ClientError as e: - code = e.response.get("Error", {}).get("Code") - if code == "NoSuchKey": + try: + client.head_object(Bucket=bucket, Key=key) + except ClientError as e: + code = e.response.get("Error", {}).get("Code") + if code in ("NoSuchKey", "404"): + return Response( + { + "detail": "The scan has no reports, or the report generation task has not started yet." + }, + status=status.HTTP_404_NOT_FOUND, + ) return Response( - { - "detail": "The scan has no reports, or the report generation task has not started yet." - }, - status=status.HTTP_404_NOT_FOUND, + {"detail": "There is a problem with credentials."}, + status=status.HTTP_403_FORBIDDEN, ) - return Response( - {"detail": "There is a problem with credentials."}, - status=status.HTTP_403_FORBIDDEN, - ) - content = s3_obj["Body"].read() + filename = os.path.basename(key) + # escape quotes and strip CR/LF so a malformed key cannot break out of the header + safe_filename = ( + filename.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\r", "") + .replace("\n", "") + ) + params = { + "Bucket": bucket, + "Key": key, + "ResponseContentDisposition": f'attachment; filename="{safe_filename}"', + } + if content_type: + params["ResponseContentType"] = content_type + url = client.generate_presigned_url( + "get_object", + Params=params, + ExpiresIn=300, + ) + return HttpResponseRedirect(url) else: files = glob.glob(path_pattern) if not files: @@ -2049,12 +2239,16 @@ class ScanViewSet(BaseRLSViewSet): bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "") key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/") loader = self._load_file( - key_prefix, s3=True, bucket=bucket, list_objects=False + key_prefix, + s3=True, + bucket=bucket, + list_objects=False, + content_type="application/x-zip-compressed", ) else: loader = self._load_file(scan.output_location, s3=False) - if isinstance(loader, Response): + if isinstance(loader, HttpResponseBase): return loader content, filename = loader @@ -2092,18 +2286,69 @@ class ScanViewSet(BaseRLSViewSet): prefix = os.path.join( os.path.dirname(key_prefix), "compliance", f"{name}.csv" ) - loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True) + loader = self._load_file( + prefix, + s3=True, + bucket=bucket, + list_objects=True, + content_type="text/csv", + ) else: base = os.path.dirname(scan.output_location) pattern = os.path.join(base, "compliance", f"*_{name}.csv") loader = self._load_file(pattern, s3=False) - if isinstance(loader, Response): + if isinstance(loader, HttpResponseBase): return loader content, filename = loader return self._serve_file(content, filename, "text/csv") + @action( + detail=True, + methods=["get"], + url_name="cis", + ) + def cis(self, request, pk=None): + scan = self.get_object() + running_resp = self._get_task_status(scan) + if running_resp: + return running_resp + + if not scan.output_location: + return Response( + { + "detail": "The scan has no reports, or the CIS report generation task has not started yet." + }, + status=status.HTTP_404_NOT_FOUND, + ) + + if scan.output_location.startswith("s3://"): + bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "") + key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/") + prefix = os.path.join( + os.path.dirname(key_prefix), + "cis", + "*_cis_report.pdf", + ) + loader = self._load_file( + prefix, + s3=True, + bucket=bucket, + list_objects=True, + content_type="application/pdf", + ) + else: + base = os.path.dirname(scan.output_location) + pattern = os.path.join(base, "cis", "*_cis_report.pdf") + loader = self._load_file(pattern, s3=False) + + if isinstance(loader, HttpResponseBase): + return loader + + content, filename = loader + return self._serve_file(content, filename, "application/pdf") + @action( detail=True, methods=["get"], @@ -2132,13 +2377,19 @@ class ScanViewSet(BaseRLSViewSet): "threatscore", "*_threatscore_report.pdf", ) - loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True) + loader = self._load_file( + prefix, + s3=True, + bucket=bucket, + list_objects=True, + content_type="application/pdf", + ) else: base = os.path.dirname(scan.output_location) pattern = os.path.join(base, "threatscore", "*_threatscore_report.pdf") loader = self._load_file(pattern, s3=False) - if isinstance(loader, Response): + if isinstance(loader, HttpResponseBase): return loader content, filename = loader @@ -2172,13 +2423,19 @@ class ScanViewSet(BaseRLSViewSet): "ens", "*_ens_report.pdf", ) - loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True) + loader = self._load_file( + prefix, + s3=True, + bucket=bucket, + list_objects=True, + content_type="application/pdf", + ) else: base = os.path.dirname(scan.output_location) pattern = os.path.join(base, "ens", "*_ens_report.pdf") loader = self._load_file(pattern, s3=False) - if isinstance(loader, Response): + if isinstance(loader, HttpResponseBase): return loader content, filename = loader @@ -2211,13 +2468,19 @@ class ScanViewSet(BaseRLSViewSet): "nis2", "*_nis2_report.pdf", ) - loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True) + loader = self._load_file( + prefix, + s3=True, + bucket=bucket, + list_objects=True, + content_type="application/pdf", + ) else: base = os.path.dirname(scan.output_location) pattern = os.path.join(base, "nis2", "*_nis2_report.pdf") loader = self._load_file(pattern, s3=False) - if isinstance(loader, Response): + if isinstance(loader, HttpResponseBase): return loader content, filename = loader @@ -2250,13 +2513,19 @@ class ScanViewSet(BaseRLSViewSet): "csa", "*_csa_report.pdf", ) - loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True) + loader = self._load_file( + prefix, + s3=True, + bucket=bucket, + list_objects=True, + content_type="application/pdf", + ) else: base = os.path.dirname(scan.output_location) pattern = os.path.join(base, "csa", "*_csa_report.pdf") loader = self._load_file(pattern, s3=False) - if isinstance(loader, Response): + if isinstance(loader, HttpResponseBase): return loader content, filename = loader @@ -2477,7 +2746,7 @@ class AttackPathsScanViewSet(BaseRLSViewSet): return super().get_serializer_class() def get_queryset(self): - user_roles = get_role(self.request.user) + user_roles = get_role(self.request.user, self.request.tenant_id) base_queryset = AttackPathsScan.objects.filter(tenant_id=self.request.tenant_id) if user_roles.unlimited_visibility: @@ -2584,7 +2853,6 @@ class AttackPathsScanViewSet(BaseRLSViewSet): provider_id, ) query_duration = time.monotonic() - start - graph_database.clear_cache(database_name) result_nodes = len(graph.get("nodes", [])) result_relationships = len(graph.get("relationships", [])) @@ -2652,7 +2920,6 @@ class AttackPathsScanViewSet(BaseRLSViewSet): provider_id, ) query_duration = time.monotonic() - start - graph_database.clear_cache(database_name) query_length = len(serializer.validated_data["query"]) result_nodes = len(graph.get("nodes", [])) @@ -2814,7 +3081,7 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet): required_permissions = [] def get_queryset(self): - user_roles = get_role(self.request.user) + user_roles = get_role(self.request.user, self.request.tenant_id) if user_roles.unlimited_visibility: # User has unlimited visibility, return all scans queryset = Resource.all_objects.filter(tenant_id=self.request.tenant_id) @@ -3436,7 +3703,7 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): def get_queryset(self): tenant_id = self.request.tenant_id - user_roles = get_role(self.request.user) + user_roles = get_role(self.request.user, self.request.tenant_id) if user_roles.unlimited_visibility: # User has unlimited visibility, return all findings queryset = Finding.all_objects.filter(tenant_id=tenant_id) @@ -3468,7 +3735,7 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): request, filtered_queryset, manager=Finding.all_objects, - select_related=["scan"], + select_related=["scan__provider"], prefetch_related=["resources"], ) @@ -3638,7 +3905,7 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): tenant_id = request.tenant_id filtered_queryset = self.filter_queryset(self.get_queryset()) - latest_scan_ids = ( + latest_scan_ids = list( Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) .order_by("provider_id", "-inserted_at") .distinct("provider_id") @@ -3652,7 +3919,7 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): request, filtered_queryset, manager=Finding.all_objects, - select_related=["scan"], + select_related=["scan__provider"], prefetch_related=["resources"], ) @@ -4039,9 +4306,9 @@ class RoleViewSet(BaseRLSViewSet): ) ) def partial_update(self, request, *args, **kwargs): - user_role = get_role(request.user) + user_role = get_role(request.user, request.tenant_id) # If the user is the owner of the role, the manage_account field is not editable - if user_role and kwargs["pk"] == str(user_role.id): + if kwargs["pk"] == str(user_role.id): request.data["manage_account"] = str(user_role.manage_account).lower() return super().partial_update(request, *args, **kwargs) @@ -4297,7 +4564,7 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): required_permissions = [] def get_queryset(self): - role = get_role(self.request.user) + role = get_role(self.request.user, self.request.tenant_id) unlimited_visibility = getattr( role, Permissions.UNLIMITED_VISIBILITY.value, False ) @@ -4339,7 +4606,7 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): def _compliance_summaries_queryset(self, scan_id): """Return pre-aggregated summaries constrained by RBAC visibility.""" - role = get_role(self.request.user) + role = get_role(self.request.user, self.request.tenant_id) unlimited_visibility = getattr( role, Permissions.UNLIMITED_VISIBILITY.value, False ) @@ -4881,7 +5148,7 @@ class OverviewViewSet(BaseRLSViewSet): required_permissions = [] def get_queryset(self): - role = get_role(self.request.user) + role = get_role(self.request.user, self.request.tenant_id) providers = get_providers(role) if not role.unlimited_visibility: @@ -6054,7 +6321,7 @@ class IntegrationViewSet(BaseRLSViewSet): allowed_providers = None def get_queryset(self): - user_roles = get_role(self.request.user) + user_roles = get_role(self.request.user, self.request.tenant_id) if user_roles.unlimited_visibility: # User has unlimited visibility, return all integrations queryset = Integration.objects.filter(tenant_id=self.request.tenant_id) @@ -6109,7 +6376,15 @@ class IntegrationViewSet(BaseRLSViewSet): tags=["Integration"], summary="Send findings to a Jira integration", description="Send a set of filtered findings to the given integration. At least one finding filter must be " - "provided.", + "provided.\n\n" + "## Known Limitations\n\n" + "### Issue Types with Required Custom Fields\n\n" + "Certain Jira issue types (such as Epic) may require mandatory custom fields that Prowler does not " + "currently populate when creating work items. If a selected issue type enforces required fields beyond " + 'the standard set (e.g., "Team", "Epic Name"), the work item creation will fail.\n\n' + "To avoid this, select an issue type that does not require additional custom fields - **Task**, **Bug**, " + "or **Story** typically work without restrictions. If unsure which issue types are available for a project, " + 'Prowler automatically fetches and displays them in the "Issue Type" selector when sending a finding.', responses={202: OpenApiResponse(response=TaskSerializer)}, filters=True, ) @@ -6117,7 +6392,7 @@ class IntegrationViewSet(BaseRLSViewSet): class IntegrationJiraViewSet(BaseRLSViewSet): queryset = Finding.all_objects.all() serializer_class = IntegrationJiraDispatchSerializer - http_method_names = ["post"] + http_method_names = ["get", "post"] filter_backends = [CustomDjangoFilterBackend] filterset_class = IntegrationJiraFindingsFilter # RBAC required permissions @@ -6127,9 +6402,27 @@ class IntegrationJiraViewSet(BaseRLSViewSet): def create(self, request, *args, **kwargs): raise MethodNotAllowed(method="POST") + @extend_schema(exclude=True) + def list(self, request, *args, **kwargs): + raise MethodNotAllowed(method="GET") + + @extend_schema(exclude=True) + def retrieve(self, request, *args, **kwargs): + raise MethodNotAllowed(method="GET") + + def get_serializer_class(self): + if self.action == "issue_types": + return IntegrationJiraIssueTypesSerializer + return super().get_serializer_class() + + def get_filter_backends(self): + if self.action == "issue_types": + return [] + return super().get_filter_backends() + def get_queryset(self): tenant_id = self.request.tenant_id - user_roles = get_role(self.request.user) + user_roles = get_role(self.request.user, self.request.tenant_id) if user_roles.unlimited_visibility: # User has unlimited visibility, return all findings queryset = Finding.all_objects.filter(tenant_id=tenant_id) @@ -6141,6 +6434,65 @@ class IntegrationJiraViewSet(BaseRLSViewSet): return queryset + @extend_schema( + tags=["Integration"], + summary="Get available issue types for a Jira project", + description="Fetch the available issue types from Jira for a given project key and update the integration configuration.", + parameters=[ + OpenApiParameter( + name="project_key", + type=str, + location=OpenApiParameter.QUERY, + required=True, + description="The Jira project key to fetch issue types for.", + ), + ], + ) + @action(detail=False, methods=["get"], url_name="issue-types") + def issue_types(self, request, integration_pk=None): + integration = get_object_or_404(Integration, pk=integration_pk) + + project_key = request.query_params.get("project_key") + if not project_key: + raise ValidationError({"project_key": "This query parameter is required."}) + + projects = integration.configuration.get("projects", {}) + if project_key not in projects: + raise ValidationError( + { + "project_key": "The given project key is not available for this JIRA integration." + } + ) + + try: + jira = initialize_prowler_integration(integration) + fetched_issue_types = jira.get_available_issue_types(project_key) + except Exception as e: + logger.error( + f"Failed to fetch issue types from Jira for integration {integration_pk}, " + f"project {project_key}: {e}" + ) + raise ValidationError( + { + "issue_types": "Failed to fetch issue types from Jira. Please check the integration connection." + } + ) + + # Update the integration configuration with the fetched issue types + issue_types_config = integration.configuration.get("issue_types", {}) + if not isinstance(issue_types_config, dict): + issue_types_config = {} + issue_types_config[project_key] = fetched_issue_types + + with rls_transaction(str(integration.tenant_id), using="default"): + integration.configuration["issue_types"] = issue_types_config + integration.save(using="default") + + serializer = IntegrationJiraIssueTypesSerializer( + {"project_key": project_key, "issue_types": fetched_issue_types} + ) + return Response(data=serializer.data, status=status.HTTP_200_OK) + @action(detail=False, methods=["post"], url_name="dispatches") def dispatches(self, request, integration_pk=None): get_object_or_404(Integration, pk=integration_pk) @@ -6725,11 +7077,18 @@ class MuteRuleViewSet(BaseRLSViewSet): muted_reason=mute_rule.reason, ) - # Launch background task for historical muting - with transaction.atomic(): - mute_historical_findings_task.apply_async( - kwargs={"tenant_id": tenant_id, "mute_rule_id": str(mute_rule.id)} - ) + # Launch background task for historical muting + reaggregation + transaction.on_commit( + lambda: chain( + mute_historical_findings_task.si( + tenant_id=tenant_id, + mute_rule_id=str(mute_rule.id), + ), + reaggregate_all_finding_group_summaries_task.si( + tenant_id=tenant_id, + ), + ).apply_async() + ) # Return the created mute rule serializer = self.get_serializer(mute_rule) @@ -6770,26 +7129,42 @@ class FindingGroupViewSet(BaseRLSViewSet): security analysts to see which checks are failing across their infrastructure without scrolling through thousands of individual findings. - Uses pre-aggregated FindingGroupDailySummary table for efficient queries. - Daily summaries are re-aggregated across the requested date range. + Uses a hybrid strategy: pre-aggregated daily summaries when possible, + and raw findings when finding-level filters require precise subset metrics. """ queryset = FindingGroupDailySummary.objects.all() serializer_class = FindingGroupSerializer - filterset_class = FindingGroupSummaryFilter + filterset_class = FindingGroupFilter + filter_backends = [ + jsonapi_filters.QueryParameterValidationFilter, + jsonapi_filters.OrderingFilter, + CustomDjangoFilterBackend, + ] http_method_names = ["get"] required_permissions = [] def get_filterset_class(self): - """Return appropriate filter based on action.""" + """Return the filterset class used for schema generation and the list action. + + Note: The resources and latest_resources actions do not use this method + at runtime. They manually instantiate FindingGroupFilter / + LatestFindingGroupFilter against a Finding queryset (see + _get_finding_queryset). The class returned here for those actions only + affects the OpenAPI schema generated by drf-spectacular. + """ if self.action == "latest": - return LatestFindingGroupSummaryFilter - return FindingGroupSummaryFilter + return LatestFindingGroupFilter + if self.action == "resources": + return FindingGroupFilter + if self.action == "latest_resources": + return LatestFindingGroupFilter + return FindingGroupFilter def get_queryset(self): """Get the base FindingGroupDailySummary queryset with RLS filtering.""" tenant_id = self.request.tenant_id - role = get_role(self.request.user) + role = get_role(self.request.user, self.request.tenant_id) queryset = FindingGroupDailySummary.objects.filter(tenant_id=tenant_id) if not role.unlimited_visibility: @@ -6799,7 +7174,7 @@ class FindingGroupViewSet(BaseRLSViewSet): def _get_finding_queryset(self): """Get the Finding queryset for resources drill-down (with RBAC).""" - role = get_role(self.request.user) + role = get_role(self.request.user, self.request.tenant_id) providers = get_providers(role) tenant_id = self.request.tenant_id @@ -6840,8 +7215,15 @@ class FindingGroupViewSet(BaseRLSViewSet): "resource_type__icontains": "type__icontains", } + # Fields accepted directly by LatestResourceFilter (no translation needed) + _RESOURCE_FILTER_FIELDS = { + f"{field}__{lookup}" + for field, lookups in LatestResourceFilter.Meta.fields.items() + for lookup in lookups + } | set(LatestResourceFilter.Meta.fields.keys()) + def _split_resource_filters(self, params: QueryDict) -> tuple[QueryDict, QueryDict]: - resource_keys = set(self.RESOURCE_FILTER_MAP) + resource_keys = set(self.RESOURCE_FILTER_MAP) | self._RESOURCE_FILTER_FIELDS finding_params = QueryDict(mutable=True) resource_params = QueryDict(mutable=True) for key, values in params.lists(): @@ -6862,11 +7244,16 @@ class FindingGroupViewSet(BaseRLSViewSet): queryset = queryset.filter(tenant_id=tenant_id) filter_params = QueryDict(mutable=True) - for key, mapped_key in self.RESOURCE_FILTER_MAP.items(): - if key not in params: + for key, values in params.lists(): + # Translate resource_* prefixed keys via the map + if key in self.RESOURCE_FILTER_MAP: + mapped_key = self.RESOURCE_FILTER_MAP[key] + elif key in self._RESOURCE_FILTER_FIELDS: + mapped_key = key + else: continue + if key == "resources" or key.endswith("__in"): - values = params.getlist(key) items: list[str] = [] for value in values: if value is None: @@ -6891,49 +7278,210 @@ class FindingGroupViewSet(BaseRLSViewSet): return filterset.qs.values("id") + def _get_finding_level_filter_keys(self, latest: bool = False) -> set[str]: + """Derive filters that require querying raw findings.""" + summary_filterset = ( + LatestFindingGroupSummaryFilter if latest else FindingGroupSummaryFilter + ) + finding_filterset = LatestFindingGroupFilter if latest else FindingGroupFilter + + summary_supported = set(summary_filterset.base_filters.keys()) + finding_supported = set(finding_filterset.base_filters.keys()) + return finding_supported - summary_supported + + def _requires_finding_level_aggregation( + self, params: QueryDict, latest: bool = False + ) -> bool: + finding_level_keys = self._get_finding_level_filter_keys(latest=latest) + return any(key in finding_level_keys for key in params.keys()) + def _aggregate_daily_summaries(self, queryset): - """ - Re-aggregate daily summaries across the date range. - - Takes pre-computed daily summaries and aggregates them by check_id - to produce totals across the selected date range. - """ - from django.db.models import CharField - from django.db.models.functions import Cast - + """Re-aggregate summary rows by check_id.""" return queryset.values("check_id").annotate( - # Max severity across days severity_order=Max("severity_order"), - # Sum counts across days pass_count=Sum("pass_count"), fail_count=Sum("fail_count"), + manual_count=Sum("manual_count"), + pass_muted_count=Sum("pass_muted_count"), + fail_muted_count=Sum("fail_muted_count"), + manual_muted_count=Sum("manual_muted_count"), muted_count=Sum("muted_count"), + # The group is muted only if every contributing daily summary is + # itself fully muted. BoolAnd returns False as soon as one row has + # at least one actionable finding. + muted=BoolAnd("muted"), new_count=Sum("new_count"), changed_count=Sum("changed_count"), + new_fail_count=Sum("new_fail_count"), + new_fail_muted_count=Sum("new_fail_muted_count"), + new_pass_count=Sum("new_pass_count"), + new_pass_muted_count=Sum("new_pass_muted_count"), + new_manual_count=Sum("new_manual_count"), + new_manual_muted_count=Sum("new_manual_muted_count"), + changed_fail_count=Sum("changed_fail_count"), + changed_fail_muted_count=Sum("changed_fail_muted_count"), + changed_pass_count=Sum("changed_pass_count"), + changed_pass_muted_count=Sum("changed_pass_muted_count"), + changed_manual_count=Sum("changed_manual_count"), + changed_manual_muted_count=Sum("changed_manual_muted_count"), resources_total=Sum("resources_total"), resources_fail=Sum("resources_fail"), - # Collect provider types using StringAgg (cast enum to text first) impacted_providers_str=StringAgg( Cast("provider__provider", CharField()), delimiter=",", distinct=True, default="", ), - # Min/Max timing across days - first_seen_at=Min("first_seen_at"), - last_seen_at=Max("last_seen_at"), - failing_since=Min("failing_since"), - # Get check metadata from first row (same for all days) + agg_first_seen_at=Min("first_seen_at"), + agg_last_seen_at=Max("last_seen_at"), + agg_failing_since=Min("failing_since"), check_title=Max("check_title"), check_description=Max("check_description"), ) + def _aggregate_findings(self, queryset): + """Aggregate findings by check_id for finding-group endpoints.""" + severity_case = Case( + *[ + When(severity=severity, then=Value(order)) + for severity, order in SEVERITY_ORDER.items() + ], + output_field=IntegerField(), + ) + + # `pass_count`, `fail_count` and `manual_count` only count non-muted + # findings. Muted findings are tracked separately via the + # `*_muted_count` fields. + return ( + queryset.values("check_id") + .annotate( + severity_order=Max(severity_case), + pass_count=Count("id", filter=Q(status="PASS", muted=False)), + fail_count=Count("id", filter=Q(status="FAIL", muted=False)), + manual_count=Count("id", filter=Q(status="MANUAL", muted=False)), + pass_muted_count=Count("id", filter=Q(status="PASS", muted=True)), + fail_muted_count=Count("id", filter=Q(status="FAIL", muted=True)), + manual_muted_count=Count("id", filter=Q(status="MANUAL", muted=True)), + muted_count=Count("id", filter=Q(muted=True)), + nonmuted_count=Count("id", filter=Q(muted=False)), + new_count=Count("id", filter=Q(delta="new", muted=False)), + changed_count=Count("id", filter=Q(delta="changed", muted=False)), + new_fail_count=Count( + "id", filter=Q(delta="new", status="FAIL", muted=False) + ), + new_fail_muted_count=Count( + "id", filter=Q(delta="new", status="FAIL", muted=True) + ), + new_pass_count=Count( + "id", filter=Q(delta="new", status="PASS", muted=False) + ), + new_pass_muted_count=Count( + "id", filter=Q(delta="new", status="PASS", muted=True) + ), + new_manual_count=Count( + "id", filter=Q(delta="new", status="MANUAL", muted=False) + ), + new_manual_muted_count=Count( + "id", filter=Q(delta="new", status="MANUAL", muted=True) + ), + changed_fail_count=Count( + "id", filter=Q(delta="changed", status="FAIL", muted=False) + ), + changed_fail_muted_count=Count( + "id", filter=Q(delta="changed", status="FAIL", muted=True) + ), + changed_pass_count=Count( + "id", filter=Q(delta="changed", status="PASS", muted=False) + ), + changed_pass_muted_count=Count( + "id", filter=Q(delta="changed", status="PASS", muted=True) + ), + changed_manual_count=Count( + "id", filter=Q(delta="changed", status="MANUAL", muted=False) + ), + changed_manual_muted_count=Count( + "id", filter=Q(delta="changed", status="MANUAL", muted=True) + ), + resources_total=Count("resources__id", distinct=True), + resources_fail=Count( + "resources__id", + distinct=True, + filter=Q(status="FAIL", muted=False), + ), + impacted_providers_str=StringAgg( + Cast("scan__provider__provider", CharField()), + delimiter=",", + distinct=True, + default="", + ), + agg_first_seen_at=Min("first_seen_at"), + agg_last_seen_at=Max("inserted_at"), + agg_failing_since=Min( + "first_seen_at", filter=Q(status="FAIL", muted=False) + ), + check_title=Coalesce( + Max(KeyTextTransform("checktitle", "check_metadata")), + Max(KeyTextTransform("CheckTitle", "check_metadata")), + Max(KeyTextTransform("Checktitle", "check_metadata")), + ), + check_description=Coalesce( + Max(KeyTextTransform("description", "check_metadata")), + Max(KeyTextTransform("Description", "check_metadata")), + ), + ) + .annotate( + # Group is muted only if it has zero non-muted findings. + muted=Case( + When(nonmuted_count=0, then=Value(True)), + default=Value(False), + output_field=BooleanField(), + ), + ) + ) + + def _split_computed_aggregate_filters( + self, params: QueryDict + ) -> tuple[QueryDict, QueryDict]: + """Split finding filters from computed aggregate filters.""" + computed_keys = { + "status", + "status__in", + "severity", + "severity__in", + "muted", + "include_muted", + } + finding_params = QueryDict(mutable=True) + computed_params = QueryDict(mutable=True) + + for key, values in params.lists(): + if key in computed_keys: + computed_params.setlist(key, values) + else: + finding_params.setlist(key, values) + + return finding_params, computed_params + + def _get_latest_findings_per_provider(self, filtered_queryset): + """Keep only findings from each provider's most recent completed scan.""" + latest_scan_ids = ( + Scan.objects.filter( + tenant_id=self.request.tenant_id, + state=StateChoices.COMPLETED, + ) + .order_by("provider_id", "-completed_at", "-inserted_at") + .distinct("provider_id") + .values("id") + ) + return filtered_queryset.filter(scan_id__in=latest_scan_ids) + def _post_process_aggregation(self, aggregated_data): """ Post-process aggregation results to add computed fields. - Converts severity integer back to string - - Computes aggregated status (FAIL > PASS > MUTED) + - Computes aggregated status (FAIL > PASS > MANUAL); the orthogonal + ``muted`` boolean is already on the row from the SQL aggregation - Converts provider string to list """ results = [] @@ -6944,13 +7492,32 @@ class FindingGroupViewSet(BaseRLSViewSet): severity_order, "informational" ) - # Compute aggregated status + if "agg_first_seen_at" in row: + row["first_seen_at"] = row.pop("agg_first_seen_at") + if "agg_last_seen_at" in row: + row["last_seen_at"] = row.pop("agg_last_seen_at") + if "agg_failing_since" in row: + row["failing_since"] = row.pop("agg_failing_since") + + # Drop the helper count we use to derive `muted` in the + # finding-level aggregation path. + row.pop("nonmuted_count", None) + + # Muted findings are treated as resolved/accepted, so they do not + # contribute to a failing status. A group is FAIL only when there + # is at least one non-muted FAIL; otherwise any pass (muted or + # not) or any muted fail makes the group PASS. Only groups whose + # findings are exclusively MANUAL fall through to MANUAL. if row.get("fail_count", 0) > 0: row["status"] = "FAIL" - elif row.get("pass_count", 0) > 0: + elif ( + row.get("pass_count", 0) > 0 + or row.get("pass_muted_count", 0) > 0 + or row.get("fail_muted_count", 0) > 0 + ): row["status"] = "PASS" else: - row["status"] = "MUTED" + row["status"] = "MANUAL" # Convert provider string to list providers_str = row.pop("impacted_providers_str", "") or "" @@ -6962,22 +7529,60 @@ class FindingGroupViewSet(BaseRLSViewSet): return results - def _validate_sort_fields(self, sort_param): - """Validate and map JSON:API sort fields for aggregated finding groups.""" - sort_field_map = { - "check_id": "check_id", - "severity": "severity_order", - "fail_count": "fail_count", - "pass_count": "pass_count", - "muted_count": "muted_count", - "new_count": "new_count", - "changed_count": "changed_count", - "resources_total": "resources_total", - "resources_fail": "resources_fail", - "first_seen_at": "first_seen_at", - "last_seen_at": "last_seen_at", - "failing_since": "failing_since", - } + _FINDING_GROUP_SORT_MAP = { + "check_id": "check_id", + "check_title": "check_title", + "severity": "severity_order", + "status": "status_order", + "muted": "muted", + "delta": "delta_order", + "fail_count": "fail_count", + "pass_count": "pass_count", + "manual_count": "manual_count", + "muted_count": "muted_count", + "pass_muted_count": "pass_muted_count", + "fail_muted_count": "fail_muted_count", + "manual_muted_count": "manual_muted_count", + "new_count": "new_count", + "new_fail_count": "new_fail_count", + "new_fail_muted_count": "new_fail_muted_count", + "new_pass_count": "new_pass_count", + "new_pass_muted_count": "new_pass_muted_count", + "new_manual_count": "new_manual_count", + "new_manual_muted_count": "new_manual_muted_count", + "changed_count": "changed_count", + "changed_fail_count": "changed_fail_count", + "changed_fail_muted_count": "changed_fail_muted_count", + "changed_pass_count": "changed_pass_count", + "changed_pass_muted_count": "changed_pass_muted_count", + "changed_manual_count": "changed_manual_count", + "changed_manual_muted_count": "changed_manual_muted_count", + "resources_total": "resources_total", + "resources_fail": "resources_fail", + "first_seen_at": "agg_first_seen_at", + "last_seen_at": "agg_last_seen_at", + "failing_since": "agg_failing_since", + } + + _RESOURCE_SORT_MAP = { + "status": "status_order", + "severity": "severity_order", + "delta": "delta_order", + "first_seen_at": "first_seen_at", + "last_seen_at": "last_seen_at", + "resource.uid": "resource_uid", + "resource.name": "resource_name", + "resource.region": "resource_region", + "resource.service": "resource_service", + "resource.type": "resource_type", + "provider.uid": "provider_uid", + "provider.alias": "provider_alias", + } + + def _validate_sort_fields(self, sort_param, sort_field_map=None): + """Validate and map JSON:API sort fields using the given field map.""" + if sort_field_map is None: + sort_field_map = self._FINDING_GROUP_SORT_MAP ordering = [] for field in sort_param.split(","): @@ -6987,7 +7592,6 @@ class FindingGroupViewSet(BaseRLSViewSet): is_desc = field.startswith("-") raw_field = field[1:] if is_desc else field if raw_field not in sort_field_map: - # Validate sort fields explicitly to return JSON:API 400 instead of FieldError. raise ValidationError( [ { @@ -7003,6 +7607,56 @@ class FindingGroupViewSet(BaseRLSViewSet): return ordering + def _apply_aggregated_computed_filters(self, queryset, computed_params: QueryDict): + """Apply computed filters (status/severity/muted) on aggregated finding-group rows.""" + if not computed_params: + return queryset + + if computed_params.get("status") or computed_params.getlist("status__in"): + queryset = queryset.annotate( + aggregated_status=Case( + When(fail_count__gt=0, then=Value("FAIL")), + When(pass_count__gt=0, then=Value("PASS")), + When(pass_muted_count__gt=0, then=Value("PASS")), + When(fail_muted_count__gt=0, then=Value("PASS")), + default=Value("MANUAL"), + output_field=CharField(), + ) + ) + + # Exclude fully-muted groups by default unless the caller has opted in + # via either `include_muted` or an explicit `muted` filter (the latter + # gives the caller direct control over the column). + if "include_muted" not in computed_params and "muted" not in computed_params: + queryset = queryset.exclude(muted=True) + + filterset = FindingGroupAggregatedComputedFilter( + computed_params, queryset=queryset + ) + if not filterset.is_valid(): + raise ValidationError(filterset.errors) + + return filterset.qs + + def _resolve_finding_ids(self, filtered_queryset): + """ + Materialize and request-cache the finding_ids list used to anchor + RFM lookups. + + Turning `finding_id__in=Subquery(findings_qs)` into `finding_id__in= + [uuid, ...]` nudges PostgreSQL out of a Merge Semi Join that ends up + reading hundreds of thousands of RFM index entries just to post- + filter tenant_id. Caching on the ViewSet instance (one instance per + request) avoids duplicating the findings round-trip when several + helpers build different RFM querysets from the same filtered set. + """ + cached = getattr(self, "_finding_ids_cache", None) + if cached is not None and cached[0] is filtered_queryset: + return cached[1] + finding_ids = list(filtered_queryset.order_by().values_list("id", flat=True)) + self._finding_ids_cache = (filtered_queryset, finding_ids) + return finding_ids + def _build_resource_mapping_queryset( self, filtered_queryset, resource_ids=None, tenant_id: str | None = None ): @@ -7012,10 +7666,10 @@ class FindingGroupViewSet(BaseRLSViewSet): Starting from ResourceFindingMapping avoids scanning all mappings before applying check_id/date filters on findings. """ - finding_ids = filtered_queryset.order_by().values("id") + finding_ids = self._resolve_finding_ids(filtered_queryset) mapping_queryset = ResourceFindingMapping.objects.filter( - finding_id__in=Subquery(finding_ids) + finding_id__in=finding_ids ) if tenant_id: mapping_queryset = mapping_queryset.filter(tenant_id=tenant_id) @@ -7048,18 +7702,14 @@ class FindingGroupViewSet(BaseRLSViewSet): provider_type=Max("resource__provider__provider"), provider_uid=Max("resource__provider__uid"), provider_alias=Max("resource__provider__alias"), + # status_order considers ALL findings (muted or not) so it + # surfaces FAIL/PASS/MANUAL based on the underlying check + # outcome. Whether the resource is actionable is signalled by + # the orthogonal `muted` flag below. status_order=Max( Case( - When( - finding__status="FAIL", - finding__muted=False, - then=Value(3), - ), - When( - finding__status="PASS", - finding__muted=False, - then=Value(2), - ), + When(finding__status="FAIL", then=Value(3)), + When(finding__status="PASS", then=Value(2)), default=Value(1), output_field=IntegerField(), ) @@ -7073,11 +7723,156 @@ class FindingGroupViewSet(BaseRLSViewSet): output_field=IntegerField(), ) ), + delta_order=Max( + Case( + When( + finding__delta="new", + finding__muted=False, + then=Value(2), + ), + When( + finding__delta="changed", + finding__muted=False, + then=Value(1), + ), + default=Value(0), + output_field=IntegerField(), + ) + ), first_seen_at=Min("finding__first_seen_at"), last_seen_at=Max("finding__inserted_at"), + # True only if every finding for this resource+check is muted. + muted=BoolAnd("finding__muted"), + # Max() on muted_reason / check_metadata is safe because + # all findings for the same resource+check share identical + # values (mute rules and metadata are applied per-check). + muted_reason=Max("finding__muted_reason"), + resource_group=Max( + KeyTextTransform("resourcegroup", "finding__check_metadata") + ), + # Most recent matching Finding for this (resource, check): + # Finding.id is a UUIDv7 (time-ordered in its high 48 bits). + # Cast to text first because PostgreSQL has no built-in + # `max(uuid)` aggregate; on the canonical lowercase form a + # lexicographic Max() still resolves to the latest snapshot. + finding_id=Max(Cast("finding__id", output_field=CharField())), ) .filter(resource_id__isnull=False) - .order_by("resource_id") + ) + + # Annotations needed for each sort field (lightweight versions for ordering) + _RESOURCE_SORT_ANNOTATIONS = { + "status_order": lambda: Max( + Case( + When(finding__status="FAIL", then=Value(3)), + When(finding__status="PASS", then=Value(2)), + default=Value(1), + output_field=IntegerField(), + ) + ), + "severity_order": lambda: Max( + Case( + *[ + When(finding__severity=severity, then=Value(order)) + for severity, order in SEVERITY_ORDER.items() + ], + output_field=IntegerField(), + ) + ), + "delta_order": lambda: Max( + Case( + When( + finding__delta="new", + finding__muted=False, + then=Value(2), + ), + When( + finding__delta="changed", + finding__muted=False, + then=Value(1), + ), + default=Value(0), + output_field=IntegerField(), + ) + ), + "first_seen_at": lambda: Min("finding__first_seen_at"), + "last_seen_at": lambda: Max("finding__inserted_at"), + "resource_uid": lambda: Max("resource__uid"), + "resource_name": lambda: Max("resource__name"), + "resource_region": lambda: Max("resource__region"), + "resource_service": lambda: Max("resource__service"), + "resource_type": lambda: Max("resource__type"), + "provider_uid": lambda: Max("resource__provider__uid"), + "provider_alias": lambda: Max("resource__provider__alias"), + } + + def _build_resource_ordering_queryset( + self, filtered_queryset, resource_ids, tenant_id, ordering + ): + """Build a lightweight aggregation with only the columns needed for sorting.""" + mapping_qs = self._build_resource_mapping_queryset( + filtered_queryset, resource_ids=resource_ids, tenant_id=tenant_id + ) + + # Collect only the annotations required by the requested ordering + annotations = {} + for field in ordering: + col = field.lstrip("-") + if col != "resource_id" and col in self._RESOURCE_SORT_ANNOTATIONS: + annotations[col] = self._RESOURCE_SORT_ANNOTATIONS[col]() + + return ( + mapping_qs.values("resource_id") + .annotate(**annotations) + .filter(resource_id__isnull=False) + .order_by(*ordering) + ) + + def _orphan_findings_queryset(self, filtered_queryset, finding_ids=None): + """Findings in the filtered set with no ResourceFindingMapping entries.""" + orphan_qs = filtered_queryset.filter( + ~Exists(ResourceFindingMapping.objects.filter(finding_id=OuterRef("pk"))) + ) + if finding_ids is not None: + orphan_qs = orphan_qs.filter(id__in=finding_ids) + return orphan_qs + + def _has_orphan_findings(self, filtered_queryset) -> bool: + """Return True if any finding in the filtered set has no resource mapping.""" + return self._orphan_findings_queryset(filtered_queryset).exists() + + def _orphan_aggregation_values(self, orphan_queryset): + """Raw rows for orphan findings; resource payload synthesized from metadata. + + check_metadata is stored with lowercase keys (see + `prowler.lib.outputs.finding.Finding.get_metadata`) and + `Finding.resource_groups` is already denormalized at ingest time. + """ + return orphan_queryset.annotate( + _provider_type=F("scan__provider__provider"), + _provider_uid=F("scan__provider__uid"), + _provider_alias=F("scan__provider__alias"), + _svc=KeyTextTransform("servicename", "check_metadata"), + _region=KeyTextTransform("region", "check_metadata"), + _rtype=KeyTextTransform("resourcetype", "check_metadata"), + _rgroup=F("resource_groups"), + ).values( + "id", + "uid", + "status", + "severity", + "delta", + "muted", + "muted_reason", + "first_seen_at", + "inserted_at", + "_provider_type", + "_provider_uid", + "_provider_alias", + "_svc", + "_region", + "_rtype", + "_rgroup", ) def _post_process_resources(self, resource_data): @@ -7091,11 +7886,23 @@ class FindingGroupViewSet(BaseRLSViewSet): elif status_order == 2: status = "PASS" else: - status = "MUTED" + status = "MANUAL" + + delta_order = row.get("delta_order", 0) + if delta_order == 2: + delta = "new" + elif delta_order == 1: + delta = "changed" + else: + delta = None + + resource_id = row["resource_id"] + finding_id = str(row["finding_id"]) if row.get("finding_id") else None results.append( { - "resource_id": row["resource_id"], + "row_id": resource_id, + "resource_id": resource_id, "resource_uid": row["resource_uid"], "resource_name": row["resource_name"], "resource_service": row["resource_service"], @@ -7108,58 +7915,368 @@ class FindingGroupViewSet(BaseRLSViewSet): "severity": SEVERITY_ORDER_REVERSE.get( severity_order, "informational" ), + "delta": delta, "first_seen_at": row["first_seen_at"], "last_seen_at": row["last_seen_at"], + "muted": bool(row.get("muted", False)), + "muted_reason": row.get("muted_reason"), + "resource_group": row.get("resource_group", ""), + "finding_id": finding_id, } ) return results + def _post_process_orphans(self, orphan_rows): + """Convert orphan finding rows into the same API shape as mapping rows.""" + results = [] + for row in orphan_rows: + status_val = row["status"] + status = status_val if status_val in ("FAIL", "PASS") else "MANUAL" + + muted = bool(row["muted"]) + delta_val = row.get("delta") + delta = delta_val if delta_val in ("new", "changed") and not muted else None + + finding_id = str(row["id"]) + + results.append( + { + "row_id": finding_id, + "resource_id": None, + "resource_uid": row["uid"], + "resource_name": row["uid"], + "resource_service": row["_svc"] or "", + "resource_region": row["_region"] or "", + "resource_type": row["_rtype"] or "", + "provider_type": row["_provider_type"], + "provider_uid": row["_provider_uid"], + "provider_alias": row["_provider_alias"], + "status": status, + "severity": row["severity"], + "delta": delta, + "first_seen_at": row["first_seen_at"], + "last_seen_at": row["inserted_at"], + "muted": muted, + "muted_reason": row.get("muted_reason"), + "resource_group": row["_rgroup"] or "", + "finding_id": finding_id, + } + ) + + return results + + def _build_aggregated_queryset(self, finding_params, latest=False): + """Select the summary or findings path and return an aggregated queryset.""" + finding_filterset_class = ( + LatestFindingGroupFilter if latest else FindingGroupFilter + ) + summary_filterset_class = ( + LatestFindingGroupSummaryFilter if latest else FindingGroupSummaryFilter + ) + + if self._requires_finding_level_aggregation(finding_params, latest=latest): + finding_queryset = self._get_finding_queryset() + filterset = finding_filterset_class( + finding_params, queryset=finding_queryset + ) + if not filterset.is_valid(): + raise ValidationError(filterset.errors) + filtered_queryset = filterset.qs + if latest: + filtered_queryset = self._get_latest_findings_per_provider( + filtered_queryset + ) + return self._aggregate_findings(filtered_queryset) + + summary_queryset = self.get_queryset() + filterset = summary_filterset_class(finding_params, queryset=summary_queryset) + if not filterset.is_valid(): + raise ValidationError(filterset.errors) + filtered_queryset = filterset.qs + # Only include summaries from each provider's most recent date + # (within the filtered range). + # We use a subquery to strip the Window annotation so it does not + # leak into the GROUP BY of _aggregate_daily_summaries. + latest_per_provider = filtered_queryset.annotate( + _max_provider_date=Window( + expression=Max("inserted_at"), + partition_by=[F("provider_id")], + ), + ).filter(inserted_at=F("_max_provider_date")) + clean_queryset = FindingGroupDailySummary.objects.filter( + pk__in=latest_per_provider.values("pk") + ) + return self._aggregate_daily_summaries(clean_queryset) + + def _sorted_paginated_response( + self, + request, + aggregated_queryset, + ): + """Apply ordering, pagination, post-processing, and return the Response.""" + sort_param = request.query_params.get("sort") + if sort_param: + ordering = self._validate_sort_fields( + sort_param, self._FINDING_GROUP_SORT_MAP + ) + if ordering: + if any(field.lstrip("-") == "status_order" for field in ordering): + aggregated_queryset = aggregated_queryset.annotate( + status_order=Case( + When(fail_count__gt=0, then=Value(3)), + When(pass_count__gt=0, then=Value(2)), + When(pass_muted_count__gt=0, then=Value(2)), + When(fail_muted_count__gt=0, then=Value(2)), + default=Value(1), + output_field=IntegerField(), + ) + ) + + expanded_ordering = [] + for field in ordering: + if field.lstrip("-") == "delta_order": + sign = "-" if field.startswith("-") else "" + expanded_ordering.append(f"{sign}new_count") + expanded_ordering.append(f"{sign}changed_count") + else: + expanded_ordering.append(field) + aggregated_queryset = aggregated_queryset.order_by(*expanded_ordering) + else: + aggregated_queryset = aggregated_queryset.order_by( + "-fail_count", "-severity_order", "check_id" + ) + + page = self.paginate_queryset(aggregated_queryset) + if page is not None: + processed_data = self._post_process_aggregation(page) + serializer = self.get_serializer(processed_data, many=True) + return self.get_paginated_response(serializer.data) + + processed_data = self._post_process_aggregation(aggregated_queryset) + serializer = self.get_serializer(processed_data, many=True) + return Response(serializer.data) + + def _validate_resource_sort(self, request): + """Validate the sort parameter for resource endpoints (raises 400 if invalid).""" + sort_param = request.query_params.get("sort") + if sort_param: + self._validate_sort_fields(sort_param, self._RESOURCE_SORT_MAP) + + def _paginated_resource_response( + self, request, filtered_queryset, resource_ids, tenant_id + ): + """Paginate and return resources, appending orphan findings when present. + + Hot path (no orphans, or resource filter applied): resources come from + ResourceFindingMapping aggregation. Untouched pre-existing behaviour. + + Orphan fallback: findings without a mapping (e.g. IaC) are appended + after mapping rows as synthesised resource-like rows so they remain + visible in the UI without paying the aggregation cost on the hot path. + """ + sort_param = request.query_params.get("sort") + ordering = None + if sort_param: + validated = self._validate_sort_fields(sort_param, self._RESOURCE_SORT_MAP) + ordering = validated if validated else None + + # Resource filters can only match findings with resources; skip orphan + # detection entirely when they are present. + if resource_ids is not None: + return self._mapping_paginated_response( + request, filtered_queryset, resource_ids, tenant_id, ordering + ) + + # Serve the mapping response directly and piggyback on the paginator + # count to detect orphan-only groups, instead of paying a separate + # has_mappings.exists() semi-join over ResourceFindingMapping on + # every non-IaC request. TODO: once the ephemeral resources strategy + # is decided, mixed groups should route to _combined_paginated_response. + response = self._mapping_paginated_response( + request, filtered_queryset, resource_ids, tenant_id, ordering + ) + + page = getattr(self.paginator, "page", None) + mapping_total = page.paginator.count if page is not None else None + if mapping_total == 0: + # Pure orphan group (e.g. IaC): synthesize resource-like rows. + return self._combined_paginated_response( + request, filtered_queryset, tenant_id, ordering + ) + + return response + + def _mapping_paginated_response( + self, request, filtered_queryset, resource_ids, tenant_id, ordering + ): + """Mapping-only paginated response (original fast path).""" + if ordering: + if "resource_id" not in {field.lstrip("-") for field in ordering}: + ordering.append("resource_id") + + # Phase 1: lightweight aggregation with only sort keys, paginate + ordering_qs = self._build_resource_ordering_queryset( + filtered_queryset, + resource_ids=resource_ids, + tenant_id=tenant_id, + ordering=ordering, + ) + page = self.paginate_queryset(ordering_qs) + if page is not None: + page_ids = [row["resource_id"] for row in page] + resource_data = self._build_resource_aggregation( + filtered_queryset, resource_ids=page_ids, tenant_id=tenant_id + ) + id_order = {rid: idx for idx, rid in enumerate(page_ids)} + results = self._post_process_resources(resource_data) + results.sort(key=lambda r: id_order.get(r["resource_id"], 0)) + serializer = FindingGroupResourceSerializer(results, many=True) + return self.get_paginated_response(serializer.data) + + page_ids = [row["resource_id"] for row in ordering_qs] + resource_data = self._build_resource_aggregation( + filtered_queryset, resource_ids=page_ids, tenant_id=tenant_id + ) + id_order = {rid: idx for idx, rid in enumerate(page_ids)} + results = self._post_process_resources(resource_data) + results.sort(key=lambda r: id_order.get(r["resource_id"], 0)) + serializer = FindingGroupResourceSerializer(results, many=True) + return Response(serializer.data) + + mapping_qs = self._build_resource_mapping_queryset( + filtered_queryset, resource_ids=resource_ids, tenant_id=tenant_id + ) + resource_id_qs = ( + mapping_qs.values_list("resource_id", flat=True) + .distinct() + .order_by("resource_id") + ) + + page_ids = self.paginate_queryset(resource_id_qs) + if page_ids is not None: + resource_data = self._build_resource_aggregation( + filtered_queryset, resource_ids=page_ids, tenant_id=tenant_id + ) + id_order = {rid: idx for idx, rid in enumerate(page_ids)} + results = self._post_process_resources(resource_data) + results.sort(key=lambda r: id_order.get(r["resource_id"], 0)) + serializer = FindingGroupResourceSerializer(results, many=True) + return self.get_paginated_response(serializer.data) + + resource_data = self._build_resource_aggregation( + filtered_queryset, resource_ids=resource_ids, tenant_id=tenant_id + ).order_by("resource_id") + results = self._post_process_resources(resource_data) + serializer = FindingGroupResourceSerializer(results, many=True) + return Response(serializer.data) + + def _combined_paginated_response( + self, request, filtered_queryset, tenant_id, ordering + ): + """Mapping rows + orphan findings appended at end. + + Orphans sit after mapping rows regardless of sort. This keeps the + mapping-only code path intact for checks that have no orphans (the + common case) and avoids paying UNION/coalesce costs there. + """ + mapping_qs = self._build_resource_mapping_queryset( + filtered_queryset, resource_ids=None, tenant_id=tenant_id + ) + mapping_count = mapping_qs.values("resource_id").distinct().count() + + orphan_ids = list( + self._orphan_findings_queryset(filtered_queryset) + .order_by("id") + .values_list("id", flat=True) + ) + orphan_count = len(orphan_ids) + total = mapping_count + orphan_count + + # Paginate a simple [0..total) index sequence so DRF produces proper + # links/meta; then slice mapping / orphan sources accordingly. + page = self.paginate_queryset(range(total)) + page_indices = list(page) if page is not None else list(range(total)) + + mapping_indices = [i for i in page_indices if i < mapping_count] + orphan_positions = [ + i - mapping_count for i in page_indices if i >= mapping_count + ] + + mapping_results = [] + if mapping_indices: + start = mapping_indices[0] + stop = mapping_indices[-1] + 1 + if ordering: + ordering_fields = list(ordering) + if "resource_id" not in { + field.lstrip("-") for field in ordering_fields + }: + ordering_fields.append("resource_id") + ordered_qs = self._build_resource_ordering_queryset( + filtered_queryset, + resource_ids=None, + tenant_id=tenant_id, + ordering=ordering_fields, + ) + slice_rids = [row["resource_id"] for row in ordered_qs[start:stop]] + else: + slice_rids = list( + mapping_qs.values_list("resource_id", flat=True) + .distinct() + .order_by("resource_id")[start:stop] + ) + if slice_rids: + resource_data = self._build_resource_aggregation( + filtered_queryset, + resource_ids=slice_rids, + tenant_id=tenant_id, + ) + rows_by_rid = {row["resource_id"]: row for row in resource_data} + ordered_rows = [ + rows_by_rid[rid] for rid in slice_rids if rid in rows_by_rid + ] + mapping_results = self._post_process_resources(ordered_rows) + + orphan_results = [] + if orphan_positions: + slice_fids = [orphan_ids[pos] for pos in orphan_positions] + raw_rows = list( + self._orphan_aggregation_values( + self._orphan_findings_queryset( + filtered_queryset, finding_ids=slice_fids + ) + ) + ) + rows_by_fid = {row["id"]: row for row in raw_rows} + ordered_rows = [ + rows_by_fid[fid] for fid in slice_fids if fid in rows_by_fid + ] + orphan_results = self._post_process_orphans(ordered_rows) + + results = mapping_results + orphan_results + serializer = FindingGroupResourceSerializer(results, many=True) + if page is not None: + return self.get_paginated_response(serializer.data) + return Response(serializer.data) + def list(self, request, *args, **kwargs): """ List finding groups with aggregation and filtering. Returns findings grouped by check_id with aggregated metrics. Requires at least one date filter for performance. - Uses pre-aggregated daily summaries for efficient queries. + Uses summaries when possible and raw findings for finding-level filters. """ - queryset = self.get_queryset() - - # Apply filters normalized_params = self._normalize_jsonapi_params(request.query_params) - filterset = self.filterset_class(normalized_params, queryset=queryset) - if not filterset.is_valid(): - raise ValidationError(filterset.errors) - filtered_queryset = filterset.qs - - # Re-aggregate daily summaries across the date range - aggregated_queryset = self._aggregate_daily_summaries(filtered_queryset) - - # Apply ordering (respect JSON:API sort param or use default) - sort_param = request.query_params.get("sort") - if sort_param: - # Convert JSON:API sort notation (prefix '-' for descending) - ordering = self._validate_sort_fields(sort_param) - if ordering: - aggregated_queryset = aggregated_queryset.order_by(*ordering) - else: - # Default ordering: failures first, then severity, then check_id - aggregated_queryset = aggregated_queryset.order_by( - "-fail_count", "-severity_order", "check_id" - ) - - # Paginate - page = self.paginate_queryset(aggregated_queryset) - if page is not None: - # Post-process the page - processed_data = self._post_process_aggregation(page) - serializer = self.get_serializer(processed_data, many=True) - return self.get_paginated_response(serializer.data) - - # Post-process all results (no pagination) - processed_data = self._post_process_aggregation(aggregated_queryset) - serializer = self.get_serializer(processed_data, many=True) - return Response(serializer.data) + finding_params, computed_params = self._split_computed_aggregate_filters( + normalized_params + ) + aggregated_qs = self._build_aggregated_queryset(finding_params, latest=False) + aggregated_qs = self._apply_aggregated_computed_filters( + aggregated_qs, computed_params + ) + return self._sorted_paginated_response(request, aggregated_qs) @extend_schema( summary="List latest finding groups", @@ -7177,56 +8294,22 @@ class FindingGroupViewSet(BaseRLSViewSet): """ List the latest finding group state per check_id. - Returns findings grouped by check_id using the latest available - inserted_at date per check_id, without requiring date filters. + Returns findings grouped by check_id using latest data per + (check_id, provider), without requiring date filters. """ - queryset = self.get_queryset() - - # Apply other filters (provider_id, provider_type, check_id, etc.) normalized_params = self._normalize_jsonapi_params(request.query_params) - # Remove date filters since we're using latest for key in list(normalized_params.keys()): if key.startswith("inserted_at"): del normalized_params[key] - filterset_class = self.get_filterset_class() - filterset = filterset_class(normalized_params, queryset=queryset) - if not filterset.is_valid(): - raise ValidationError(filterset.errors) - filtered_queryset = filterset.qs - - # Keep only rows from the latest inserted_at date per check_id - latest_per_check = filtered_queryset.annotate( - latest_inserted_at=Window( - expression=Max("inserted_at"), - partition_by=[F("check_id")], - ) - ).filter(inserted_at=F("latest_inserted_at")) - - # Re-aggregate daily summaries - aggregated_queryset = self._aggregate_daily_summaries(latest_per_check) - - # Apply ordering - sort_param = request.query_params.get("sort") - if sort_param: - ordering = self._validate_sort_fields(sort_param) - if ordering: - aggregated_queryset = aggregated_queryset.order_by(*ordering) - else: - aggregated_queryset = aggregated_queryset.order_by( - "-fail_count", "-severity_order", "check_id" - ) - - # Paginate - page = self.paginate_queryset(aggregated_queryset) - if page is not None: - processed_data = self._post_process_aggregation(page) - serializer = self.get_serializer(processed_data, many=True) - return self.get_paginated_response(serializer.data) - - processed_data = self._post_process_aggregation(aggregated_queryset) - serializer = self.get_serializer(processed_data, many=True) - return Response(serializer.data) + finding_params, computed_params = self._split_computed_aggregate_filters( + normalized_params + ) + aggregated_qs = self._build_aggregated_queryset(finding_params, latest=True) + aggregated_qs = self._apply_aggregated_computed_filters( + aggregated_qs, computed_params + ) + return self._sorted_paginated_response(request, aggregated_qs) @extend_schema( summary="List resources for a finding group", @@ -7237,6 +8320,7 @@ class FindingGroupViewSet(BaseRLSViewSet): and timing information including how long they have been failing. """, tags=["Finding Groups"], + filters=True, ) @action(detail=True, methods=["get"], url_path="resources") def resources(self, request, pk=None): @@ -7249,57 +8333,33 @@ class FindingGroupViewSet(BaseRLSViewSet): check_id = pk queryset = self._get_finding_queryset() - # Apply date filters from request to Finding queryset + # 1. Normalize and split params normalized_params = self._normalize_jsonapi_params(request.query_params) finding_params, resource_params = self._split_resource_filters( normalized_params ) + # 2. Validate all inputs before any DB existence check filterset = FindingGroupFilter(finding_params, queryset=queryset) if not filterset.is_valid(): raise ValidationError(filterset.errors) + # Access .qs to trigger filter_queryset validation (e.g. required date filters) filtered_queryset = filterset.qs - - # Filter by check_id - filtered_queryset = filtered_queryset.filter(check_id=check_id) - - # Check if any findings exist for this check_id - if not filtered_queryset.exists(): - raise NotFound(f"Finding group '{check_id}' not found.") - resource_ids = self._resource_ids_from_params( resource_params, request.tenant_id ) - mapping_queryset = self._build_resource_mapping_queryset( - filtered_queryset, - resource_ids=resource_ids, - tenant_id=request.tenant_id, - ) - resource_id_queryset = ( - mapping_queryset.values_list("resource_id", flat=True) - .distinct() - .order_by("resource_id") - ) + self._validate_resource_sort(request) - page_ids = self.paginate_queryset(resource_id_queryset) - if page_ids is not None: - resource_data = self._build_resource_aggregation( - filtered_queryset, - resource_ids=page_ids, - tenant_id=request.tenant_id, - ) - results = self._post_process_resources(resource_data) - serializer = FindingGroupResourceSerializer(results, many=True) - return self.get_paginated_response(serializer.data) + # 3. Check if the finding group exists (scoped to tenant/RBAC, ignoring user filters) + if not queryset.filter(check_id=check_id).exists(): + raise NotFound(f"Finding group '{check_id}' not found.") - resource_data = self._build_resource_aggregation( - filtered_queryset, - resource_ids=resource_ids, - tenant_id=request.tenant_id, + # 4. Narrow to check_id + filtered_queryset = filtered_queryset.filter(check_id=check_id) + + return self._paginated_resource_response( + request, filtered_queryset, resource_ids, request.tenant_id ) - results = self._post_process_resources(resource_data) - serializer = FindingGroupResourceSerializer(results, many=True) - return Response(serializer.data) @extend_schema( summary="List resources for a finding group from latest scans", @@ -7311,6 +8371,7 @@ class FindingGroupViewSet(BaseRLSViewSet): and timing information. No date filters required. """, tags=["Finding Groups"], + filters=True, ) @action( detail=False, @@ -7328,10 +8389,13 @@ class FindingGroupViewSet(BaseRLSViewSet): tenant_id = request.tenant_id queryset = self._get_finding_queryset() - # Get latest completed scan for each provider + # Order by -completed_at (matching the /latest summary path and the + # daily summary upsert keyed on midnight(completed_at)) so that + # overlapping scans do not make /resources and /latest read from + # different scans and report diverging counts. latest_scan_ids = ( Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) - .order_by("provider_id", "-inserted_at") + .order_by("provider_id", "-completed_at", "-inserted_at") .distinct("provider_id") .values_list("id", flat=True) ) @@ -7342,55 +8406,31 @@ class FindingGroupViewSet(BaseRLSViewSet): if key.startswith("inserted_at"): del normalized_params[key] + # 1. Normalize and split params finding_params, resource_params = self._split_resource_filters( normalized_params ) + # 2. Validate all inputs before any DB existence check filterset = LatestFindingGroupFilter(finding_params, queryset=queryset) if not filterset.is_valid(): raise ValidationError(filterset.errors) filtered_queryset = filterset.qs + resource_ids = self._resource_ids_from_params( + resource_params, request.tenant_id + ) + self._validate_resource_sort(request) - # Filter to latest scans and check_id + # 3. Check if the finding group exists (scoped to tenant/RBAC + latest scans) + if not queryset.filter(scan_id__in=latest_scan_ids, check_id=check_id).exists(): + raise NotFound(f"Finding group '{check_id}' not found.") + + # 4. Narrow to latest scans + check_id filtered_queryset = filtered_queryset.filter( scan_id__in=latest_scan_ids, check_id=check_id, ) - # Check if any findings exist for this check_id - if not filtered_queryset.exists(): - raise NotFound(f"Finding group '{check_id}' not found.") - - resource_ids = self._resource_ids_from_params( - resource_params, request.tenant_id + return self._paginated_resource_response( + request, filtered_queryset, resource_ids, request.tenant_id ) - mapping_queryset = self._build_resource_mapping_queryset( - filtered_queryset, - resource_ids=resource_ids, - tenant_id=request.tenant_id, - ) - resource_id_queryset = ( - mapping_queryset.values_list("resource_id", flat=True) - .distinct() - .order_by("resource_id") - ) - - page_ids = self.paginate_queryset(resource_id_queryset) - if page_ids is not None: - resource_data = self._build_resource_aggregation( - filtered_queryset, - resource_ids=page_ids, - tenant_id=request.tenant_id, - ) - results = self._post_process_resources(resource_data) - serializer = FindingGroupResourceSerializer(results, many=True) - return self.get_paginated_response(serializer.data) - - resource_data = self._build_resource_aggregation( - filtered_queryset, - resource_ids=resource_ids, - tenant_id=request.tenant_id, - ) - results = self._post_process_resources(resource_data) - serializer = FindingGroupResourceSerializer(results, many=True) - return Response(serializer.data) diff --git a/api/src/backend/config/celery.py b/api/src/backend/config/celery.py index aaa1b1c386..c46c8a426c 100644 --- a/api/src/backend/config/celery.py +++ b/api/src/backend/config/celery.py @@ -17,8 +17,10 @@ celery_app.config_from_object("django.conf:settings", namespace="CELERY") celery_app.conf.update(result_extended=True, result_expires=None) celery_app.conf.broker_transport_options = { - "visibility_timeout": BROKER_VISIBILITY_TIMEOUT + "visibility_timeout": BROKER_VISIBILITY_TIMEOUT, + "queue_order_strategy": "priority", } +celery_app.conf.task_default_priority = 6 celery_app.conf.result_backend_transport_options = { "visibility_timeout": BROKER_VISIBILITY_TIMEOUT } diff --git a/api/src/backend/config/django/base.py b/api/src/backend/config/django/base.py index 9304c12938..238825591f 100644 --- a/api/src/backend/config/django/base.py +++ b/api/src/backend/config/django/base.py @@ -299,3 +299,8 @@ DJANGO_DELETION_BATCH_SIZE = env.int("DJANGO_DELETION_BATCH_SIZE", 5000) # SAML requirement CSRF_COOKIE_SECURE = True SESSION_COOKIE_SECURE = True + +# Attack Paths +ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int( + "ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 2880 +) # 48h diff --git a/api/src/backend/config/django/production.py b/api/src/backend/config/django/production.py index b2769237fc..91bd50d0d1 100644 --- a/api/src/backend/config/django/production.py +++ b/api/src/backend/config/django/production.py @@ -3,6 +3,10 @@ from config.env import env DEBUG = env.bool("DJANGO_DEBUG", default=False) ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"]) +CORS_ALLOWED_ORIGINS = env.list( + "DJANGO_CORS_ALLOWED_ORIGINS", + default=["http://localhost", "http://127.0.0.1"], +) # Database # TODO Use Django database routers https://docs.djangoproject.com/en/5.0/topics/db/multi-db/#automatic-database-routing diff --git a/api/src/backend/config/guniconf.py b/api/src/backend/config/guniconf.py index a5b625874b..536fd97abb 100644 --- a/api/src/backend/config/guniconf.py +++ b/api/src/backend/config/guniconf.py @@ -15,7 +15,7 @@ from config.django.production import LOGGING as DJANGO_LOGGERS, DEBUG # noqa: E from config.custom_logging import BackendLogger # noqa: E402 BIND_ADDRESS = env("DJANGO_BIND_ADDRESS", default="127.0.0.1") -PORT = env("DJANGO_PORT", default=8000) +PORT = env("DJANGO_PORT", default=8080) # Server settings bind = f"{BIND_ADDRESS}:{PORT}" diff --git a/api/src/backend/config/settings/celery.py b/api/src/backend/config/settings/celery.py index a2aba00007..b7030ebea4 100644 --- a/api/src/backend/config/settings/celery.py +++ b/api/src/backend/config/settings/celery.py @@ -1,10 +1,52 @@ +from urllib.parse import quote + from config.env import env +_VALID_SCHEMES = {"redis", "rediss"} + + +def _build_celery_broker_url( + scheme: str, + username: str, + password: str, + host: str, + port: str, + db: str, +) -> str: + if scheme not in _VALID_SCHEMES: + raise ValueError( + f"Invalid VALKEY_SCHEME '{scheme}'. Must be one of: {', '.join(sorted(_VALID_SCHEMES))}" + ) + + encoded_username = quote(username, safe="") if username else "" + encoded_password = quote(password, safe="") if password else "" + + auth = "" + if encoded_username and encoded_password: + auth = f"{encoded_username}:{encoded_password}@" + elif encoded_password: + auth = f":{encoded_password}@" + elif encoded_username: + auth = f"{encoded_username}@" + + return f"{scheme}://{auth}{host}:{port}/{db}" + + +VALKEY_SCHEME = env("VALKEY_SCHEME", default="redis") +VALKEY_USERNAME = env("VALKEY_USERNAME", default="") +VALKEY_PASSWORD = env("VALKEY_PASSWORD", default="") VALKEY_HOST = env("VALKEY_HOST", default="valkey") VALKEY_PORT = env("VALKEY_PORT", default="6379") VALKEY_DB = env("VALKEY_DB", default="0") -CELERY_BROKER_URL = f"redis://{VALKEY_HOST}:{VALKEY_PORT}/{VALKEY_DB}" +CELERY_BROKER_URL = _build_celery_broker_url( + VALKEY_SCHEME, + VALKEY_USERNAME, + VALKEY_PASSWORD, + VALKEY_HOST, + VALKEY_PORT, + VALKEY_DB, +) CELERY_RESULT_BACKEND = "django-db" CELERY_TASK_TRACK_STARTED = True diff --git a/api/src/backend/config/settings/sentry.py b/api/src/backend/config/settings/sentry.py index 6b8c554258..580821f7b2 100644 --- a/api/src/backend/config/settings/sentry.py +++ b/api/src/backend/config/settings/sentry.py @@ -1,4 +1,5 @@ import sentry_sdk + from config.env import env IGNORED_EXCEPTIONS = [ @@ -85,8 +86,20 @@ def before_send(event, hint): # Ignore logs with the ignored_exceptions # https://docs.python.org/3/library/logging.html#logrecord-objects if "log_record" in hint: - log_msg = hint["log_record"].msg - log_lvl = hint["log_record"].levelno + log_record = hint["log_record"] + log_msg = log_record.getMessage() + log_lvl = log_record.levelno + + # The Neo4j driver logs transient connection errors (defunct + # connections, resets) at ERROR level via the `neo4j.io` logger. + # `RetryableSession` handles these with retries. If all retries + # are exhausted, the exception propagates and Sentry captures + # it as a normal exception event. + if ( + getattr(log_record, "name", "").startswith("neo4j.io") + and "defunct" in log_msg + ): + return None # Handle Error and Critical events and discard the rest if log_lvl <= 40 and any(ignored in log_msg for ignored in IGNORED_EXCEPTIONS): @@ -107,6 +120,7 @@ sentry_sdk.init( # see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info before_send=before_send, send_default_pii=True, + traces_sample_rate=env.float("DJANGO_SENTRY_TRACES_SAMPLE_RATE", default=0.02), _experiments={ # Set continuous_profiling_auto_start to True # to automatically start the profiler on when diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index 6535706cf7..c5228c77be 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -14,8 +14,8 @@ from rest_framework import status from rest_framework.test import APIClient from tasks.jobs.backfill import ( backfill_resource_scan_summaries, - backfill_scan_category_summaries, - backfill_scan_resource_group_summaries, + aggregate_scan_category_summaries, + aggregate_scan_resource_group_summaries, ) from api.attack_paths import ( @@ -111,8 +111,9 @@ def disable_logging(): logging.disable(logging.CRITICAL) -@pytest.fixture(scope="session", autouse=True) -def create_test_user(django_db_setup, django_db_blocker): +@pytest.fixture(scope="session") +def _session_test_user(django_db_setup, django_db_blocker): + """Create the test user once per session. Internal; use create_test_user instead.""" with django_db_blocker.unblock(): user = User.objects.create_user( name="testing", @@ -122,6 +123,21 @@ def create_test_user(django_db_setup, django_db_blocker): return user +@pytest.fixture(autouse=True) +def create_test_user(_session_test_user, django_db_blocker): + """Re-create the session-scoped test user when a TransactionTestCase + has truncated the users table.""" + with django_db_blocker.unblock(): + if not User.objects.filter(pk=_session_test_user.pk).exists(): + User.objects.create_user( + id=_session_test_user.pk, + name="testing", + email=TEST_USER, + password=TEST_PASSWORD, + ) + return _session_test_user + + @pytest.fixture(scope="function") def create_test_user_rbac(django_db_setup, django_db_blocker, tenants_fixture): with django_db_blocker.unblock(): @@ -549,6 +565,12 @@ def providers_fixture(tenants_fixture): alias="googleworkspace_testing", tenant_id=tenant.id, ) + provider13 = Provider.objects.create( + provider="vercel", + uid="team_abcdef1234567890ab", + alias="vercel_testing", + tenant_id=tenant.id, + ) return ( provider1, @@ -563,6 +585,7 @@ def providers_fixture(tenants_fixture): provider10, provider11, provider12, + provider13, ) @@ -1422,8 +1445,8 @@ def latest_scan_finding_with_categories( ) finding.add_resources([resource]) backfill_resource_scan_summaries(tenant_id, str(scan.id)) - backfill_scan_category_summaries(tenant_id, str(scan.id)) - backfill_scan_resource_group_summaries(tenant_id, str(scan.id)) + aggregate_scan_category_summaries(tenant_id, str(scan.id)) + aggregate_scan_resource_group_summaries(tenant_id, str(scan.id)) return finding @@ -2012,6 +2035,7 @@ def finding_groups_fixture( "CheckId": "s3_bucket_public_access", "checktitle": "Ensure S3 buckets do not allow public access", "Description": "S3 buckets should be configured to restrict public access.", + "resourcegroup": "storage", }, first_seen_at="2024-01-02T00:00:00Z", muted=False, @@ -2036,6 +2060,7 @@ def finding_groups_fixture( "CheckId": "s3_bucket_public_access", "checktitle": "Ensure S3 buckets do not allow public access", "Description": "S3 buckets should be configured to restrict public access.", + "resourcegroup": "storage", }, first_seen_at="2024-01-03T00:00:00Z", muted=False, @@ -2234,6 +2259,89 @@ def finding_groups_fixture( return findings +@pytest.fixture +def finding_groups_title_variants_fixture( + tenants_fixture, providers_fixture, scans_fixture, resources_fixture +): + """ + Two providers report the same check_id with different checktitle values. + + Simulates a Prowler version upgrade where the check title changed but the + check_id stayed the same. Used to verify that check_title__icontains + resolves to check_id first, so results include all providers regardless + of which title variant matches the search term. + """ + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + scan1, scan2, *_ = scans_fixture + resource1, resource2, *_ = resources_fixture + + findings = [] + + # Provider 1 — OLD title variant + finding_old = Finding.objects.create( + tenant_id=tenant.id, + uid="fg_title_variant_old", + scan=scan1, + delta="new", + status=Status.FAIL, + status_extended="Secret scanning not enabled", + impact=Severity.high, + impact_extended="High risk", + severity=Severity.high, + raw_result={"status": Status.FAIL, "severity": Severity.high}, + tags={}, + check_id="github_secret_scanning_enabled", + check_metadata={ + "CheckId": "github_secret_scanning_enabled", + "checktitle": "Ensure repository has secret scanning enabled", + "Description": "Checks if secret scanning is enabled.", + }, + first_seen_at="2024-01-01T00:00:00Z", + muted=False, + ) + finding_old.add_resources([resource1]) + findings.append(finding_old) + + # Provider 2 — NEW title variant (same check_id, different checktitle) + finding_new = Finding.objects.create( + tenant_id=tenant.id, + uid="fg_title_variant_new", + scan=scan2, + delta="new", + status=Status.FAIL, + status_extended="Secret scanning not enabled on repo", + impact=Severity.high, + impact_extended="High risk", + severity=Severity.high, + raw_result={"status": Status.FAIL, "severity": Severity.high}, + tags={}, + check_id="github_secret_scanning_enabled", + check_metadata={ + "CheckId": "github_secret_scanning_enabled", + "checktitle": "Check if secret scanning is enabled in GitHub", + "Description": "Checks if secret scanning is enabled.", + }, + first_seen_at="2024-01-02T00:00:00Z", + muted=False, + ) + finding_new.add_resources([resource2]) + findings.append(finding_new) + + from tasks.jobs.scan import aggregate_finding_group_summaries + + aggregate_finding_group_summaries( + tenant_id=str(tenant.id), + scan_id=str(scan1.id), + ) + aggregate_finding_group_summaries( + tenant_id=str(tenant.id), + scan_id=str(scan2.id), + ) + + return findings + + def pytest_collection_modifyitems(items): """Ensure test_rbac.py is executed first.""" items.sort(key=lambda item: 0 if "test_rbac.py" in item.nodeid else 1) diff --git a/api/src/backend/tasks/assets/img/cis_logo.png b/api/src/backend/tasks/assets/img/cis_logo.png new file mode 100644 index 0000000000..7c1568da5e Binary files /dev/null and b/api/src/backend/tasks/assets/img/cis_logo.png differ diff --git a/api/src/backend/tasks/jobs/attack_paths/aws.py b/api/src/backend/tasks/jobs/attack_paths/aws.py index e7f26b5173..692eee61cd 100644 --- a/api/src/backend/tasks/jobs/attack_paths/aws.py +++ b/api/src/backend/tasks/jobs/attack_paths/aws.py @@ -1,6 +1,8 @@ # Portions of this file are based on code from the Cartography project # (https://github.com/cartography-cncf/cartography), which is licensed under the Apache 2.0 License. +import time + from typing import Any import aioboto3 @@ -33,7 +35,7 @@ def start_aws_ingestion( For the scan progress updates: - The caller of this function (`tasks.jobs.attack_paths.scan.run`) has set it to 2. - - When the control returns to the caller, it will be set to 95. + - When the control returns to the caller, it will be set to 93. """ # Initialize variables common to all jobs @@ -47,7 +49,7 @@ def start_aws_ingestion( } boto3_session = get_boto3_session(prowler_api_provider, prowler_sdk_provider) - regions: list[str] = list(prowler_sdk_provider._enabled_regions) + regions: list[str] = resolve_aws_regions(prowler_api_provider, prowler_sdk_provider) requested_syncs = list(cartography_aws.RESOURCE_FUNCTIONS.keys()) sync_args = cartography_aws._build_aws_sync_kwargs( @@ -89,34 +91,50 @@ def start_aws_ingestion( logger.info( f"Syncing function permission_relationships for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.RESOURCE_FUNCTIONS["permission_relationships"](**sync_args) + logger.info( + f"Synced function permission_relationships for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 88) if "resourcegroupstaggingapi" in requested_syncs: logger.info( f"Syncing function resourcegroupstaggingapi for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.RESOURCE_FUNCTIONS["resourcegroupstaggingapi"](**sync_args) + logger.info( + f"Synced function resourcegroupstaggingapi for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 89) logger.info( f"Syncing ec2_iaminstanceprofile scoped analysis for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.run_scoped_analysis_job( "aws_ec2_iaminstanceprofile.json", neo4j_session, common_job_parameters, ) + logger.info( + f"Synced ec2_iaminstanceprofile scoped analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 90) logger.info( f"Syncing lambda_ecr analysis for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.run_analysis_job( "aws_lambda_ecr.json", neo4j_session, common_job_parameters, ) + logger.info( + f"Synced lambda_ecr analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) if all( s in requested_syncs @@ -125,25 +143,34 @@ def start_aws_ingestion( logger.info( f"Syncing lb_container_exposure scoped analysis for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.run_scoped_analysis_job( "aws_lb_container_exposure.json", neo4j_session, common_job_parameters, ) + logger.info( + f"Synced lb_container_exposure scoped analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) if all(s in requested_syncs for s in ["ec2:network_acls", "ec2:load_balancer_v2"]): logger.info( f"Syncing lb_nacl_direct scoped analysis for AWS account {prowler_api_provider.uid}" ) + t0 = time.perf_counter() cartography_aws.run_scoped_analysis_job( "aws_lb_nacl_direct.json", neo4j_session, common_job_parameters, ) + logger.info( + f"Synced lb_nacl_direct scoped analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 91) logger.info(f"Syncing metadata for AWS account {prowler_api_provider.uid}") + t0 = time.perf_counter() cartography_aws.merge_module_sync_metadata( neo4j_session, group_type="AWSAccount", @@ -152,24 +179,23 @@ def start_aws_ingestion( update_tag=cartography_config.update_tag, stat_handler=cartography_aws.stat_handler, ) + logger.info( + f"Synced metadata for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 92) # Removing the added extra field del common_job_parameters["AWS_ID"] - logger.info(f"Syncing cleanup_job for AWS account {prowler_api_provider.uid}") - cartography_aws.run_cleanup_job( - "aws_post_ingestion_principals_cleanup.json", - neo4j_session, - common_job_parameters, - ) - db_utils.update_attack_paths_scan_progress(attack_paths_scan, 93) - logger.info(f"Syncing analysis for AWS account {prowler_api_provider.uid}") + t0 = time.perf_counter() cartography_aws._perform_aws_analysis( requested_syncs, neo4j_session, common_job_parameters ) - db_utils.update_attack_paths_scan_progress(attack_paths_scan, 94) + logger.info( + f"Synced analysis for AWS account {prowler_api_provider.uid} in {time.perf_counter() - t0:.3f}s" + ) + db_utils.update_attack_paths_scan_progress(attack_paths_scan, 93) return failed_syncs @@ -200,6 +226,48 @@ def get_boto3_session( return boto3_session +def resolve_aws_regions( + prowler_api_provider: ProwlerAPIProvider, + prowler_sdk_provider: ProwlerSDKProvider, +) -> list[str]: + """Resolve the regions to scan, falling back when `_enabled_regions` is `None`. + + The SDK silently sets `_enabled_regions` to `None` when `ec2:DescribeRegions` + fails (missing IAM permission, transient error). Without a fallback the + Cartography ingestion crashes with a non-actionable `TypeError`. Try the + user's `audited_regions` next, then the partition's static region list. + Excluded regions are honored on every branch. + """ + if prowler_sdk_provider._enabled_regions is not None: + regions = set(prowler_sdk_provider._enabled_regions) + + elif prowler_sdk_provider.identity.audited_regions: + regions = set(prowler_sdk_provider.identity.audited_regions) + + else: + partition = prowler_sdk_provider.identity.partition + try: + regions = prowler_sdk_provider.get_available_aws_service_regions( + "ec2", partition + ) + + except KeyError: + raise RuntimeError( + f"No region data available for partition {partition!r}; " + f"cannot determine regions to scan for " + f"{prowler_api_provider.uid}" + ) + + logger.warning( + f"Could not enumerate enabled regions for AWS account " + f"{prowler_api_provider.uid}; falling back to all regions in " + f"partition {partition!r}" + ) + + excluded = set(getattr(prowler_sdk_provider, "_excluded_regions", None) or ()) + return sorted(regions - excluded) + + def get_aioboto3_session(boto3_session: boto3.Session) -> aioboto3.Session: return aioboto3.Session(botocore_session=boto3_session._session) @@ -234,6 +302,8 @@ def sync_aws_account( ) try: + func_t0 = time.perf_counter() + # `ecr:image_layers` uses `aioboto3_session` instead of `boto3_session` if func_name == "ecr:image_layers": cartography_aws.RESOURCE_FUNCTIONS[func_name]( @@ -257,7 +327,15 @@ def sync_aws_account( else: cartography_aws.RESOURCE_FUNCTIONS[func_name](**sync_args) + logger.info( + f"Synced function {func_name} for AWS account {prowler_api_provider.uid} in {time.perf_counter() - func_t0:.3f}s" + ) + except Exception as e: + logger.info( + f"Synced function {func_name} for AWS account {prowler_api_provider.uid} in {time.perf_counter() - func_t0:.3f}s (FAILED)" + ) + exception_message = utils.stringify_exception( e, f"Exception for AWS sync function: {func_name}" ) @@ -277,3 +355,16 @@ def sync_aws_account( ) return failed_syncs + + +def extract_short_uid(uid: str) -> str: + """Return the short identifier from an AWS ARN or resource ID. + + Supported inputs end in one of: + - `/` (e.g. `instance/i-xxx`) + - `:` (e.g. `function:name`) + - `` (e.g. `bucket-name` or `i-xxx`) + + If `uid` is already a short resource ID, it is returned unchanged. + """ + return uid.rsplit("/", 1)[-1].rsplit(":", 1)[-1] diff --git a/api/src/backend/tasks/jobs/attack_paths/cleanup.py b/api/src/backend/tasks/jobs/attack_paths/cleanup.py new file mode 100644 index 0000000000..65ba583a3e --- /dev/null +++ b/api/src/backend/tasks/jobs/attack_paths/cleanup.py @@ -0,0 +1,255 @@ +from datetime import datetime, timedelta, timezone + +from celery import current_app, states +from celery.utils.log import get_task_logger +from config.django.base import ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES +from tasks.jobs.attack_paths.db_utils import ( + _mark_scan_finished, + recover_graph_data_ready, +) + +from api.attack_paths import database as graph_database +from api.db_router import MainRouter +from api.db_utils import rls_transaction +from api.models import AttackPathsScan, StateChoices + +logger = get_task_logger(__name__) + + +def cleanup_stale_attack_paths_scans() -> dict: + """ + Mark stale `AttackPathsScan` rows as `FAILED`. + + Covers two stuck-state scenarios: + 1. `EXECUTING` scans whose workers are dead, or that have exceeded the + stale threshold while alive. + 2. `SCHEDULED` scans that never made it to a worker — parent scan + crashed before dispatch, broker lost the message, etc. Detected by + age plus the parent `Scan` no longer being in flight. + """ + threshold = timedelta(minutes=ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES) + now = datetime.now(tz=timezone.utc) + cutoff = now - threshold + + cleaned_up: list[str] = [] + cleaned_up.extend(_cleanup_stale_executing_scans(cutoff)) + cleaned_up.extend(_cleanup_stale_scheduled_scans(cutoff)) + + logger.info( + f"Stale `AttackPathsScan` cleanup: {len(cleaned_up)} scan(s) cleaned up" + ) + return {"cleaned_up_count": len(cleaned_up), "scan_ids": cleaned_up} + + +def _cleanup_stale_executing_scans(cutoff: datetime) -> list[str]: + """ + Two-pass detection for `EXECUTING` scans: + 1. If `TaskResult.worker` exists, ping the worker. + - Dead worker: cleanup immediately (any age). + - Alive + past threshold: revoke the task, then cleanup. + - Alive + within threshold: skip. + 2. If no worker field: fall back to time-based heuristic only. + """ + executing_scans = list( + AttackPathsScan.all_objects.using(MainRouter.admin_db) + .filter(state=StateChoices.EXECUTING) + .select_related("task__task_runner_task") + ) + + # Cache worker liveness so each worker is pinged at most once + workers = { + tr.worker + for scan in executing_scans + if (tr := getattr(scan.task, "task_runner_task", None) if scan.task else None) + and tr.worker + } + worker_alive = {w: _is_worker_alive(w) for w in workers} + + cleaned_up: list[str] = [] + + for scan in executing_scans: + task_result = ( + getattr(scan.task, "task_runner_task", None) if scan.task else None + ) + worker = task_result.worker if task_result else None + + if worker: + alive = worker_alive.get(worker, True) + + if alive: + if scan.started_at and scan.started_at >= cutoff: + continue + + # Alive but stale — revoke before cleanup + _revoke_task(task_result) + reason = "Scan exceeded stale threshold — cleaned up by periodic task" + else: + reason = "Worker dead — cleaned up by periodic task" + else: + # No worker recorded — time-based heuristic only + if scan.started_at and scan.started_at >= cutoff: + continue + reason = ( + "No worker recorded, scan exceeded stale threshold — " + "cleaned up by periodic task" + ) + + if _cleanup_scan(scan, task_result, reason): + cleaned_up.append(str(scan.id)) + + return cleaned_up + + +def _cleanup_stale_scheduled_scans(cutoff: datetime) -> list[str]: + """ + Cleanup `SCHEDULED` scans that never reached a worker. + + Detection: + - `state == SCHEDULED` + - `started_at < cutoff` + - parent `Scan` is no longer in flight (terminal state or missing). This + avoids cleaning up rows whose parent Prowler scan is legitimately still + running. + + For each match: revoke the queued task (best-effort; harmless if already + consumed), atomically flip to `FAILED`, and mark the `TaskResult`. The + temp Neo4j database is never created while `SCHEDULED`, so no drop is + needed. + """ + scheduled_scans = list( + AttackPathsScan.all_objects.using(MainRouter.admin_db) + .filter( + state=StateChoices.SCHEDULED, + started_at__lt=cutoff, + ) + .select_related("task__task_runner_task", "scan") + ) + + cleaned_up: list[str] = [] + parent_terminal = ( + StateChoices.COMPLETED, + StateChoices.FAILED, + StateChoices.CANCELLED, + ) + + for scan in scheduled_scans: + parent_scan = scan.scan + if parent_scan is not None and parent_scan.state not in parent_terminal: + continue + + task_result = ( + getattr(scan.task, "task_runner_task", None) if scan.task else None + ) + if task_result: + _revoke_task(task_result, terminate=False) + + reason = "Scan never started — cleaned up by periodic task" + if _cleanup_scheduled_scan(scan, task_result, reason): + cleaned_up.append(str(scan.id)) + + return cleaned_up + + +def _is_worker_alive(worker: str) -> bool: + """Ping a specific Celery worker. Returns `True` if it responds or on error.""" + try: + response = current_app.control.inspect(destination=[worker], timeout=1.0).ping() + return response is not None and worker in response + except Exception: + logger.exception(f"Failed to ping worker {worker}, treating as alive") + return True + + +def _revoke_task(task_result, terminate: bool = True) -> None: + """Revoke a Celery task. Non-fatal on failure. + + `terminate=True` SIGTERMs the worker if the task is mid-execution; use + for EXECUTING cleanup. `terminate=False` only marks the task id revoked + across workers, so any worker pulling the queued message discards it; + use for SCHEDULED cleanup where the task hasn't run yet. + """ + try: + kwargs = {"terminate": True, "signal": "SIGTERM"} if terminate else {} + current_app.control.revoke(task_result.task_id, **kwargs) + logger.info(f"Revoked task {task_result.task_id}") + except Exception: + logger.exception(f"Failed to revoke task {task_result.task_id}") + + +def _cleanup_scan(scan, task_result, reason: str) -> bool: + """ + Clean up a single stale `AttackPathsScan`: + drop temp DB, mark `FAILED`, update `TaskResult`, recover `graph_data_ready`. + + Returns `True` if the scan was actually cleaned up, `False` if skipped. + """ + scan_id_str = str(scan.id) + + # 1. Drop temp Neo4j database + tmp_db_name = graph_database.get_database_name(scan.id, temporary=True) + try: + graph_database.drop_database(tmp_db_name) + except Exception: + logger.exception(f"Failed to drop temp database {tmp_db_name}") + + fresh_scan = _finalize_failed_scan(scan, StateChoices.EXECUTING, reason) + if fresh_scan is None: + return False + + # Mark `TaskResult` as `FAILURE` (not RLS-protected, outside lock) + if task_result: + task_result.status = states.FAILURE + task_result.date_done = datetime.now(tz=timezone.utc) + task_result.save(update_fields=["status", "date_done"]) + + recover_graph_data_ready(fresh_scan) + + logger.info(f"Cleaned up stale scan {scan_id_str}: {reason}") + return True + + +def _cleanup_scheduled_scan(scan, task_result, reason: str) -> bool: + """ + Clean up a `SCHEDULED` scan that never reached a worker. + + Skips the temp Neo4j drop — the database is only created once the worker + enters `EXECUTING`, so dropping it here just produces noisy log output. + + Returns `True` if the scan was actually cleaned up, `False` if skipped. + """ + scan_id_str = str(scan.id) + + fresh_scan = _finalize_failed_scan(scan, StateChoices.SCHEDULED, reason) + if fresh_scan is None: + return False + + if task_result: + task_result.status = states.FAILURE + task_result.date_done = datetime.now(tz=timezone.utc) + task_result.save(update_fields=["status", "date_done"]) + + logger.info(f"Cleaned up scheduled scan {scan_id_str}: {reason}") + return True + + +def _finalize_failed_scan(scan, expected_state: str, reason: str): + """ + Atomically lock the row, verify it's still in `expected_state`, and + mark it `FAILED`. Returns the locked row on success, `None` if the + row is gone or has already moved on. + """ + scan_id_str = str(scan.id) + with rls_transaction(str(scan.tenant_id)): + try: + fresh_scan = AttackPathsScan.objects.select_for_update().get(id=scan.id) + except AttackPathsScan.DoesNotExist: + logger.warning(f"Scan {scan_id_str} no longer exists, skipping") + return None + + if fresh_scan.state != expected_state: + logger.info(f"Scan {scan_id_str} is now {fresh_scan.state}, skipping") + return None + + _mark_scan_finished(fresh_scan, StateChoices.FAILED, {"global_error": reason}) + + return fresh_scan diff --git a/api/src/backend/tasks/jobs/attack_paths/config.py b/api/src/backend/tasks/jobs/attack_paths/config.py index 1667d314a7..0816626b67 100644 --- a/api/src/backend/tasks/jobs/attack_paths/config.py +++ b/api/src/backend/tasks/jobs/attack_paths/config.py @@ -1,25 +1,30 @@ from dataclasses import dataclass from typing import Callable +from uuid import UUID from config.env import env - from tasks.jobs.attack_paths import aws - -# Batch size for Neo4j operations +# Batch size for Neo4j write operations (resource labeling, cleanup) BATCH_SIZE = env.int("ATTACK_PATHS_BATCH_SIZE", 1000) +# Batch size for Postgres findings fetch (keyset pagination page size) +FINDINGS_BATCH_SIZE = env.int("ATTACK_PATHS_FINDINGS_BATCH_SIZE", 1000) +# Batch size for temp-to-tenant graph sync (nodes and relationships per cursor page) +SYNC_BATCH_SIZE = env.int("ATTACK_PATHS_SYNC_BATCH_SIZE", 1000) # Neo4j internal labels (Prowler-specific, not provider-specific) +# - `Internet`: Singleton node representing external internet access for exposed-resource queries # - `ProwlerFinding`: Label for finding nodes created by Prowler and linked to cloud resources # - `_ProviderResource`: Added to ALL synced nodes for provider isolation and drop/query ops -# - `Internet`: Singleton node representing external internet access for exposed-resource queries +INTERNET_NODE_LABEL = "Internet" PROWLER_FINDING_LABEL = "ProwlerFinding" PROVIDER_RESOURCE_LABEL = "_ProviderResource" -INTERNET_NODE_LABEL = "Internet" -# Phase 1 dual-write: deprecated label kept for drop_subgraph and infrastructure queries -# Remove in Phase 2 once all nodes use the private label exclusively -DEPRECATED_PROVIDER_RESOURCE_LABEL = "ProviderResource" +# Dynamic isolation labels that contain entity UUIDs and are added to every synced node during sync +# Format: _Tenant_{uuid_no_hyphens}, _Provider_{uuid_no_hyphens} +TENANT_LABEL_PREFIX = "_Tenant_" +PROVIDER_LABEL_PREFIX = "_Provider_" +DYNAMIC_ISOLATION_PREFIXES = [TENANT_LABEL_PREFIX, PROVIDER_LABEL_PREFIX] @dataclass(frozen=True) @@ -31,8 +36,9 @@ class ProviderConfig: uid_field: str # e.g., "arn" # Label for resources connected to the account node, enabling indexed finding lookups. resource_label: str # e.g., "_AWSResource" - deprecated_resource_label: str # e.g., "AWSResource" ingestion_function: Callable + # Maps a Postgres resource UID (e.g. full ARN) to the short-id form Cartography stores on some node types (e.g. `i-xxx` for EC2Instance). + short_uid_extractor: Callable[[str], str] # Provider Configurations @@ -43,8 +49,8 @@ AWS_CONFIG = ProviderConfig( root_node_label="AWSAccount", uid_field="arn", resource_label="_AWSResource", - deprecated_resource_label="AWSResource", ingestion_function=aws.start_aws_ingestion, + short_uid_extractor=aws.extract_short_uid, ) PROVIDER_CONFIGS: dict[str, ProviderConfig] = { @@ -56,18 +62,14 @@ PROVIDER_CONFIGS: dict[str, ProviderConfig] = { INTERNAL_LABELS: list[str] = [ "Tenant", # From Cartography, but it looks like it's ours PROVIDER_RESOURCE_LABEL, - DEPRECATED_PROVIDER_RESOURCE_LABEL, - # Add all provider-specific resource labels *[config.resource_label for config in PROVIDER_CONFIGS.values()], - *[config.deprecated_resource_label for config in PROVIDER_CONFIGS.values()], ] # Provider isolation properties +PROVIDER_ELEMENT_ID_PROPERTY = "_provider_element_id" + PROVIDER_ISOLATION_PROPERTIES: list[str] = [ - "_provider_id", - "_provider_element_id", - "provider_id", - "provider_element_id", + PROVIDER_ELEMENT_ID_PROPERTY, ] # Cartography bookkeeping metadata @@ -117,7 +119,40 @@ def get_provider_resource_label(provider_type: str) -> str: return config.resource_label if config else "_UnknownProviderResource" -def get_deprecated_provider_resource_label(provider_type: str) -> str: - """Get the deprecated resource label for a provider type (e.g., `AWSResource`).""" +def _identity_short_uid(uid: str) -> str: + """Fallback short-uid extractor for providers without a custom mapping.""" + return uid + + +def get_short_uid_extractor(provider_type: str) -> Callable[[str], str]: + """Get the short-uid extractor for a provider type. + + Returns an identity function when the provider is unknown, so callers can + rely on a callable always being returned. + """ config = PROVIDER_CONFIGS.get(provider_type) - return config.deprecated_resource_label if config else "UnknownProviderResource" + return config.short_uid_extractor if config else _identity_short_uid + + +# Dynamic Isolation Label Helpers +# -------------------------------- + + +def _normalize_uuid(value: str | UUID) -> str: + """Strip hyphens from a UUID string for use in Neo4j labels.""" + return str(value).replace("-", "") + + +def get_tenant_label(tenant_id: str | UUID) -> str: + """Get the Neo4j label for a tenant (e.g., `_Tenant_019c41ee7df37deca684d839f95619f8`).""" + return f"{TENANT_LABEL_PREFIX}{_normalize_uuid(tenant_id)}" + + +def get_provider_label(provider_id: str | UUID) -> str: + """Get the Neo4j label for a provider (e.g., `_Provider_019c41ee7df37deca684d839f95619f8`).""" + return f"{PROVIDER_LABEL_PREFIX}{_normalize_uuid(provider_id)}" + + +def is_dynamic_isolation_label(label: str) -> bool: + """Check if a label is a dynamic tenant/provider isolation label.""" + return any(label.startswith(prefix) for prefix in DYNAMIC_ISOLATION_PREFIXES) diff --git a/api/src/backend/tasks/jobs/attack_paths/db_utils.py b/api/src/backend/tasks/jobs/attack_paths/db_utils.py index 9fb52b0ead..a6e7da7f87 100644 --- a/api/src/backend/tasks/jobs/attack_paths/db_utils.py +++ b/api/src/backend/tasks/jobs/attack_paths/db_utils.py @@ -3,15 +3,13 @@ from typing import Any from cartography.config import Config as CartographyConfig from celery.utils.log import get_task_logger +from tasks.jobs.attack_paths.config import is_provider_available from api.attack_paths import database as graph_database from api.db_utils import rls_transaction -from api.models import ( - AttackPathsScan as ProwlerAPIAttackPathsScan, - Provider as ProwlerAPIProvider, - StateChoices, -) -from tasks.jobs.attack_paths.config import is_provider_available +from api.models import AttackPathsScan as ProwlerAPIAttackPathsScan +from api.models import Provider as ProwlerAPIProvider +from api.models import StateChoices logger = get_task_logger(__name__) @@ -69,25 +67,80 @@ def retrieve_attack_paths_scan( return None +def set_attack_paths_scan_task_id( + tenant_id: str, + scan_pk: str, + task_id: str, +) -> None: + """Persist the Celery `task_id` on the `AttackPathsScan` row. + + Called at dispatch time (when `apply_async` returns) so the row carries + the task id even while still `SCHEDULED`. This lets the periodic + cleanup revoke queued messages for scans that never reached a worker. + """ + with rls_transaction(tenant_id): + ProwlerAPIAttackPathsScan.objects.filter(id=scan_pk).update(task_id=task_id) + + def starting_attack_paths_scan( attack_paths_scan: ProwlerAPIAttackPathsScan, - task_id: str, cartography_config: CartographyConfig, -) -> None: - with rls_transaction(attack_paths_scan.tenant_id): - attack_paths_scan.task_id = task_id - attack_paths_scan.state = StateChoices.EXECUTING - attack_paths_scan.started_at = datetime.now(tz=timezone.utc) - attack_paths_scan.update_tag = cartography_config.update_tag +) -> bool: + """Flip the row from `SCHEDULED` to `EXECUTING` atomically. - attack_paths_scan.save( - update_fields=[ - "task_id", - "state", - "started_at", - "update_tag", - ] - ) + Returns `False` if the row is gone or has already moved past + `SCHEDULED` (e.g., periodic cleanup raced ahead and marked it + `FAILED` while the worker message was still in flight). + """ + with rls_transaction(attack_paths_scan.tenant_id): + try: + locked = ProwlerAPIAttackPathsScan.objects.select_for_update().get( + id=attack_paths_scan.id + ) + except ProwlerAPIAttackPathsScan.DoesNotExist: + return False + + if locked.state != StateChoices.SCHEDULED: + return False + + locked.state = StateChoices.EXECUTING + locked.started_at = datetime.now(tz=timezone.utc) + locked.update_tag = cartography_config.update_tag + locked.save(update_fields=["state", "started_at", "update_tag"]) + + # Keep the in-memory object the caller is holding in sync. + attack_paths_scan.state = locked.state + attack_paths_scan.started_at = locked.started_at + attack_paths_scan.update_tag = locked.update_tag + return True + + +def _mark_scan_finished( + attack_paths_scan: ProwlerAPIAttackPathsScan, + state: StateChoices, + ingestion_exceptions: dict[str, Any], +) -> None: + """Set terminal fields on a scan. Caller must be inside a transaction.""" + now = datetime.now(tz=timezone.utc) + duration = ( + int((now - attack_paths_scan.started_at).total_seconds()) + if attack_paths_scan.started_at + else 0 + ) + attack_paths_scan.state = state + attack_paths_scan.progress = 100 + attack_paths_scan.completed_at = now + attack_paths_scan.duration = duration + attack_paths_scan.ingestion_exceptions = ingestion_exceptions + attack_paths_scan.save( + update_fields=[ + "state", + "progress", + "completed_at", + "duration", + "ingestion_exceptions", + ] + ) def finish_attack_paths_scan( @@ -96,28 +149,7 @@ def finish_attack_paths_scan( ingestion_exceptions: dict[str, Any], ) -> None: with rls_transaction(attack_paths_scan.tenant_id): - now = datetime.now(tz=timezone.utc) - duration = ( - int((now - attack_paths_scan.started_at).total_seconds()) - if attack_paths_scan.started_at - else 0 - ) - - attack_paths_scan.state = state - attack_paths_scan.progress = 100 - attack_paths_scan.completed_at = now - attack_paths_scan.duration = duration - attack_paths_scan.ingestion_exceptions = ingestion_exceptions - - attack_paths_scan.save( - update_fields=[ - "state", - "progress", - "completed_at", - "duration", - "ingestion_exceptions", - ] - ) + _mark_scan_finished(attack_paths_scan, state, ingestion_exceptions) def update_attack_paths_scan_progress( @@ -155,6 +187,37 @@ def set_provider_graph_data_ready( attack_paths_scan.refresh_from_db(fields=["graph_data_ready"]) +def recover_graph_data_ready( + attack_paths_scan: ProwlerAPIAttackPathsScan, +) -> None: + """ + Best-effort recovery of `graph_data_ready` after a scan failure. + + Queries Neo4j to check if the provider still has data in the tenant + database. If data exists, restores `graph_data_ready=True` for all scans + of this provider. Never raises. + + Trade-off: if the worker crashed mid-sync, partial data may exist and + this will re-enable queries against it. We accept that because leaving + `graph_data_ready=False` permanently (blocking all queries until the + next successful scan) is a worse outcome for the user. + """ + try: + tenant_db = graph_database.get_database_name(attack_paths_scan.tenant_id) + if graph_database.has_provider_data( + tenant_db, str(attack_paths_scan.provider_id) + ): + set_provider_graph_data_ready(attack_paths_scan, True) + logger.info( + f"Recovered `graph_data_ready` for provider {attack_paths_scan.provider_id}" + ) + + except Exception: + logger.exception( + f"Failed to recover `graph_data_ready` for provider {attack_paths_scan.provider_id}" + ) + + def fail_attack_paths_scan( tenant_id: str, scan_id: str, @@ -165,23 +228,26 @@ def fail_attack_paths_scan( Used as a safety net when the Celery task fails outside the job's own error handling. """ attack_paths_scan = retrieve_attack_paths_scan(tenant_id, scan_id) - if attack_paths_scan and attack_paths_scan.state not in ( - StateChoices.COMPLETED, - StateChoices.FAILED, - ): - tmp_db_name = graph_database.get_database_name( - attack_paths_scan.id, temporary=True + if not attack_paths_scan: + return + + tmp_db_name = graph_database.get_database_name(attack_paths_scan.id, temporary=True) + try: + graph_database.drop_database(tmp_db_name) + except Exception: + logger.exception( + f"Failed to drop temp database {tmp_db_name} during failure handling" ) + + with rls_transaction(tenant_id): try: - graph_database.drop_database(tmp_db_name) - - except Exception: - logger.exception( - f"Failed to drop temp database {tmp_db_name} during failure handling" + fresh = ProwlerAPIAttackPathsScan.objects.select_for_update().get( + id=attack_paths_scan.id ) + except ProwlerAPIAttackPathsScan.DoesNotExist: + return + if fresh.state in (StateChoices.COMPLETED, StateChoices.FAILED): + return + _mark_scan_finished(fresh, StateChoices.FAILED, {"global_error": error}) - finish_attack_paths_scan( - attack_paths_scan, - StateChoices.FAILED, - {"global_error": error}, - ) + recover_graph_data_ready(fresh) diff --git a/api/src/backend/tasks/jobs/attack_paths/findings.py b/api/src/backend/tasks/jobs/attack_paths/findings.py index 468f805cdd..3581f0ca0f 100644 --- a/api/src/backend/tasks/jobs/attack_paths/findings.py +++ b/api/src/backend/tasks/jobs/attack_paths/findings.py @@ -5,136 +5,107 @@ This module handles: - Adding resource labels to Cartography nodes for efficient lookups - Loading Prowler findings into the graph - Linking findings to resources -- Cleaning up stale findings """ from collections import defaultdict -from dataclasses import asdict, dataclass, fields -from typing import Any, Generator +from typing import Any, Callable, Generator from uuid import UUID import neo4j from cartography.config import Config as CartographyConfig from celery.utils.log import get_task_logger +from tasks.jobs.attack_paths.config import ( + BATCH_SIZE, + FINDINGS_BATCH_SIZE, + get_node_uid_field, + get_provider_resource_label, + get_root_node_label, + get_short_uid_extractor, +) +from tasks.jobs.attack_paths.queries import ( + ADD_RESOURCE_LABEL_TEMPLATE, + INSERT_FINDING_TEMPLATE, + render_cypher_template, +) from api.db_router import READ_REPLICA_ALIAS from api.db_utils import rls_transaction from api.models import Finding as FindingModel from api.models import Provider, ResourceFindingMapping from prowler.config import config as ProwlerConfig -from tasks.jobs.attack_paths.config import ( - BATCH_SIZE, - get_deprecated_provider_resource_label, - get_node_uid_field, - get_provider_resource_label, - get_root_node_label, -) -from tasks.jobs.attack_paths.indexes import IndexType, create_indexes -from tasks.jobs.attack_paths.queries import ( - ADD_RESOURCE_LABEL_TEMPLATE, - CLEANUP_FINDINGS_TEMPLATE, - INSERT_FINDING_TEMPLATE, - render_cypher_template, -) logger = get_task_logger(__name__) -# Type Definitions -# ----------------- - -# Maps dataclass field names to Django ORM query field names -_DB_FIELD_MAP: dict[str, str] = { - "check_title": "check_metadata__checktitle", -} +# Django ORM field names for `.values()` queries +# Most map 1:1 to Neo4j property names, exceptions are remapped in `_to_neo4j_dict` +_DB_QUERY_FIELDS = [ + "id", + "uid", + "inserted_at", + "updated_at", + "first_seen_at", + "scan_id", + "delta", + "status", + "status_extended", + "severity", + "check_id", + "check_metadata__checktitle", + "muted", + "muted_reason", +] -@dataclass(slots=True) -class Finding: - """ - Finding data for Neo4j ingestion. - - Can be created from a Django .values() query result using from_db_record(). - """ - - id: str - uid: str - inserted_at: str - updated_at: str - first_seen_at: str - scan_id: str - delta: str - status: str - status_extended: str - severity: str - check_id: str - check_title: str - muted: bool - muted_reason: str | None - resource_uid: str | None = None - - @classmethod - def get_db_query_fields(cls) -> tuple[str, ...]: - """Get field names for Django .values() query.""" - return tuple( - _DB_FIELD_MAP.get(f.name, f.name) - for f in fields(cls) - if f.name != "resource_uid" - ) - - @classmethod - def from_db_record(cls, record: dict[str, Any], resource_uid: str) -> "Finding": - """Create a Finding from a Django .values() query result.""" - return cls( - id=str(record["id"]), - uid=record["uid"], - inserted_at=record["inserted_at"], - updated_at=record["updated_at"], - first_seen_at=record["first_seen_at"], - scan_id=str(record["scan_id"]), - delta=record["delta"], - status=record["status"], - status_extended=record["status_extended"], - severity=record["severity"], - check_id=str(record["check_id"]), - check_title=record["check_metadata__checktitle"], - muted=record["muted"], - muted_reason=record["muted_reason"], - resource_uid=resource_uid, - ) - - def to_dict(self) -> dict[str, Any]: - """Convert to dict for Neo4j ingestion.""" - return asdict(self) +def _to_neo4j_dict( + record: dict[str, Any], resource_uid: str, resource_short_uid: str +) -> dict[str, Any]: + """Transform a Django `.values()` record into a `dict` ready for Neo4j ingestion.""" + return { + "id": str(record["id"]), + "uid": record["uid"], + "inserted_at": record["inserted_at"], + "updated_at": record["updated_at"], + "first_seen_at": record["first_seen_at"], + "scan_id": str(record["scan_id"]), + "delta": record["delta"], + "status": record["status"], + "status_extended": record["status_extended"], + "severity": record["severity"], + "check_id": str(record["check_id"]), + "check_title": record["check_metadata__checktitle"], + "muted": record["muted"], + "muted_reason": record["muted_reason"], + "resource_uid": resource_uid, + "resource_short_uid": resource_short_uid, + } # Public API # ---------- -def create_findings_indexes(neo4j_session: neo4j.Session) -> None: - """Create indexes for Prowler findings and resource lookups.""" - create_indexes(neo4j_session, IndexType.FINDINGS) - - def analysis( neo4j_session: neo4j.Session, prowler_api_provider: Provider, scan_id: str, config: CartographyConfig, -) -> None: +) -> tuple[int, int]: """ Main entry point for Prowler findings analysis. - Adds resource labels, loads findings, and cleans up stale data. + Adds resource labels and loads findings. + Returns (labeled_nodes, findings_loaded). """ - add_resource_label( + total_labeled = add_resource_label( neo4j_session, prowler_api_provider.provider, str(prowler_api_provider.uid) ) findings_data = stream_findings_with_resources(prowler_api_provider, scan_id) - load_findings(neo4j_session, findings_data, prowler_api_provider, config) - cleanup_findings(neo4j_session, prowler_api_provider, config) + total_loaded = load_findings( + neo4j_session, findings_data, prowler_api_provider, config + ) + return total_labeled, total_loaded def add_resource_label( @@ -153,9 +124,6 @@ def add_resource_label( { "__ROOT_LABEL__": get_root_node_label(provider_type), "__RESOURCE_LABEL__": get_provider_resource_label(provider_type), - "__DEPRECATED_RESOURCE_LABEL__": get_deprecated_provider_resource_label( - provider_type - ), }, ) @@ -184,15 +152,14 @@ def add_resource_label( def load_findings( neo4j_session: neo4j.Session, - findings_batches: Generator[list[Finding], None, None], + findings_batches: Generator[list[dict[str, Any]], None, None], prowler_api_provider: Provider, config: CartographyConfig, -) -> None: +) -> int: """Load Prowler findings into the graph, linking them to resources.""" query = render_cypher_template( INSERT_FINDING_TEMPLATE, { - "__ROOT_NODE_LABEL__": get_root_node_label(prowler_api_provider.provider), "__NODE_UID_FIELD__": get_node_uid_field(prowler_api_provider.provider), "__RESOURCE_LABEL__": get_provider_resource_label( prowler_api_provider.provider @@ -201,47 +168,32 @@ def load_findings( ) parameters = { - "provider_uid": str(prowler_api_provider.uid), "last_updated": config.update_tag, "prowler_version": ProwlerConfig.prowler_version, } batch_num = 0 total_records = 0 + edges_merged = 0 + edges_dropped = 0 for batch in findings_batches: batch_num += 1 batch_size = len(batch) total_records += batch_size - parameters["findings_data"] = [f.to_dict() for f in batch] + parameters["findings_data"] = batch logger.info(f"Loading findings batch {batch_num} ({batch_size} records)") - neo4j_session.run(query, parameters) + summary = neo4j_session.run(query, parameters).single() + if summary is not None: + edges_merged += summary.get("merged_count", 0) + edges_dropped += summary.get("dropped_count", 0) - logger.info(f"Finished loading {total_records} records in {batch_num} batches") - - -def cleanup_findings( - neo4j_session: neo4j.Session, - prowler_api_provider: Provider, - config: CartographyConfig, -) -> None: - """Remove stale findings (classic Cartography behaviour).""" - parameters = { - "provider_uid": str(prowler_api_provider.uid), - "last_updated": config.update_tag, - "batch_size": BATCH_SIZE, - } - - batch = 1 - deleted_count = 1 - while deleted_count > 0: - logger.info(f"Cleaning findings batch {batch}") - - result = neo4j_session.run(CLEANUP_FINDINGS_TEMPLATE, parameters) - - deleted_count = result.single().get("deleted_findings_count", 0) - batch += 1 + logger.info( + f"Finished loading {total_records} records in {batch_num} batches " + f"(edges_merged={edges_merged}, edges_dropped={edges_dropped})" + ) + return total_records # Findings Streaming (Generator-based) @@ -251,21 +203,23 @@ def cleanup_findings( def stream_findings_with_resources( prowler_api_provider: Provider, scan_id: str, -) -> Generator[list[Finding], None, None]: +) -> Generator[list[dict[str, Any]], None, None]: """ Stream findings with their associated resources in batches. Uses keyset pagination for efficient traversal of large datasets. - Memory efficient: yields one batch at a time, never holds all findings in memory. + Memory efficient: yields one batch at a time as dicts ready for Neo4j ingestion, + never holds all findings in memory. """ logger.info( f"Starting findings stream for scan {scan_id} " - f"(tenant {prowler_api_provider.tenant_id}) with batch size {BATCH_SIZE}" + f"(tenant {prowler_api_provider.tenant_id}) with batch size {FINDINGS_BATCH_SIZE}" ) tenant_id = prowler_api_provider.tenant_id + short_uid_extractor = get_short_uid_extractor(prowler_api_provider.provider) for batch in _paginate_findings(tenant_id, scan_id): - enriched = _enrich_batch_with_resources(batch, tenant_id) + enriched = _enrich_batch_with_resources(batch, tenant_id, short_uid_extractor) if enriched: yield enriched @@ -309,15 +263,16 @@ def _fetch_findings_batch( Uses read replica and RLS-scoped transaction. """ with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): - # Use all_objects to avoid the ActiveProviderManager's implicit JOIN - # through Scan -> Provider (to check is_deleted=False). - # The provider is already validated as active in this context. - qs = FindingModel.all_objects.filter(scan_id=scan_id).order_by("id") + # Use `all_objects` to get `Findings` even on soft-deleted `Providers` + # But even the provider is already validated as active in this context + qs = FindingModel.all_objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).order_by("id") if after_id is not None: qs = qs.filter(id__gt=after_id) - return list(qs.values(*Finding.get_db_query_fields())[:BATCH_SIZE]) + return list(qs.values(*_DB_QUERY_FIELDS)[:FINDINGS_BATCH_SIZE]) # Batch Enrichment @@ -327,7 +282,8 @@ def _fetch_findings_batch( def _enrich_batch_with_resources( findings_batch: list[dict[str, Any]], tenant_id: str, -) -> list[Finding]: + short_uid_extractor: Callable[[str], str], +) -> list[dict[str, Any]]: """ Enrich findings with their resource UIDs. @@ -338,7 +294,7 @@ def _enrich_batch_with_resources( resource_map = _build_finding_resource_map(finding_ids, tenant_id) return [ - Finding.from_db_record(finding, resource_uid) + _to_neo4j_dict(finding, resource_uid, short_uid_extractor(resource_uid)) for finding in findings_batch for resource_uid in resource_map.get(finding["id"], []) ] diff --git a/api/src/backend/tasks/jobs/attack_paths/indexes.py b/api/src/backend/tasks/jobs/attack_paths/indexes.py index 69edfe6719..0de94a162e 100644 --- a/api/src/backend/tasks/jobs/attack_paths/indexes.py +++ b/api/src/backend/tasks/jobs/attack_paths/indexes.py @@ -1,38 +1,25 @@ -from enum import Enum - import neo4j from cartography.client.core.tx import run_write_query from celery.utils.log import get_task_logger from tasks.jobs.attack_paths.config import ( - DEPRECATED_PROVIDER_RESOURCE_LABEL, INTERNET_NODE_LABEL, PROWLER_FINDING_LABEL, + PROVIDER_ELEMENT_ID_PROPERTY, PROVIDER_RESOURCE_LABEL, ) logger = get_task_logger(__name__) -class IndexType(Enum): - """Types of indexes that can be created.""" - - FINDINGS = "findings" - SYNC = "sync" - - -# Indexes for Prowler findings and resource lookups +# Indexes for Prowler Findings and resource lookups FINDINGS_INDEX_STATEMENTS = [ # Resource indexes for Prowler Finding lookups "CREATE INDEX aws_resource_arn IF NOT EXISTS FOR (n:_AWSResource) ON (n.arn);", "CREATE INDEX aws_resource_id IF NOT EXISTS FOR (n:_AWSResource) ON (n.id);", - "CREATE INDEX deprecated_aws_resource_arn IF NOT EXISTS FOR (n:AWSResource) ON (n.arn);", - "CREATE INDEX deprecated_aws_resource_id IF NOT EXISTS FOR (n:AWSResource) ON (n.id);", # Prowler Finding indexes f"CREATE INDEX prowler_finding_id IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.id);", - f"CREATE INDEX prowler_finding_provider_uid IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.provider_uid);", - f"CREATE INDEX prowler_finding_lastupdated IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.lastupdated);", f"CREATE INDEX prowler_finding_status IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.status);", # Internet node index for MERGE lookups f"CREATE INDEX internet_id IF NOT EXISTS FOR (n:{INTERNET_NODE_LABEL}) ON (n.id);", @@ -40,33 +27,19 @@ FINDINGS_INDEX_STATEMENTS = [ # Indexes for provider resource sync operations SYNC_INDEX_STATEMENTS = [ - f"CREATE INDEX provider_element_id IF NOT EXISTS FOR (n:{PROVIDER_RESOURCE_LABEL}) ON (n._provider_element_id);", - f"CREATE INDEX provider_resource_provider_id IF NOT EXISTS FOR (n:{PROVIDER_RESOURCE_LABEL}) ON (n._provider_id);", - f"CREATE INDEX deprecated_provider_element_id IF NOT EXISTS FOR (n:{DEPRECATED_PROVIDER_RESOURCE_LABEL}) ON (n.provider_element_id);", - f"CREATE INDEX deprecated_provider_resource_provider_id IF NOT EXISTS FOR (n:{DEPRECATED_PROVIDER_RESOURCE_LABEL}) ON (n.provider_id);", + f"CREATE INDEX provider_resource_element_id IF NOT EXISTS FOR (n:{PROVIDER_RESOURCE_LABEL}) ON (n.{PROVIDER_ELEMENT_ID_PROPERTY});", ] -def create_indexes(neo4j_session: neo4j.Session, index_type: IndexType) -> None: - """ - Create indexes for the specified type. - - Args: - `neo4j_session`: The Neo4j session to use - `index_type`: The type of indexes to create (FINDINGS or SYNC) - """ - if index_type == IndexType.FINDINGS: - logger.info("Creating indexes for Prowler Findings node types") - for statement in FINDINGS_INDEX_STATEMENTS: - run_write_query(neo4j_session, statement) - - elif index_type == IndexType.SYNC: - logger.info("Ensuring ProviderResource indexes exist") - for statement in SYNC_INDEX_STATEMENTS: - neo4j_session.run(statement) +def create_findings_indexes(neo4j_session: neo4j.Session) -> None: + """Create indexes for Prowler findings and resource lookups.""" + logger.info("Creating indexes for Prowler Findings node types") + for statement in FINDINGS_INDEX_STATEMENTS: + run_write_query(neo4j_session, statement) -def create_all_indexes(neo4j_session: neo4j.Session) -> None: - """Create all indexes (both findings and sync).""" - create_indexes(neo4j_session, IndexType.FINDINGS) - create_indexes(neo4j_session, IndexType.SYNC) +def create_sync_indexes(neo4j_session: neo4j.Session) -> None: + """Create indexes for provider resource sync operations.""" + logger.info("Ensuring ProviderResource indexes exist") + for statement in SYNC_INDEX_STATEMENTS: + neo4j_session.run(statement) diff --git a/api/src/backend/tasks/jobs/attack_paths/queries.py b/api/src/backend/tasks/jobs/attack_paths/queries.py index 4eada6684f..eb1d82a96e 100644 --- a/api/src/backend/tasks/jobs/attack_paths/queries.py +++ b/api/src/backend/tasks/jobs/attack_paths/queries.py @@ -2,6 +2,7 @@ from tasks.jobs.attack_paths.config import ( INTERNET_NODE_LABEL, PROWLER_FINDING_LABEL, + PROVIDER_ELEMENT_ID_PROPERTY, PROVIDER_RESOURCE_LABEL, ) @@ -26,70 +27,64 @@ ADD_RESOURCE_LABEL_TEMPLATE = """ MATCH (account:__ROOT_LABEL__ {id: $provider_uid})-->(r) WHERE NOT r:__ROOT_LABEL__ AND NOT r:__RESOURCE_LABEL__ WITH r LIMIT $batch_size - SET r:__RESOURCE_LABEL__:__DEPRECATED_RESOURCE_LABEL__ + SET r:__RESOURCE_LABEL__ RETURN COUNT(r) AS labeled_count """ INSERT_FINDING_TEMPLATE = f""" - MATCH (account:__ROOT_NODE_LABEL__ {{id: $provider_uid}}) UNWIND $findings_data AS finding_data - OPTIONAL MATCH (account)-->(resource_by_uid:__RESOURCE_LABEL__) - WHERE resource_by_uid.__NODE_UID_FIELD__ = finding_data.resource_uid - WITH account, finding_data, resource_by_uid - - OPTIONAL MATCH (account)-->(resource_by_id:__RESOURCE_LABEL__) + OPTIONAL MATCH (resource_by_uid:__RESOURCE_LABEL__ {{__NODE_UID_FIELD__: finding_data.resource_uid}}) + OPTIONAL MATCH (resource_by_id:__RESOURCE_LABEL__ {{id: finding_data.resource_uid}}) WHERE resource_by_uid IS NULL - AND resource_by_id.id = finding_data.resource_uid - WITH account, finding_data, COALESCE(resource_by_uid, resource_by_id) AS resource - WHERE resource IS NOT NULL + OPTIONAL MATCH (resource_by_short:__RESOURCE_LABEL__ {{id: finding_data.resource_short_uid}}) + WHERE resource_by_uid IS NULL AND resource_by_id IS NULL + WITH finding_data, + resource_by_uid, + resource_by_id, + head(collect(resource_by_short)) AS resource_by_short + WITH finding_data, + COALESCE(resource_by_uid, resource_by_id, resource_by_short) AS resource - MERGE (finding:{PROWLER_FINDING_LABEL} {{id: finding_data.id}}) - ON CREATE SET - finding.id = finding_data.id, - finding.uid = finding_data.uid, - finding.inserted_at = finding_data.inserted_at, - finding.updated_at = finding_data.updated_at, - finding.first_seen_at = finding_data.first_seen_at, - finding.scan_id = finding_data.scan_id, - finding.delta = finding_data.delta, - finding.status = finding_data.status, - finding.status_extended = finding_data.status_extended, - finding.severity = finding_data.severity, - finding.check_id = finding_data.check_id, - finding.check_title = finding_data.check_title, - finding.muted = finding_data.muted, - finding.muted_reason = finding_data.muted_reason, - finding.provider_uid = $provider_uid, - finding.firstseen = timestamp(), - finding.lastupdated = $last_updated, - finding._module_name = 'cartography:prowler', - finding._module_version = $prowler_version - ON MATCH SET - finding.status = finding_data.status, - finding.status_extended = finding_data.status_extended, - finding.lastupdated = $last_updated + FOREACH (_ IN CASE WHEN resource IS NOT NULL THEN [1] ELSE [] END | + MERGE (finding:{PROWLER_FINDING_LABEL} {{id: finding_data.id}}) + ON CREATE SET + finding.id = finding_data.id, + finding.uid = finding_data.uid, + finding.inserted_at = finding_data.inserted_at, + finding.updated_at = finding_data.updated_at, + finding.first_seen_at = finding_data.first_seen_at, + finding.scan_id = finding_data.scan_id, + finding.delta = finding_data.delta, + finding.status = finding_data.status, + finding.status_extended = finding_data.status_extended, + finding.severity = finding_data.severity, + finding.check_id = finding_data.check_id, + finding.check_title = finding_data.check_title, + finding.muted = finding_data.muted, + finding.muted_reason = finding_data.muted_reason, + finding.firstseen = timestamp(), + finding.lastupdated = $last_updated, + finding._module_name = 'cartography:prowler', + finding._module_version = $prowler_version + ON MATCH SET + finding.status = finding_data.status, + finding.status_extended = finding_data.status_extended, + finding.lastupdated = $last_updated + MERGE (resource)-[rel:HAS_FINDING]->(finding) + ON CREATE SET + rel.firstseen = timestamp(), + rel.lastupdated = $last_updated, + rel._module_name = 'cartography:prowler', + rel._module_version = $prowler_version + ON MATCH SET + rel.lastupdated = $last_updated + ) - MERGE (resource)-[rel:HAS_FINDING]->(finding) - ON CREATE SET - rel.provider_uid = $provider_uid, - rel.firstseen = timestamp(), - rel.lastupdated = $last_updated, - rel._module_name = 'cartography:prowler', - rel._module_version = $prowler_version - ON MATCH SET - rel.lastupdated = $last_updated -""" + WITH sum(CASE WHEN resource IS NOT NULL THEN 1 ELSE 0 END) AS merged_count, + sum(CASE WHEN resource IS NULL THEN 1 ELSE 0 END) AS dropped_count -CLEANUP_FINDINGS_TEMPLATE = f""" - MATCH (finding:{PROWLER_FINDING_LABEL} {{provider_uid: $provider_uid}}) - WHERE finding.lastupdated < $last_updated - - WITH finding LIMIT $batch_size - - DETACH DELETE finding - - RETURN COUNT(finding) AS deleted_findings_count + RETURN merged_count, dropped_count """ # Internet queries (used by internet.py) @@ -149,22 +144,16 @@ RELATIONSHIPS_FETCH_QUERY = """ LIMIT $batch_size """ -NODE_SYNC_TEMPLATE = """ +NODE_SYNC_TEMPLATE = f""" UNWIND $rows AS row - MERGE (n:__NODE_LABELS__ {_provider_element_id: row.provider_element_id}) + MERGE (n:__NODE_LABELS__ {{{PROVIDER_ELEMENT_ID_PROPERTY}: row.provider_element_id}}) SET n += row.props - SET n._provider_id = $provider_id - SET n.provider_element_id = row.provider_element_id - SET n.provider_id = $provider_id -""" # The last two lines are deprecated properties +""" RELATIONSHIP_SYNC_TEMPLATE = f""" UNWIND $rows AS row - MATCH (s:{PROVIDER_RESOURCE_LABEL} {{_provider_element_id: row.start_element_id}}) - MATCH (t:{PROVIDER_RESOURCE_LABEL} {{_provider_element_id: row.end_element_id}}) - MERGE (s)-[r:__REL_TYPE__ {{_provider_element_id: row.provider_element_id}}]->(t) + MATCH (s:{PROVIDER_RESOURCE_LABEL} {{{PROVIDER_ELEMENT_ID_PROPERTY}: row.start_element_id}}) + MATCH (t:{PROVIDER_RESOURCE_LABEL} {{{PROVIDER_ELEMENT_ID_PROPERTY}: row.end_element_id}}) + MERGE (s)-[r:__REL_TYPE__ {{{PROVIDER_ELEMENT_ID_PROPERTY}: row.provider_element_id}}]->(t) SET r += row.props - SET r._provider_id = $provider_id - SET r.provider_element_id = row.provider_element_id - SET r.provider_id = $provider_id -""" # The last two lines are deprecated properties +""" diff --git a/api/src/backend/tasks/jobs/attack_paths/scan.py b/api/src/backend/tasks/jobs/attack_paths/scan.py index f12736d807..452dae00d0 100644 --- a/api/src/backend/tasks/jobs/attack_paths/scan.py +++ b/api/src/backend/tasks/jobs/attack_paths/scan.py @@ -38,12 +38,12 @@ Pipeline steps: Stale findings from previous scans are cleaned up. 7. Sync the temp database into the tenant database: - - Drop the old provider subgraph (matched by _provider_id property). + - Drop the old provider subgraph (matched by dynamic _Provider_{uuid} label). graph_data_ready is set to False for all scans of this provider while the swap happens so the API doesn't serve partial data. - Copy nodes and relationships in batches. Every synced node gets a - _ProviderResource label and _provider_id / _provider_element_id - properties for multi-provider isolation. + _ProviderResource label and dynamic _Tenant_{uuid} / _Provider_{uuid} + isolation labels, plus a _provider_element_id property for MERGE keys. - Set graph_data_ready back to True. 8. Drop the temporary database, mark the AttackPathsScan as COMPLETED. @@ -63,16 +63,14 @@ from cartography.intel import analysis as cartography_analysis from cartography.intel import create_indexes as cartography_create_indexes from cartography.intel import ontology as cartography_ontology from celery.utils.log import get_task_logger +from tasks.jobs.attack_paths import db_utils, findings, indexes, internet, sync, utils +from tasks.jobs.attack_paths.config import get_cartography_ingestion_function from api.attack_paths import database as graph_database from api.db_utils import rls_transaction -from api.models import ( - Provider as ProwlerAPIProvider, - StateChoices, -) +from api.models import Provider as ProwlerAPIProvider +from api.models import StateChoices from api.utils import initialize_prowler_provider -from tasks.jobs.attack_paths import db_utils, findings, internet, sync, utils -from tasks.jobs.attack_paths.config import get_cartography_ingestion_function # Without this Celery goes crazy with Cartography logging logging.getLogger("cartography").setLevel(logging.ERROR) @@ -99,6 +97,19 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: ) attack_paths_scan = db_utils.retrieve_attack_paths_scan(tenant_id, scan_id) + # Idempotency guard: cleanup may have flipped this row to a terminal state + # while the message was still in flight. Bail out before touching state. + if attack_paths_scan and attack_paths_scan.state in ( + StateChoices.FAILED, + StateChoices.COMPLETED, + StateChoices.CANCELLED, + ): + logger.warning( + f"Attack Paths scan {attack_paths_scan.id} already in terminal " + f"state {attack_paths_scan.state}; skipping execution" + ) + return {} + # Checks before starting the scan if not cartography_ingestion_function: ingestion_exceptions = { @@ -116,12 +127,17 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: else: if not attack_paths_scan: + # Safety net for in-flight messages or direct task invocations; dispatcher normally pre-creates the row. logger.warning( f"No Attack Paths Scan found for scan {scan_id} and tenant {tenant_id}, let's create it then" ) attack_paths_scan = db_utils.create_attack_paths_scan( tenant_id, scan_id, prowler_api_provider.id ) + if attack_paths_scan and task_id: + db_utils.set_attack_paths_scan_task_id( + tenant_id, attack_paths_scan.id, task_id + ) tmp_database_name = graph_database.get_database_name( attack_paths_scan.id, temporary=True @@ -143,10 +159,24 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: ) # Starting the Attack Paths scan - db_utils.starting_attack_paths_scan( - attack_paths_scan, task_id, tenant_cartography_config + if not db_utils.starting_attack_paths_scan( + attack_paths_scan, tenant_cartography_config + ): + logger.warning( + f"Attack Paths scan {attack_paths_scan.id} no longer in SCHEDULED state; cleanup likely raced ahead" + ) + return {} + + scan_t0 = time.perf_counter() + logger.info( + f"Starting Attack Paths scan ({attack_paths_scan.id}) for " + f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}" ) + subgraph_dropped = False + sync_completed = False + provider_gated = False + try: logger.info( f"Creating Neo4j database {tmp_cartography_config.neo4j_database} for tenant {prowler_api_provider.tenant_id}" @@ -164,10 +194,11 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: ) as tmp_neo4j_session: # Indexes creation cartography_create_indexes.run(tmp_neo4j_session, tmp_cartography_config) - findings.create_findings_indexes(tmp_neo4j_session) + indexes.create_findings_indexes(tmp_neo4j_session) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 2) # The real scan, where iterates over cloud services + t0 = time.perf_counter() ingestion_exceptions = utils.call_within_event_loop( cartography_ingestion_function, tmp_neo4j_session, @@ -176,19 +207,23 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: prowler_sdk_provider, attack_paths_scan, ) + logger.info( + f"Cartography ingestion completed in {time.perf_counter() - t0:.3f}s " + f"(failed_syncs={len(ingestion_exceptions)})" + ) # Post-processing: Just keeping it to be more Cartography compliant logger.info( f"Syncing Cartography ontology for AWS account {prowler_api_provider.uid}" ) cartography_ontology.run(tmp_neo4j_session, tmp_cartography_config) - db_utils.update_attack_paths_scan_progress(attack_paths_scan, 95) + db_utils.update_attack_paths_scan_progress(attack_paths_scan, 94) logger.info( f"Syncing Cartography analysis for AWS account {prowler_api_provider.uid}" ) cartography_analysis.run(tmp_neo4j_session, tmp_cartography_config) - db_utils.update_attack_paths_scan_progress(attack_paths_scan, 96) + db_utils.update_attack_paths_scan_progress(attack_paths_scan, 95) # Creating Internet node and CAN_ACCESS relationships logger.info( @@ -197,14 +232,20 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: internet.analysis( tmp_neo4j_session, prowler_api_provider, tmp_cartography_config ) + db_utils.update_attack_paths_scan_progress(attack_paths_scan, 96) # Adding Prowler Finding nodes and relationships logger.info( f"Syncing Prowler analysis for AWS account {prowler_api_provider.uid}" ) - findings.analysis( + t0 = time.perf_counter() + labeled_nodes, findings_loaded = findings.analysis( tmp_neo4j_session, prowler_api_provider, scan_id, tmp_cartography_config ) + logger.info( + f"Prowler analysis completed in {time.perf_counter() - t0:.3f}s " + f"(findings={findings_loaded}, labeled_nodes={labeled_nodes})" + ) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 97) logger.info( @@ -220,42 +261,56 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: cartography_create_indexes.run( tenant_neo4j_session, tenant_cartography_config ) - findings.create_findings_indexes(tenant_neo4j_session) - sync.create_sync_indexes(tenant_neo4j_session) + indexes.create_findings_indexes(tenant_neo4j_session) + indexes.create_sync_indexes(tenant_neo4j_session) logger.info(f"Deleting existing provider graph in {tenant_database_name}") db_utils.set_provider_graph_data_ready(attack_paths_scan, False) - graph_database.drop_subgraph( + provider_gated = True + + t0 = time.perf_counter() + deleted_nodes = graph_database.drop_subgraph( database=tenant_database_name, provider_id=str(prowler_api_provider.id), ) + logger.info( + f"Deleted existing provider graph in {time.perf_counter() - t0:.3f}s " + f"(deleted_nodes={deleted_nodes})" + ) + subgraph_dropped = True db_utils.update_attack_paths_scan_progress(attack_paths_scan, 98) logger.info( f"Syncing graph from {tmp_database_name} into {tenant_database_name}" ) - sync.sync_graph( + t0 = time.perf_counter() + sync_result = sync.sync_graph( source_database=tmp_database_name, target_database=tenant_database_name, + tenant_id=str(prowler_api_provider.tenant_id), provider_id=str(prowler_api_provider.id), ) + logger.info( + f"Synced graph in {time.perf_counter() - t0:.3f}s " + f"(nodes={sync_result['nodes']}, relationships={sync_result['relationships']})" + ) + sync_completed = True db_utils.set_graph_data_ready(attack_paths_scan, True) db_utils.update_attack_paths_scan_progress(attack_paths_scan, 99) logger.info(f"Clearing Neo4j cache for database {tenant_database_name}") graph_database.clear_cache(tenant_database_name) - logger.info( - f"Completed Cartography ({attack_paths_scan.id}) for " - f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}" - ) - logger.info(f"Dropping temporary Neo4j database {tmp_database_name}") graph_database.drop_database(tmp_database_name) db_utils.finish_attack_paths_scan( attack_paths_scan, StateChoices.COMPLETED, ingestion_exceptions ) + logger.info( + f"Attack Paths scan completed in {time.perf_counter() - scan_t0:.3f}s " + f"(state=completed, failed_syncs={len(ingestion_exceptions)})" + ) return ingestion_exceptions except Exception as e: @@ -263,23 +318,39 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]: logger.exception(exception_message) ingestion_exceptions["global_error"] = exception_message - # Handling databases changes + # Recover graph_data_ready based on how far the swap got. + # Partial drop (mid-batch failure) may leave `subgraph_dropped=False` + # with data partially deleted, so we prefer that over permanently blocked queries. + try: + if sync_completed: + db_utils.set_graph_data_ready(attack_paths_scan, True) + elif provider_gated and not subgraph_dropped: + db_utils.set_provider_graph_data_ready(attack_paths_scan, True) + + except Exception: + logger.error( + f"Failed to recover `graph_data_ready` for provider {attack_paths_scan.provider_id}", + exc_info=True, + ) + + # Dropping the temporary database if it still exists try: graph_database.drop_database(tmp_cartography_config.neo4j_database) except Exception as e: logger.error( - f"Failed to drop temporary Neo4j database {tmp_cartography_config.neo4j_database} during cleanup: {e}", + f"Failed to drop temporary Neo4j database `{tmp_cartography_config.neo4j_database}` during cleanup: {e}", exc_info=True, ) + # Set Attack Paths scan state to FAILED try: db_utils.finish_attack_paths_scan( attack_paths_scan, StateChoices.FAILED, ingestion_exceptions ) except Exception as e: logger.error( - f"Could not mark attack paths scan {attack_paths_scan.id} as FAILED (row may have been deleted): {e}", + f"Could not mark Attack Paths scan {attack_paths_scan.id} as `FAILED` (row may have been deleted): {e}", exc_info=True, ) diff --git a/api/src/backend/tasks/jobs/attack_paths/sync.py b/api/src/backend/tasks/jobs/attack_paths/sync.py index 2fc94ab540..f720a12e82 100644 --- a/api/src/backend/tasks/jobs/attack_paths/sync.py +++ b/api/src/backend/tasks/jobs/attack_paths/sync.py @@ -5,19 +5,20 @@ This module handles syncing graph data from temporary scan databases to the tenant database, adding provider isolation labels and properties. """ +import time + from collections import defaultdict from typing import Any +import neo4j from celery.utils.log import get_task_logger - -from api.attack_paths import database as graph_database from tasks.jobs.attack_paths.config import ( - BATCH_SIZE, - DEPRECATED_PROVIDER_RESOURCE_LABEL, PROVIDER_ISOLATION_PROPERTIES, PROVIDER_RESOURCE_LABEL, + SYNC_BATCH_SIZE, + get_provider_label, + get_tenant_label, ) -from tasks.jobs.attack_paths.indexes import IndexType, create_indexes from tasks.jobs.attack_paths.queries import ( NODE_FETCH_QUERY, NODE_SYNC_TEMPLATE, @@ -26,17 +27,15 @@ from tasks.jobs.attack_paths.queries import ( render_cypher_template, ) +from api.attack_paths import database as graph_database + logger = get_task_logger(__name__) -def create_sync_indexes(neo4j_session) -> None: - """Create indexes for provider resource sync operations.""" - create_indexes(neo4j_session, IndexType.SYNC) - - def sync_graph( source_database: str, target_database: str, + tenant_id: str, provider_id: str, ) -> dict[str, int]: """ @@ -45,6 +44,7 @@ def sync_graph( Args: `source_database`: The temporary scan database `target_database`: The tenant database + `tenant_id`: The tenant ID for isolation `provider_id`: The provider ID for isolation Returns: @@ -53,6 +53,7 @@ def sync_graph( nodes_synced = sync_nodes( source_database, target_database, + tenant_id, provider_id, ) relationships_synced = sync_relationships( @@ -70,67 +71,57 @@ def sync_graph( def sync_nodes( source_database: str, target_database: str, + tenant_id: str, provider_id: str, ) -> int: """ Sync nodes from source to target database. - Adds `_ProviderResource` label and `_provider_id` property to all nodes. + Adds `_ProviderResource` label and dynamic `_Tenant_{id}` and `_Provider_{id}` + isolation labels to all nodes. + + Source and target sessions are opened sequentially per batch to avoid + holding two Bolt connections simultaneously for the entire sync duration. """ + t0 = time.perf_counter() last_id = -1 total_synced = 0 - with ( - graph_database.get_session(source_database) as source_session, - graph_database.get_session(target_database) as target_session, - ): - while True: - rows = list( - source_session.run( - NODE_FETCH_QUERY, - {"last_id": last_id, "batch_size": BATCH_SIZE}, - ) + while True: + grouped: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) + batch_count = 0 + + with graph_database.get_session(source_database) as source_session: + result = source_session.run( + NODE_FETCH_QUERY, + {"last_id": last_id, "batch_size": SYNC_BATCH_SIZE}, ) + for record in result: + batch_count += 1 + last_id = record["internal_id"] + key, value = _node_to_sync_dict(record, provider_id) + grouped[key].append(value) - if not rows: - break - - last_id = rows[-1]["internal_id"] - - grouped: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) - for row in rows: - labels = tuple(sorted(set(row["labels"] or []))) - props = dict(row["props"] or {}) - _strip_internal_properties(props) - provider_element_id = f"{provider_id}:{row['element_id']}" - grouped[labels].append( - { - "provider_element_id": provider_element_id, - "props": props, - } - ) + if batch_count == 0: + break + with graph_database.get_session(target_database) as target_session: for labels, batch in grouped.items(): label_set = set(labels) label_set.add(PROVIDER_RESOURCE_LABEL) - label_set.add(DEPRECATED_PROVIDER_RESOURCE_LABEL) + label_set.add(get_tenant_label(tenant_id)) + label_set.add(get_provider_label(provider_id)) node_labels = ":".join(f"`{label}`" for label in sorted(label_set)) query = render_cypher_template( NODE_SYNC_TEMPLATE, {"__NODE_LABELS__": node_labels} ) - target_session.run( - query, - { - "rows": batch, - "provider_id": provider_id, - }, - ) + target_session.run(query, {"rows": batch}) - total_synced += len(rows) - logger.info( - f"Synced {total_synced} nodes from {source_database} to {target_database}" - ) + total_synced += batch_count + logger.info( + f"Synced {total_synced} nodes from {source_database} to {target_database} in {time.perf_counter() - t0:.3f}s" + ) return total_synced @@ -143,62 +134,76 @@ def sync_relationships( """ Sync relationships from source to target database. - Adds `_provider_id` property to all relationships. + Matches source and target nodes by `_provider_element_id` in the tenant database. + + Source and target sessions are opened sequentially per batch to avoid + holding two Bolt connections simultaneously for the entire sync duration. """ + t0 = time.perf_counter() last_id = -1 total_synced = 0 - with ( - graph_database.get_session(source_database) as source_session, - graph_database.get_session(target_database) as target_session, - ): - while True: - rows = list( - source_session.run( - RELATIONSHIPS_FETCH_QUERY, - {"last_id": last_id, "batch_size": BATCH_SIZE}, - ) + while True: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + batch_count = 0 + + with graph_database.get_session(source_database) as source_session: + result = source_session.run( + RELATIONSHIPS_FETCH_QUERY, + {"last_id": last_id, "batch_size": SYNC_BATCH_SIZE}, ) + for record in result: + batch_count += 1 + last_id = record["internal_id"] + key, value = _rel_to_sync_dict(record, provider_id) + grouped[key].append(value) - if not rows: - break - - last_id = rows[-1]["internal_id"] - - grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) - for row in rows: - props = dict(row["props"] or {}) - _strip_internal_properties(props) - rel_type = row["rel_type"] - grouped[rel_type].append( - { - "start_element_id": f"{provider_id}:{row['start_element_id']}", - "end_element_id": f"{provider_id}:{row['end_element_id']}", - "provider_element_id": f"{provider_id}:{rel_type}:{row['internal_id']}", - "props": props, - } - ) + if batch_count == 0: + break + with graph_database.get_session(target_database) as target_session: for rel_type, batch in grouped.items(): query = render_cypher_template( RELATIONSHIP_SYNC_TEMPLATE, {"__REL_TYPE__": rel_type} ) - target_session.run( - query, - { - "rows": batch, - "provider_id": provider_id, - }, - ) + target_session.run(query, {"rows": batch}) - total_synced += len(rows) - logger.info( - f"Synced {total_synced} relationships from {source_database} to {target_database}" - ) + total_synced += batch_count + logger.info( + f"Synced {total_synced} relationships from {source_database} to {target_database} in {time.perf_counter() - t0:.3f}s" + ) return total_synced +def _node_to_sync_dict( + record: neo4j.Record, provider_id: str +) -> tuple[tuple[str, ...], dict[str, Any]]: + """Transform a source node record into a (grouping_key, sync_dict) pair.""" + props = dict(record["props"] or {}) + _strip_internal_properties(props) + labels = tuple(sorted(set(record["labels"] or []))) + return labels, { + "provider_element_id": f"{provider_id}:{record['element_id']}", + "props": props, + } + + +def _rel_to_sync_dict( + record: neo4j.Record, provider_id: str +) -> tuple[str, dict[str, Any]]: + """Transform a source relationship record into a (grouping_key, sync_dict) pair.""" + props = dict(record["props"] or {}) + _strip_internal_properties(props) + rel_type = record["rel_type"] + return rel_type, { + "start_element_id": f"{provider_id}:{record['start_element_id']}", + "end_element_id": f"{provider_id}:{record['end_element_id']}", + "provider_element_id": f"{provider_id}:{rel_type}:{record['internal_id']}", + "props": props, + } + + def _strip_internal_properties(props: dict[str, Any]) -> None: """Remove provider isolation properties before the += spread in sync templates.""" for key in PROVIDER_ISOLATION_PROPERTIES: diff --git a/api/src/backend/tasks/jobs/backfill.py b/api/src/backend/tasks/jobs/backfill.py index ff43fb33b3..825dcb7ca8 100644 --- a/api/src/backend/tasks/jobs/backfill.py +++ b/api/src/backend/tasks/jobs/backfill.py @@ -297,12 +297,15 @@ def backfill_daily_severity_summaries(tenant_id: str, days: int = None): } -def backfill_scan_category_summaries(tenant_id: str, scan_id: str): +def aggregate_scan_category_summaries(tenant_id: str, scan_id: str): """ Backfill ScanCategorySummary for a completed scan. Aggregates category counts from all findings in the scan and creates one ScanCategorySummary row per (category, severity) combination. + Idempotent: re-runs replace the scan's existing rows so counts stay in + sync with `Finding.muted` updates triggered outside scan completion + (e.g. mute rules). Args: tenant_id: Target tenant UUID @@ -312,11 +315,6 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str): dict: Status indicating whether backfill was performed """ with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): - if ScanCategorySummary.objects.filter( - tenant_id=tenant_id, scan_id=scan_id - ).exists(): - return {"status": "already backfilled"} - if not Scan.objects.filter( tenant_id=tenant_id, id=scan_id, @@ -337,9 +335,6 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str): cache=category_counts, ) - if not category_counts: - return {"status": "no categories to backfill"} - category_summaries = [ ScanCategorySummary( tenant_id=tenant_id, @@ -353,20 +348,38 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str): for (category, severity), counts in category_counts.items() ] - with rls_transaction(tenant_id): - ScanCategorySummary.objects.bulk_create( - category_summaries, batch_size=500, ignore_conflicts=True - ) + if category_summaries: + with rls_transaction(tenant_id): + # Upsert so re-runs (post-mute reaggregation) don't trip + # `unique_category_severity_per_scan`; race-safe under concurrent writers. + ScanCategorySummary.objects.bulk_create( + category_summaries, + batch_size=500, + update_conflicts=True, + unique_fields=["tenant_id", "scan_id", "category", "severity"], + update_fields=[ + "total_findings", + "failed_findings", + "new_failed_findings", + ], + ) + + if not category_counts: + return {"status": "no categories to backfill"} return {"status": "backfilled", "categories_count": len(category_counts)} -def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str): +def aggregate_scan_resource_group_summaries(tenant_id: str, scan_id: str): """ Backfill ScanGroupSummary for a completed scan. Aggregates resource group counts from all findings in the scan and creates one ScanGroupSummary row per (resource_group, severity) combination. + Idempotent: re-runs replace the scan's existing rows so counts stay in + sync with `Finding.muted` updates triggered outside scan completion + (e.g. mute rules) and with resource-inventory views reading from this + table. Args: tenant_id: Target tenant UUID @@ -376,11 +389,6 @@ def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str): dict: Status indicating whether backfill was performed """ with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): - if ScanGroupSummary.objects.filter( - tenant_id=tenant_id, scan_id=scan_id - ).exists(): - return {"status": "already backfilled"} - if not Scan.objects.filter( tenant_id=tenant_id, id=scan_id, @@ -418,9 +426,6 @@ def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str): group_resources_cache=group_resources_cache, ) - if not resource_group_counts: - return {"status": "no resource groups to backfill"} - # Compute group-level resource counts (same value for all severity rows in a group) group_resource_counts = { grp: len(uids) for grp, uids in group_resources_cache.items() @@ -439,10 +444,25 @@ def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str): for (grp, severity), counts in resource_group_counts.items() ] - with rls_transaction(tenant_id): - ScanGroupSummary.objects.bulk_create( - resource_group_summaries, batch_size=500, ignore_conflicts=True - ) + if resource_group_summaries: + with rls_transaction(tenant_id): + # Upsert so re-runs (post-mute reaggregation) don't trip + # `unique_resource_group_severity_per_scan`; race-safe under concurrent writers. + ScanGroupSummary.objects.bulk_create( + resource_group_summaries, + batch_size=500, + update_conflicts=True, + unique_fields=["tenant_id", "scan_id", "resource_group", "severity"], + update_fields=[ + "total_findings", + "failed_findings", + "new_failed_findings", + "resources_count", + ], + ) + + if not resource_group_counts: + return {"status": "no resource groups to backfill"} return {"status": "backfilled", "resource_groups_count": len(resource_group_counts)} diff --git a/api/src/backend/tasks/jobs/export.py b/api/src/backend/tasks/jobs/export.py index 4b8498f7e7..3be9b81544 100644 --- a/api/src/backend/tasks/jobs/export.py +++ b/api/src/backend/tasks/jobs/export.py @@ -32,9 +32,13 @@ from prowler.lib.outputs.compliance.cis.cis_aws import AWSCIS from prowler.lib.outputs.compliance.cis.cis_azure import AzureCIS from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS from prowler.lib.outputs.compliance.cis.cis_github import GithubCIS +from prowler.lib.outputs.compliance.cis.cis_googleworkspace import GoogleWorkspaceCIS from prowler.lib.outputs.compliance.cis.cis_kubernetes import KubernetesCIS from prowler.lib.outputs.compliance.cis.cis_m365 import M365CIS from prowler.lib.outputs.compliance.cis.cis_oraclecloud import OracleCloudCIS +from prowler.lib.outputs.compliance.cisa_scuba.cisa_scuba_googleworkspace import ( + GoogleWorkspaceCISASCuBA, +) from prowler.lib.outputs.compliance.csa.csa_alibabacloud import AlibabaCloudCSA from prowler.lib.outputs.compliance.csa.csa_aws import AWSCSA from prowler.lib.outputs.compliance.csa.csa_azure import AzureCSA @@ -93,7 +97,7 @@ COMPLIANCE_CLASS_MAP = { (lambda name: name.startswith("iso27001_"), AWSISO27001), (lambda name: name.startswith("kisa"), AWSKISAISMSP), (lambda name: name == "prowler_threatscore_aws", ProwlerThreatScoreAWS), - (lambda name: name == "ccc_aws", CCC_AWS), + (lambda name: name.startswith("ccc_"), CCC_AWS), (lambda name: name.startswith("c5_"), AWSC5), (lambda name: name.startswith("csa_"), AWSCSA), ], @@ -102,7 +106,7 @@ COMPLIANCE_CLASS_MAP = { (lambda name: name == "mitre_attack_azure", AzureMitreAttack), (lambda name: name.startswith("ens_"), AzureENS), (lambda name: name.startswith("iso27001_"), AzureISO27001), - (lambda name: name == "ccc_azure", CCC_Azure), + (lambda name: name.startswith("ccc_"), CCC_Azure), (lambda name: name == "prowler_threatscore_azure", ProwlerThreatScoreAzure), (lambda name: name == "c5_azure", AzureC5), (lambda name: name.startswith("csa_"), AzureCSA), @@ -113,7 +117,7 @@ COMPLIANCE_CLASS_MAP = { (lambda name: name.startswith("ens_"), GCPENS), (lambda name: name.startswith("iso27001_"), GCPISO27001), (lambda name: name == "prowler_threatscore_gcp", ProwlerThreatScoreGCP), - (lambda name: name == "ccc_gcp", CCC_GCP), + (lambda name: name.startswith("ccc_"), CCC_GCP), (lambda name: name == "c5_gcp", GCPC5), (lambda name: name.startswith("csa_"), GCPCSA), ], @@ -133,6 +137,10 @@ COMPLIANCE_CLASS_MAP = { "github": [ (lambda name: name.startswith("cis_"), GithubCIS), ], + "googleworkspace": [ + (lambda name: name.startswith("cis_"), GoogleWorkspaceCIS), + (lambda name: name.startswith("cisa_scuba_"), GoogleWorkspaceCISASCuBA), + ], "iac": [ # IaC provider doesn't have specific compliance frameworks yet # Trivy handles its own compliance checks diff --git a/api/src/backend/tasks/jobs/report.py b/api/src/backend/tasks/jobs/report.py index a41a8d6292..12f48b7040 100644 --- a/api/src/backend/tasks/jobs/report.py +++ b/api/src/backend/tasks/jobs/report.py @@ -1,11 +1,19 @@ +import gc +import os +import re +import time +from collections.abc import Iterable from pathlib import Path from shutil import rmtree +from uuid import UUID +import fcntl from celery.utils.log import get_task_logger from config.django.base import DJANGO_TMP_OUTPUT_DIRECTORY from tasks.jobs.export import _generate_compliance_output_directory, _upload_to_s3 from tasks.jobs.reports import ( FRAMEWORK_REGISTRY, + CISReportGenerator, CSAReportGenerator, ENSReportGenerator, NIS2ReportGenerator, @@ -14,12 +22,398 @@ from tasks.jobs.reports import ( from tasks.jobs.threatscore import compute_threatscore_metrics from tasks.jobs.threatscore_utils import _aggregate_requirement_statistics_from_database -from api.db_router import READ_REPLICA_ALIAS +from api.db_router import READ_REPLICA_ALIAS, MainRouter from api.db_utils import rls_transaction -from api.models import Provider, ScanSummary, ThreatScoreSnapshot +from api.models import Provider, Scan, ScanSummary, StateChoices, ThreatScoreSnapshot +from prowler.lib.check.compliance_models import Compliance from prowler.lib.outputs.finding import Finding as FindingOutput logger = get_task_logger(__name__) +STALE_TMP_OUTPUT_MAX_AGE_HOURS = 48 +STALE_TMP_OUTPUT_MAX_DELETIONS_PER_RUN = 50 +STALE_TMP_OUTPUT_THROTTLE_SECONDS = 60 * 60 +STALE_TMP_OUTPUT_LOCK_FILE_NAME = ".stale_tmp_cleanup.lock" + +# Refuse to ever run rmtree against shared system roots; the configured +# DJANGO_TMP_OUTPUT_DIRECTORY must be a dedicated subdirectory. +_FORBIDDEN_CLEANUP_ROOTS = frozenset( + Path(p).resolve() + for p in ("/", "/tmp", "/var", "/var/tmp", "/home", "/root", "/etc", "/usr") +) + + +def _resolve_stale_tmp_safe_root() -> Path | None: + """Resolve the configured tmp output directory, rejecting unsafe roots.""" + try: + configured_root = Path(DJANGO_TMP_OUTPUT_DIRECTORY).resolve() + except OSError: + return None + if configured_root in _FORBIDDEN_CLEANUP_ROOTS: + return None + return configured_root + + +STALE_TMP_OUTPUT_SAFE_ROOT = _resolve_stale_tmp_safe_root() + +# Matches CIS compliance_ids like "cis_1.4_aws", "cis_5.0_azure", +# "cis_1.10_kubernetes", "cis_3.0.1_aws". Requires at least one dotted +# component so malformed inputs like "cis_._aws" or "cis_5._aws" are rejected +# at the regex stage, rather than by a later ValueError fallback. +_CIS_VARIANT_RE = re.compile(r"^cis_(?P\d+(?:\.\d+)+)_(?P.+)$") + + +def _pick_latest_cis_variant(compliance_ids: Iterable[str]) -> str | None: + """Return the CIS compliance_id with the highest semantic version. + + CIS ships many variants per provider (e.g. cis_1.4_aws, ..., cis_6.0_aws). + A lexicographic sort is incorrect for version strings like ``1.10`` vs + ``1.2``; this helper parses the version into a tuple of ints so ``1.10`` + is correctly ordered after ``1.2``. Malformed names are skipped so a + broken JSON cannot crash the whole CIS pipeline. + + Args: + compliance_ids: Iterable of CIS compliance identifiers. Expected to + belong to a single provider (callers should pass the already + filtered keys from ``Compliance.get_bulk(provider_type)``). + + Returns: + The compliance_id with the highest parsed version, or ``None`` if no + well-formed CIS identifier was found. + """ + best_key: tuple[int, ...] | None = None + best_name: str | None = None + for name in compliance_ids: + match = _CIS_VARIANT_RE.match(name) + if not match: + continue + try: + key = tuple(int(part) for part in match.group("version").split(".")) + except ValueError: + # Defensive: the regex already guarantees numeric chunks, but we + # keep the guard so a future regex change cannot crash callers. + continue + if best_key is None or key > best_key: + best_key = key + best_name = name + return best_name + + +def _should_run_stale_cleanup( + root_path: Path, + throttle_seconds: int = STALE_TMP_OUTPUT_THROTTLE_SECONDS, +) -> bool: + """Throttle stale cleanup to at most once per hour per host.""" + lock_file_path = root_path / STALE_TMP_OUTPUT_LOCK_FILE_NAME + now_timestamp = int(time.time()) + + try: + with lock_file_path.open("a+", encoding="ascii") as lock_file: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return False + lock_file.seek(0) + previous_value = lock_file.read().strip() + try: + last_run_timestamp = int(previous_value) if previous_value else 0 + except ValueError: + last_run_timestamp = 0 + + if now_timestamp - last_run_timestamp < throttle_seconds: + return False + + lock_file.seek(0) + lock_file.truncate() + lock_file.write(str(now_timestamp)) + lock_file.flush() + os.fsync(lock_file.fileno()) + except OSError as error: + logger.warning("Skipping stale tmp cleanup: lock file error (%s)", error) + return False + + return True + + +def _is_scan_metadata_protected( + scan_path: Path, + scan_state: str | None, + output_location: str | None, +) -> bool: + """ + Return True when metadata indicates the directory must not be deleted. + + Protected cases: + - Scan is still EXECUTING. + - Scan has a local output artifact path (non-S3) under this scan directory. + """ + if scan_state == StateChoices.EXECUTING.value: + return True + + output_location = output_location or "" + if output_location and not output_location.startswith("s3://"): + try: + resolved_output_location = Path(output_location).resolve() + except OSError: + # Conservative fallback: if we cannot resolve a local output path, + # keep the directory to avoid deleting potentially needed artifacts. + return True + + if ( + resolved_output_location == scan_path + or scan_path in resolved_output_location.parents + ): + return True + + return False + + +def _is_scan_directory_protected( + tenant_id: str, + scan_id: str, + scan_path: Path, +) -> bool: + """ + DB-backed wrapper used when batch metadata is not already available. + """ + try: + scan_uuid = UUID(scan_id) + except ValueError: + return False + + try: + scan = ( + Scan.all_objects.using(MainRouter.admin_db) + .filter(tenant_id=tenant_id, id=scan_uuid) + .only("state", "output_location") + .first() + ) + except Exception as error: + logger.warning( + "Skipping stale tmp cleanup for %s/%s due to scan lookup error: %s", + tenant_id, + scan_id, + error, + ) + return True + + if not scan: + return False + + return _is_scan_metadata_protected( + scan_path=scan_path, + scan_state=scan.state, + output_location=scan.output_location, + ) + + +def _cleanup_stale_tmp_output_directories( + tmp_output_root: str, + max_age_hours: int = STALE_TMP_OUTPUT_MAX_AGE_HOURS, + exclude_scan: tuple[str, str] | None = None, + max_deletions_per_run: int = STALE_TMP_OUTPUT_MAX_DELETIONS_PER_RUN, +) -> int: + """ + Opportunistically delete stale scan directories under the tmp output root. + + Expected directory layout: + ///... + + Each run that wins the per-host throttle sweeps every tenant directory so + leftover artifacts cannot pile up for tenants whose own tasks happen to + lose the throttle race. + + Args: + tmp_output_root: Base tmp output path. + max_age_hours: Directory max age before deletion. + exclude_scan: Optional (tenant_id, scan_id) that must never be deleted. + max_deletions_per_run: Max number of scan directories deleted per run. + + Returns: + Number of deleted scan directories. + """ + try: + if max_age_hours <= 0: + return 0 + + try: + root_path = Path(tmp_output_root).resolve() + except OSError as error: + logger.warning( + "Skipping stale tmp cleanup: unable to resolve %s (%s)", + tmp_output_root, + error, + ) + return 0 + + if ( + STALE_TMP_OUTPUT_SAFE_ROOT is None + or root_path != STALE_TMP_OUTPUT_SAFE_ROOT + ): + logger.warning( + "Skipping stale tmp cleanup: unsupported root %s (allowed: %s)", + root_path, + STALE_TMP_OUTPUT_SAFE_ROOT, + ) + return 0 + + if not root_path.exists() or not root_path.is_dir(): + return 0 + + if max_deletions_per_run <= 0: + return 0 + + if not _should_run_stale_cleanup(root_path): + return 0 + + cutoff_timestamp = time.time() - (max_age_hours * 60 * 60) + deleted_scan_dirs = 0 + + try: + tenant_dirs = list(root_path.iterdir()) + except OSError as error: + logger.warning( + "Skipping stale tmp cleanup: unable to list %s (%s)", + root_path, + error, + ) + return 0 + + for tenant_dir in tenant_dirs: + if deleted_scan_dirs >= max_deletions_per_run: + break + + if not tenant_dir.is_dir() or tenant_dir.is_symlink(): + continue + + try: + scan_dirs = list(tenant_dir.iterdir()) + except OSError: + continue + + stale_candidates: list[tuple[str, Path, UUID | None]] = [] + for scan_dir in scan_dirs: + if not scan_dir.is_dir() or scan_dir.is_symlink(): + continue + + if exclude_scan and ( + tenant_dir.name == exclude_scan[0] + and scan_dir.name == exclude_scan[1] + ): + continue + + try: + if scan_dir.stat().st_mtime >= cutoff_timestamp: + continue + except OSError: + continue + + try: + resolved_scan_dir = scan_dir.resolve() + except OSError: + continue + + if root_path not in resolved_scan_dir.parents: + logger.warning( + "Skipping stale tmp cleanup for path outside root: %s", + resolved_scan_dir, + ) + continue + + try: + scan_uuid: UUID | None = UUID(scan_dir.name) + except ValueError: + scan_uuid = None + + stale_candidates.append((scan_dir.name, resolved_scan_dir, scan_uuid)) + + if not stale_candidates: + continue + + scan_metadata_by_id: dict[UUID, tuple[str | None, str | None]] = {} + metadata_preload_succeeded = False + candidate_scan_ids = [ + candidate[2] for candidate in stale_candidates if candidate[2] + ] + if candidate_scan_ids: + try: + scan_rows = ( + Scan.all_objects.using(MainRouter.admin_db) + .filter( + tenant_id=tenant_dir.name, + id__in=candidate_scan_ids, + ) + .values_list("id", "state", "output_location") + ) + scan_metadata_by_id = { + scan_id: (scan_state, output_location) + for scan_id, scan_state, output_location in scan_rows + } + metadata_preload_succeeded = True + except Exception as error: + logger.warning( + "Skipping stale tmp cleanup metadata preload for tenant %s: %s", + tenant_dir.name, + error, + ) + else: + metadata_preload_succeeded = True + + for scan_name, resolved_scan_dir, scan_uuid in stale_candidates: + if deleted_scan_dirs >= max_deletions_per_run: + break + + should_check_scan_fallback = True + if scan_uuid and metadata_preload_succeeded: + should_check_scan_fallback = False + scan_metadata = scan_metadata_by_id.get(scan_uuid) + if scan_metadata: + scan_state, output_location = scan_metadata + if _is_scan_metadata_protected( + scan_path=resolved_scan_dir, + scan_state=scan_state, + output_location=output_location, + ): + continue + + if should_check_scan_fallback and _is_scan_directory_protected( + tenant_id=tenant_dir.name, + scan_id=scan_name, + scan_path=resolved_scan_dir, + ): + continue + + try: + rmtree(resolved_scan_dir, ignore_errors=True) + deleted_scan_dirs += 1 + except Exception as error: + logger.warning( + "Error cleaning stale tmp directory %s: %s", + resolved_scan_dir, + error, + ) + + if deleted_scan_dirs: + logger.info( + "Deleted %s stale tmp output directories older than %sh from %s", + deleted_scan_dirs, + max_age_hours, + root_path, + ) + if deleted_scan_dirs >= max_deletions_per_run: + logger.info( + "Stale tmp cleanup hit deletion limit (%s) for root %s", + max_deletions_per_run, + root_path, + ) + + return deleted_scan_dirs + except Exception as error: + logger.warning( + "Skipping stale tmp cleanup due to unexpected error: %s", + error, + exc_info=True, + ) + return 0 def generate_threatscore_report( @@ -191,6 +585,53 @@ def generate_csa_report( ) +def generate_cis_report( + tenant_id: str, + scan_id: str, + compliance_id: str, + output_path: str, + provider_id: str, + only_failed: bool = True, + include_manual: bool = False, + provider_obj: Provider | None = None, + requirement_statistics: dict[str, dict[str, int]] | None = None, + findings_cache: dict[str, list[FindingOutput]] | None = None, +) -> None: + """ + Generate a PDF compliance report for a specific CIS Benchmark variant. + + Unlike single-version frameworks (ENS, NIS2, CSA), CIS has multiple + variants per provider (e.g., cis_1.4_aws, cis_5.0_aws, cis_6.0_aws). This + wrapper is called once per variant, receiving the specific compliance_id. + + Args: + tenant_id: The tenant ID for Row-Level Security context. + scan_id: ID of the scan executed by Prowler. + compliance_id: ID of the specific CIS variant (e.g., "cis_5.0_aws"). + output_path: Output PDF file path. + provider_id: Provider ID for the scan. + only_failed: If True, only include failed requirements in detailed section. + include_manual: If True, include manual requirements in detailed section. + provider_obj: Pre-fetched Provider object to avoid duplicate queries. + requirement_statistics: Pre-aggregated requirement statistics. + findings_cache: Cache of already loaded findings to avoid duplicate queries. + """ + generator = CISReportGenerator(FRAMEWORK_REGISTRY["cis"]) + + generator.generate( + tenant_id=tenant_id, + scan_id=scan_id, + compliance_id=compliance_id, + output_path=output_path, + provider_id=provider_id, + provider_obj=provider_obj, + requirement_statistics=requirement_statistics, + findings_cache=findings_cache, + only_failed=only_failed, + include_manual=include_manual, + ) + + def generate_compliance_reports( tenant_id: str, scan_id: str, @@ -199,6 +640,7 @@ def generate_compliance_reports( generate_ens: bool = True, generate_nis2: bool = True, generate_csa: bool = True, + generate_cis: bool = True, only_failed_threatscore: bool = True, min_risk_level_threatscore: int = 4, include_manual_ens: bool = True, @@ -206,6 +648,8 @@ def generate_compliance_reports( only_failed_nis2: bool = True, only_failed_csa: bool = True, include_manual_csa: bool = False, + only_failed_cis: bool = True, + include_manual_cis: bool = False, ) -> dict[str, dict[str, bool | str]]: """ Generate multiple compliance reports with shared database queries. @@ -215,6 +659,13 @@ def generate_compliance_reports( - Aggregating requirement statistics once (shared across all reports) - Reusing compliance framework data when possible + For CIS a single PDF is produced per run: the one matching the highest + available CIS version for the scan's provider (picked dynamically from + ``Compliance.get_bulk`` via :func:`_pick_latest_cis_variant`). The + returned ``results["cis"]`` entry has the same flat shape as the other + single-version frameworks — the picked variant is an internal detail, + not surfaced in the result. + Args: tenant_id: The tenant ID for Row-Level Security context. scan_id: The ID of the scan to generate reports for. @@ -223,6 +674,8 @@ def generate_compliance_reports( generate_ens: Whether to generate ENS report. generate_nis2: Whether to generate NIS2 report. generate_csa: Whether to generate CSA CCM report. + generate_cis: Whether to generate a CIS Benchmark report for the + latest CIS version available for the provider. only_failed_threatscore: For ThreatScore, only include failed requirements. min_risk_level_threatscore: Minimum risk level for ThreatScore critical requirements. include_manual_ens: For ENS, include manual requirements. @@ -230,22 +683,39 @@ def generate_compliance_reports( only_failed_nis2: For NIS2, only include failed requirements. only_failed_csa: For CSA CCM, only include failed requirements. include_manual_csa: For CSA CCM, include manual requirements. + only_failed_cis: For CIS, only include failed requirements in detailed section. + include_manual_cis: For CIS, include manual requirements in detailed section. Returns: - Dictionary with results for each report type. + Dictionary with results for each report type. Every value has the + same flat shape: ``{"upload": bool, "path": str, "error"?: str}``. """ logger.info( "Generating compliance reports for scan %s with provider %s" - " (ThreatScore: %s, ENS: %s, NIS2: %s, CSA: %s)", + " (ThreatScore: %s, ENS: %s, NIS2: %s, CSA: %s, CIS: %s)", scan_id, provider_id, generate_threatscore, generate_ens, generate_nis2, generate_csa, + generate_cis, ) - results = {} + try: + _cleanup_stale_tmp_output_directories( + DJANGO_TMP_OUTPUT_DIRECTORY, + max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS, + exclude_scan=(tenant_id, scan_id), + ) + except Exception as error: + logger.warning( + "Skipping stale tmp cleanup before compliance reports for scan %s: %s", + scan_id, + error, + ) + + results: dict = {} # Validate that the scan has findings and get provider info with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): @@ -259,6 +729,8 @@ def generate_compliance_reports( results["nis2"] = {"upload": False, "path": ""} if generate_csa: results["csa"] = {"upload": False, "path": ""} + if generate_cis: + results["cis"] = {"upload": False, "path": ""} return results provider_obj = Provider.objects.get(id=provider_id) @@ -299,11 +771,39 @@ def generate_compliance_reports( results["csa"] = {"upload": False, "path": ""} generate_csa = False + # For CIS we do NOT pre-check the provider against a hard-coded whitelist + # (that list drifts the moment a new CIS JSON ships). Instead, we inspect + # the dynamically loaded framework map and pick the latest available CIS + # version, if any. + latest_cis: str | None = None + if generate_cis: + try: + frameworks_bulk = Compliance.get_bulk(provider_type) + latest_cis = _pick_latest_cis_variant( + name for name in frameworks_bulk.keys() if name.startswith("cis_") + ) + except Exception as e: + logger.error("Error discovering CIS variants for %s: %s", provider_type, e) + results["cis"] = {"upload": False, "path": "", "error": str(e)} + generate_cis = False + else: + if latest_cis is None: + logger.info("No CIS variants available for provider %s", provider_type) + results["cis"] = {"upload": False, "path": ""} + generate_cis = False + else: + logger.info( + "Selected latest CIS variant for provider %s: %s", + provider_type, + latest_cis, + ) + if ( not generate_threatscore and not generate_ens and not generate_nis2 and not generate_csa + and not generate_cis ): return results @@ -319,38 +819,56 @@ def generate_compliance_reports( findings_cache = {} logger.info("Created shared findings cache for all reports") - # Generate output directories + generated_report_keys: list[str] = [] + output_paths: dict[str, str] = {} + out_dir: str | None = None + + # Generate output directories only for enabled and supported report types. try: logger.info("Generating output directories") - threatscore_path = _generate_compliance_output_directory( - DJANGO_TMP_OUTPUT_DIRECTORY, - provider_uid, - tenant_id, - scan_id, - compliance_framework="threatscore", - ) - ens_path = _generate_compliance_output_directory( - DJANGO_TMP_OUTPUT_DIRECTORY, - provider_uid, - tenant_id, - scan_id, - compliance_framework="ens", - ) - nis2_path = _generate_compliance_output_directory( - DJANGO_TMP_OUTPUT_DIRECTORY, - provider_uid, - tenant_id, - scan_id, - compliance_framework="nis2", - ) - csa_path = _generate_compliance_output_directory( - DJANGO_TMP_OUTPUT_DIRECTORY, - provider_uid, - tenant_id, - scan_id, - compliance_framework="csa", - ) - out_dir = str(Path(threatscore_path).parent.parent) + if generate_threatscore: + output_paths["threatscore"] = _generate_compliance_output_directory( + DJANGO_TMP_OUTPUT_DIRECTORY, + provider_uid, + tenant_id, + scan_id, + compliance_framework="threatscore", + ) + if generate_ens: + output_paths["ens"] = _generate_compliance_output_directory( + DJANGO_TMP_OUTPUT_DIRECTORY, + provider_uid, + tenant_id, + scan_id, + compliance_framework="ens", + ) + if generate_nis2: + output_paths["nis2"] = _generate_compliance_output_directory( + DJANGO_TMP_OUTPUT_DIRECTORY, + provider_uid, + tenant_id, + scan_id, + compliance_framework="nis2", + ) + if generate_csa: + output_paths["csa"] = _generate_compliance_output_directory( + DJANGO_TMP_OUTPUT_DIRECTORY, + provider_uid, + tenant_id, + scan_id, + compliance_framework="csa", + ) + if generate_cis and latest_cis: + output_paths["cis"] = _generate_compliance_output_directory( + DJANGO_TMP_OUTPUT_DIRECTORY, + provider_uid, + tenant_id, + scan_id, + compliance_framework="cis", + ) + if output_paths: + first_output_path = next(iter(output_paths.values())) + out_dir = str(Path(first_output_path).parent.parent) except Exception as e: logger.error("Error generating output directory: %s", e) error_dict = {"error": str(e), "upload": False, "path": ""} @@ -362,10 +880,14 @@ def generate_compliance_reports( results["nis2"] = error_dict.copy() if generate_csa: results["csa"] = error_dict.copy() + if generate_cis: + results["cis"] = error_dict.copy() return results # Generate ThreatScore report if generate_threatscore: + generated_report_keys.append("threatscore") + threatscore_path = output_paths["threatscore"] compliance_id_threatscore = f"prowler_threatscore_{provider_type}" pdf_path_threatscore = f"{threatscore_path}_threatscore_report.pdf" logger.info( @@ -467,6 +989,8 @@ def generate_compliance_reports( # Generate ENS report if generate_ens: + generated_report_keys.append("ens") + ens_path = output_paths["ens"] compliance_id_ens = f"ens_rd2022_{provider_type}" pdf_path_ens = f"{ens_path}_ens_report.pdf" logger.info("Generating ENS report with compliance %s", compliance_id_ens) @@ -501,6 +1025,8 @@ def generate_compliance_reports( # Generate NIS2 report if generate_nis2: + generated_report_keys.append("nis2") + nis2_path = output_paths["nis2"] compliance_id_nis2 = f"nis2_{provider_type}" pdf_path_nis2 = f"{nis2_path}_nis2_report.pdf" logger.info("Generating NIS2 report with compliance %s", compliance_id_nis2) @@ -536,6 +1062,8 @@ def generate_compliance_reports( # Generate CSA CCM report if generate_csa: + generated_report_keys.append("csa") + csa_path = output_paths["csa"] compliance_id_csa = f"csa_ccm_4.0_{provider_type}" pdf_path_csa = f"{csa_path}_csa_report.pdf" logger.info("Generating CSA CCM report with compliance %s", compliance_id_csa) @@ -569,14 +1097,75 @@ def generate_compliance_reports( logger.error("Error generating CSA CCM report: %s", e) results["csa"] = {"upload": False, "path": "", "error": str(e)} - # Clean up temporary files if all reports were uploaded successfully - all_uploaded = all( - result.get("upload", False) - for result in results.values() - if result.get("upload") is not None + # Generate CIS Benchmark report for the latest available version only. + # CIS ships multiple versions per provider (e.g. cis_1.4_aws, cis_5.0_aws, + # cis_6.0_aws); we dynamically pick the highest semantic version at run + # time rather than hard-coding a per-provider mapping. + if generate_cis and latest_cis: + generated_report_keys.append("cis") + cis_path = output_paths["cis"] + if out_dir is None: + out_dir = str(Path(cis_path).parent.parent) + pdf_path_cis = f"{cis_path}_cis_report.pdf" + try: + generate_cis_report( + tenant_id=tenant_id, + scan_id=scan_id, + compliance_id=latest_cis, + output_path=pdf_path_cis, + provider_id=provider_id, + only_failed=only_failed_cis, + include_manual=include_manual_cis, + provider_obj=provider_obj, + requirement_statistics=requirement_statistics, + findings_cache=findings_cache, + ) + + upload_uri_cis = _upload_to_s3( + tenant_id, + scan_id, + pdf_path_cis, + f"cis/{Path(pdf_path_cis).name}", + ) + + if upload_uri_cis: + results["cis"] = { + "upload": True, + "path": upload_uri_cis, + } + logger.info( + "CIS report %s uploaded to %s", + latest_cis, + upload_uri_cis, + ) + else: + results["cis"] = {"upload": False, "path": out_dir} + logger.warning( + "CIS report %s saved locally at %s", + latest_cis, + out_dir, + ) + + except Exception as e: + logger.error("Error generating CIS report %s: %s", latest_cis, e) + results["cis"] = { + "upload": False, + "path": "", + "error": str(e), + } + finally: + # Free ReportLab/matplotlib memory before moving on. + gc.collect() + + # Clean up temporary files only if all generated reports were + # uploaded successfully. Reports skipped for provider incompatibility + # or missing CIS variants must not block cleanup. + all_uploaded = bool(generated_report_keys) and all( + results.get(report_key, {}).get("upload", False) + for report_key in generated_report_keys ) - if all_uploaded: + if all_uploaded and out_dir: try: rmtree(Path(out_dir), ignore_errors=True) logger.info("Cleaned up temporary files at %s", out_dir) @@ -595,6 +1184,7 @@ def generate_compliance_reports_job( generate_ens: bool = True, generate_nis2: bool = True, generate_csa: bool = True, + generate_cis: bool = True, ) -> dict[str, dict[str, bool | str]]: """ Celery task wrapper for generate_compliance_reports. @@ -607,9 +1197,12 @@ def generate_compliance_reports_job( generate_ens: Whether to generate ENS report. generate_nis2: Whether to generate NIS2 report. generate_csa: Whether to generate CSA CCM report. + generate_cis: Whether to generate the CIS Benchmark report for the + latest CIS version available for the provider. Returns: - Dictionary with results for each report type. + Dictionary with results for each report type. Every entry shares the + same flat ``{"upload", "path", "error"?}`` shape. """ return generate_compliance_reports( tenant_id=tenant_id, @@ -619,4 +1212,5 @@ def generate_compliance_reports_job( generate_ens=generate_ens, generate_nis2=generate_nis2, generate_csa=generate_csa, + generate_cis=generate_cis, ) diff --git a/api/src/backend/tasks/jobs/reports/__init__.py b/api/src/backend/tasks/jobs/reports/__init__.py index 1fc475a467..a538416f59 100644 --- a/api/src/backend/tasks/jobs/reports/__init__.py +++ b/api/src/backend/tasks/jobs/reports/__init__.py @@ -17,6 +17,9 @@ from .charts import ( get_chart_color_for_percentage, ) +# Framework-specific generators +from .cis import CISReportGenerator + # Reusable components # Reusable components: Color helpers, Badge components, Risk component, # Table components, Section components @@ -31,10 +34,12 @@ from .components import ( create_section_header, create_status_badge, create_summary_table, + escape_html, get_color_for_compliance, get_color_for_risk_level, get_color_for_weight, get_status_color, + truncate_text, ) # Framework configuration: Main configuration, Color constants, ENS colors, @@ -90,8 +95,6 @@ from .config import ( FrameworkConfig, get_framework_config, ) - -# Framework-specific generators from .csa import CSAReportGenerator from .ens import ENSReportGenerator from .nis2 import NIS2ReportGenerator @@ -109,6 +112,7 @@ __all__ = [ "ENSReportGenerator", "NIS2ReportGenerator", "CSAReportGenerator", + "CISReportGenerator", # Configuration "FrameworkConfig", "FRAMEWORK_REGISTRY", @@ -182,6 +186,9 @@ __all__ = [ # Section components "create_section_header", "create_summary_table", + # Text helpers + "truncate_text", + "escape_html", # Chart functions "get_chart_color_for_percentage", "create_vertical_bar_chart", diff --git a/api/src/backend/tasks/jobs/reports/cis.py b/api/src/backend/tasks/jobs/reports/cis.py new file mode 100644 index 0000000000..0fbb416a17 --- /dev/null +++ b/api/src/backend/tasks/jobs/reports/cis.py @@ -0,0 +1,755 @@ +import os +import re +from collections import defaultdict +from typing import Any + +from reportlab.lib.units import inch +from reportlab.platypus import Image, PageBreak, Paragraph, Spacer, Table, TableStyle + +from api.models import StatusChoices + +from .base import ( + BaseComplianceReportGenerator, + ComplianceData, + RequirementData, + get_requirement_metadata, +) +from .charts import ( + create_horizontal_bar_chart, + create_pie_chart, + create_stacked_bar_chart, + get_chart_color_for_percentage, +) +from .components import ColumnConfig, create_data_table, escape_html, truncate_text +from .config import ( + CHART_COLOR_GREEN_1, + CHART_COLOR_RED, + CHART_COLOR_YELLOW, + COLOR_BG_BLUE, + COLOR_BLUE, + COLOR_BORDER_GRAY, + COLOR_DARK_GRAY, + COLOR_GRAY, + COLOR_GRID_GRAY, + COLOR_HIGH_RISK, + COLOR_LIGHT_BLUE, + COLOR_SAFE, + COLOR_WHITE, +) + +# Ordered buckets used both in the executive summary tables and the charts +# section. Exposed as module constants so the two call sites never drift. +_PROFILE_BUCKET_ORDER: tuple[str, ...] = ("L1", "L2", "Other") +_ASSESSMENT_BUCKET_ORDER: tuple[str, ...] = ("Automated", "Manual") + +# Anchored matchers for profile normalization — substring checks on "L1"/"L2" +# would happily match unrelated tokens like "CL2 Worker" or "HL2" coming from +# future CIS profile enum values. +_LEVEL_2_RE = re.compile(r"(?:\bLevel\s*2\b|\bL2\b|Level_2)") +_LEVEL_1_RE = re.compile(r"(?:\bLevel\s*1\b|\bL1\b|Level_1)") + + +def _normalize_profile(profile: Any) -> str: + """Bucket a CIS Profile enum/string into one of: ``L1``, ``L2``, ``Other``. + + The ``CIS_Requirement_Attribute_Profile`` enum has values like + ``"Level 1"``, ``"Level 2"``, ``"E3 Level 1"``, ``"E5 Level 2"``. We + collapse them into three buckets to keep charts and badges readable + across CIS variants, using anchored regex matches so that future enum + values cannot accidentally promote e.g. ``"CL2 Worker"`` into ``L2``. + + Args: + profile: The profile value (enum member, string, or ``None``). + + Returns: + One of ``"L1"``, ``"L2"``, ``"Other"``. + """ + if profile is None: + return "Other" + value = getattr(profile, "value", None) or str(profile) + if _LEVEL_2_RE.search(value): + return "L2" + if _LEVEL_1_RE.search(value): + return "L1" + return "Other" + + +def _profile_badge_text(bucket: str) -> str: + """Map a normalized profile bucket (L1/L2/Other) to a short badge label.""" + return {"L1": "Level 1", "L2": "Level 2"}.get(bucket, "Other") + + +# ============================================================================= +# CIS Report Generator +# ============================================================================= + + +class CISReportGenerator(BaseComplianceReportGenerator): + """ + PDF report generator for CIS (Center for Internet Security) Benchmarks. + + CIS differs from single-version frameworks (ENS, NIS2, CSA) in that: + - Each provider has multiple CIS versions (e.g. AWS: 1.4, 1.5, ..., 6.0). + - Section names differ across versions and providers and MUST be derived + at runtime from the loaded compliance data. + - Requirements carry Profile (Level 1/Level 2) and AssessmentStatus + (Automated/Manual) attributes that drive the executive summary and + charts. + + This generator produces: + - Cover page with Prowler logo and dynamic CIS version/provider metadata + - Executive summary with overall compliance score, counts, and breakdowns + by Profile and AssessmentStatus + - Charts: overall status pie, pass rate by section (horizontal bar), + Level 1 vs Level 2 pass/fail distribution (stacked bar) + - Requirements index grouped by dynamic section + - Detailed findings for FAIL requirements with CIS-specific audit / + remediation / rationale details + """ + + # Per-run memoization cache for ``_compute_statistics``. ``generate()`` + # is the public entry point and is called once per PDF, so scoping the + # cache to the last seen ComplianceData instance is enough to avoid the + # double computation between executive summary and charts section. + _stats_cache_key: int | None = None + _stats_cache_value: dict | None = None + + # Body section ordering — ensure every top-level section starts on its + # own clean page. The base class only puts a PageBreak AFTER Charts and + # Requirements Index, so Executive Summary and Charts end up sharing a + # page. This override prepends a PageBreak so Compliance Analysis always + # begins on a fresh page. + def _build_body_sections(self, data: ComplianceData) -> list: + return [PageBreak(), *super()._build_body_sections(data)] + + # ------------------------------------------------------------------------- + # Cover page override — shows dynamic CIS version + provider in the title + # ------------------------------------------------------------------------- + + def create_cover_page(self, data: ComplianceData) -> list: + """Create the CIS report cover page with Prowler + CIS logos side by side.""" + elements = [] + + # Create logos side by side (same pattern as NIS2 / ENS) + prowler_logo_path = os.path.join( + os.path.dirname(__file__), "../../assets/img/prowler_logo.png" + ) + cis_logo_path = os.path.join( + os.path.dirname(__file__), "../../assets/img/cis_logo.png" + ) + + if os.path.exists(cis_logo_path): + prowler_logo = Image(prowler_logo_path, width=3.5 * inch, height=0.7 * inch) + cis_logo = Image(cis_logo_path, width=2.3 * inch, height=1.1 * inch) + logos_table = Table( + [[prowler_logo, cis_logo]], colWidths=[4 * inch, 2.5 * inch] + ) + logos_table.setStyle( + TableStyle( + [ + ("ALIGN", (0, 0), (0, 0), "LEFT"), + ("ALIGN", (1, 0), (1, 0), "RIGHT"), + ("VALIGN", (0, 0), (0, 0), "MIDDLE"), + ("VALIGN", (1, 0), (1, 0), "MIDDLE"), + ] + ) + ) + elements.append(logos_table) + elif os.path.exists(prowler_logo_path): + # Fallback: only the Prowler logo if the CIS asset is missing + elements.append(Image(prowler_logo_path, width=5 * inch, height=1 * inch)) + + elements.append(Spacer(1, 0.5 * inch)) + + # Dynamic title: "CIS Benchmark v5.0 — AWS Compliance Report" + provider_label = "" + if data.provider_obj: + provider_label = f" — {data.provider_obj.provider.upper()}" + title_text = ( + f"CIS Benchmark v{data.version}{provider_label}
Compliance Report" + ) + elements.append(Paragraph(title_text, self.styles["title"])) + elements.append(Spacer(1, 0.5 * inch)) + + # Metadata table via base class helper + info_rows = self._build_info_rows(data, language=self.config.language) + metadata_data = [] + for label, value in info_rows: + if label in ("Name:", "Description:") and value: + metadata_data.append( + [label, Paragraph(str(value), self.styles["normal_center"])] + ) + else: + metadata_data.append([label, value]) + + metadata_table = Table(metadata_data, colWidths=[2 * inch, 4 * inch]) + metadata_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (0, -1), COLOR_BLUE), + ("TEXTCOLOR", (0, 0), (0, -1), COLOR_WHITE), + ("FONTNAME", (0, 0), (0, -1), "FiraCode"), + ("BACKGROUND", (1, 0), (1, -1), COLOR_BG_BLUE), + ("TEXTCOLOR", (1, 0), (1, -1), COLOR_GRAY), + ("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"), + ("ALIGN", (0, 0), (-1, -1), "LEFT"), + ("VALIGN", (0, 0), (-1, -1), "TOP"), + ("FONTSIZE", (0, 0), (-1, -1), 11), + ("GRID", (0, 0), (-1, -1), 1, COLOR_BORDER_GRAY), + ("LEFTPADDING", (0, 0), (-1, -1), 10), + ("RIGHTPADDING", (0, 0), (-1, -1), 10), + ("TOPPADDING", (0, 0), (-1, -1), 8), + ("BOTTOMPADDING", (0, 0), (-1, -1), 8), + ] + ) + ) + elements.append(metadata_table) + + return elements + + # ------------------------------------------------------------------------- + # Executive Summary + # ------------------------------------------------------------------------- + + def create_executive_summary(self, data: ComplianceData) -> list: + """Create the CIS executive summary section.""" + elements = [] + + elements.append(Paragraph("Executive Summary", self.styles["h1"])) + elements.append(Spacer(1, 0.1 * inch)) + + stats = self._compute_statistics(data) + + # --- Summary metrics table --- + summary_data = [ + ["Metric", "Value"], + ["Total Requirements", str(stats["total"])], + ["Passed", str(stats["passed"])], + ["Failed", str(stats["failed"])], + ["Manual", str(stats["manual"])], + ["Overall Compliance", f"{stats['overall_compliance']:.1f}%"], + ] + summary_table = Table(summary_data, colWidths=[3 * inch, 2 * inch]) + summary_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (-1, 0), COLOR_BLUE), + ("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE), + ("BACKGROUND", (0, 2), (0, 2), COLOR_SAFE), + ("TEXTCOLOR", (0, 2), (0, 2), COLOR_WHITE), + ("BACKGROUND", (0, 3), (0, 3), COLOR_HIGH_RISK), + ("TEXTCOLOR", (0, 3), (0, 3), COLOR_WHITE), + ("BACKGROUND", (0, 4), (0, 4), COLOR_DARK_GRAY), + ("TEXTCOLOR", (0, 4), (0, 4), COLOR_WHITE), + ("FONTNAME", (0, 0), (-1, 0), "PlusJakartaSans"), + ("FONTSIZE", (0, 0), (-1, 0), 12), + ("FONTSIZE", (0, 1), (-1, -1), 10), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("GRID", (0, 0), (-1, -1), 0.5, COLOR_BORDER_GRAY), + ("BOTTOMPADDING", (0, 0), (-1, 0), 10), + ( + "ROWBACKGROUNDS", + (1, 1), + (1, -1), + [COLOR_WHITE, COLOR_BG_BLUE], + ), + ] + ) + ) + elements.append(summary_table) + elements.append(Spacer(1, 0.25 * inch)) + + # --- Profile breakdown table --- + elements.append(Paragraph("Breakdown by Profile", self.styles["h2"])) + elements.append(Spacer(1, 0.1 * inch)) + profile_counts = stats["profile_counts"] + profile_table_data = [["Profile", "Passed", "Failed", "Manual", "Total"]] + for bucket in _PROFILE_BUCKET_ORDER: + counts = profile_counts.get(bucket, {"passed": 0, "failed": 0, "manual": 0}) + total = counts["passed"] + counts["failed"] + counts["manual"] + if total == 0: + continue + profile_table_data.append( + [ + _profile_badge_text(bucket), + str(counts["passed"]), + str(counts["failed"]), + str(counts["manual"]), + str(total), + ] + ) + profile_table = Table( + profile_table_data, + colWidths=[1.5 * inch, 1 * inch, 1 * inch, 1 * inch, 1 * inch], + ) + profile_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (-1, 0), COLOR_BLUE), + ("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE), + ("FONTNAME", (0, 0), (-1, 0), "FiraCode"), + ("FONTSIZE", (0, 0), (-1, 0), 10), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("FONTSIZE", (0, 1), (-1, -1), 9), + ("GRID", (0, 0), (-1, -1), 0.5, COLOR_GRID_GRAY), + ( + "ROWBACKGROUNDS", + (0, 1), + (-1, -1), + [COLOR_WHITE, COLOR_BG_BLUE], + ), + ] + ) + ) + elements.append(profile_table) + elements.append(Spacer(1, 0.25 * inch)) + + # --- Assessment status breakdown --- + elements.append(Paragraph("Breakdown by Assessment Status", self.styles["h2"])) + elements.append(Spacer(1, 0.1 * inch)) + assessment_counts = stats["assessment_counts"] + assessment_table_data = [["Assessment", "Passed", "Failed", "Manual", "Total"]] + for bucket in _ASSESSMENT_BUCKET_ORDER: + counts = assessment_counts.get( + bucket, {"passed": 0, "failed": 0, "manual": 0} + ) + total = counts["passed"] + counts["failed"] + counts["manual"] + if total == 0: + continue + assessment_table_data.append( + [ + bucket, + str(counts["passed"]), + str(counts["failed"]), + str(counts["manual"]), + str(total), + ] + ) + assessment_table = Table( + assessment_table_data, + colWidths=[1.5 * inch, 1 * inch, 1 * inch, 1 * inch, 1 * inch], + ) + assessment_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (-1, 0), COLOR_LIGHT_BLUE), + ("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE), + ("FONTNAME", (0, 0), (-1, 0), "FiraCode"), + ("FONTSIZE", (0, 0), (-1, 0), 10), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("FONTSIZE", (0, 1), (-1, -1), 9), + ("GRID", (0, 0), (-1, -1), 0.5, COLOR_GRID_GRAY), + ( + "ROWBACKGROUNDS", + (0, 1), + (-1, -1), + [COLOR_WHITE, COLOR_BG_BLUE], + ), + ] + ) + ) + elements.append(assessment_table) + elements.append(Spacer(1, 0.25 * inch)) + + # --- Top 5 failing sections --- + top_failing = stats["top_failing_sections"] + if top_failing: + elements.append( + Paragraph("Top Sections with Lowest Compliance", self.styles["h2"]) + ) + elements.append(Spacer(1, 0.1 * inch)) + top_table_data = [["Section", "Passed", "Failed", "Compliance"]] + for section_label, section_stats in top_failing: + passed = section_stats["passed"] + failed = section_stats["failed"] + total = passed + failed + pct = (passed / total * 100) if total > 0 else 100 + top_table_data.append( + [ + truncate_text(section_label, 55), + str(passed), + str(failed), + f"{pct:.1f}%", + ] + ) + top_table = Table( + top_table_data, + colWidths=[3.5 * inch, 0.9 * inch, 0.9 * inch, 1.2 * inch], + ) + top_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (-1, 0), COLOR_HIGH_RISK), + ("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE), + ("FONTNAME", (0, 0), (-1, 0), "FiraCode"), + ("FONTSIZE", (0, 0), (-1, 0), 10), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("FONTSIZE", (0, 1), (-1, -1), 9), + ("GRID", (0, 0), (-1, -1), 0.5, COLOR_GRID_GRAY), + ( + "ROWBACKGROUNDS", + (0, 1), + (-1, -1), + [COLOR_WHITE, COLOR_BG_BLUE], + ), + ] + ) + ) + elements.append(top_table) + + return elements + + # ------------------------------------------------------------------------- + # Charts section + # ------------------------------------------------------------------------- + + def create_charts_section(self, data: ComplianceData) -> list: + """Create the CIS charts section.""" + elements = [] + + elements.append(Paragraph("Compliance Analysis", self.styles["h1"])) + elements.append(Spacer(1, 0.1 * inch)) + + # --- Pie chart: overall Pass / Fail / Manual --- + stats = self._compute_statistics(data) + pie_labels = [] + pie_values = [] + pie_colors = [] + if stats["passed"] > 0: + pie_labels.append(f"Pass ({stats['passed']})") + pie_values.append(stats["passed"]) + pie_colors.append(CHART_COLOR_GREEN_1) + if stats["failed"] > 0: + pie_labels.append(f"Fail ({stats['failed']})") + pie_values.append(stats["failed"]) + pie_colors.append(CHART_COLOR_RED) + if stats["manual"] > 0: + pie_labels.append(f"Manual ({stats['manual']})") + pie_values.append(stats["manual"]) + pie_colors.append(CHART_COLOR_YELLOW) + + if pie_values: + elements.append(Paragraph("Overall Status Distribution", self.styles["h2"])) + elements.append(Spacer(1, 0.1 * inch)) + pie_buffer = create_pie_chart( + labels=pie_labels, + values=pie_values, + colors=pie_colors, + ) + pie_buffer.seek(0) + elements.append(Image(pie_buffer, width=4.5 * inch, height=4.5 * inch)) + elements.append(Spacer(1, 0.2 * inch)) + + # --- Horizontal bar: pass rate by section --- + section_stats = stats["section_stats"] + if section_stats: + elements.append(PageBreak()) + elements.append(Paragraph("Compliance by Section", self.styles["h1"])) + elements.append(Spacer(1, 0.1 * inch)) + elements.append( + Paragraph( + "The following chart shows compliance percentage for each CIS " + "section based on automated checks:", + self.styles["normal_center"], + ) + ) + elements.append(Spacer(1, 0.1 * inch)) + + # Sort sections by pass rate descending for readability + sorted_sections = sorted( + section_stats.items(), + key=lambda item: ( + (item[1]["passed"] / (item[1]["passed"] + item[1]["failed"]) * 100) + if (item[1]["passed"] + item[1]["failed"]) > 0 + else 100 + ), + reverse=True, + ) + bar_labels = [] + bar_values = [] + for section_label, section_data in sorted_sections: + total = section_data["passed"] + section_data["failed"] + if total == 0: + continue + pct = (section_data["passed"] / total) * 100 + bar_labels.append(truncate_text(section_label, 60)) + bar_values.append(pct) + + if bar_values: + bar_buffer = create_horizontal_bar_chart( + labels=bar_labels, + values=bar_values, + xlabel="Compliance (%)", + color_func=get_chart_color_for_percentage, + label_fontsize=9, + ) + bar_buffer.seek(0) + elements.append(Image(bar_buffer, width=6.5 * inch, height=5 * inch)) + + # --- Stacked bar: Level 1 vs Level 2 pass/fail --- + profile_counts = stats["profile_counts"] + has_profile_data = any( + (counts["passed"] + counts["failed"]) > 0 + for counts in profile_counts.values() + ) + if has_profile_data: + elements.append(PageBreak()) + elements.append(Paragraph("Profile Breakdown", self.styles["h1"])) + elements.append(Spacer(1, 0.1 * inch)) + elements.append( + Paragraph( + "Distribution of Pass / Fail / Manual across CIS profile levels.", + self.styles["normal_center"], + ) + ) + elements.append(Spacer(1, 0.1 * inch)) + + profile_labels = [] + pass_series = [] + fail_series = [] + manual_series = [] + for bucket in _PROFILE_BUCKET_ORDER: + counts = profile_counts.get(bucket) + if not counts: + continue + total = counts["passed"] + counts["failed"] + counts["manual"] + if total == 0: + continue + profile_labels.append(_profile_badge_text(bucket)) + pass_series.append(counts["passed"]) + fail_series.append(counts["failed"]) + manual_series.append(counts["manual"]) + + if profile_labels: + stacked_buffer = create_stacked_bar_chart( + labels=profile_labels, + data_series={ + "Pass": pass_series, + "Fail": fail_series, + "Manual": manual_series, + }, + xlabel="Profile", + ylabel="Requirements", + ) + stacked_buffer.seek(0) + elements.append(Image(stacked_buffer, width=6 * inch, height=4 * inch)) + + return elements + + # ------------------------------------------------------------------------- + # Requirements Index + # ------------------------------------------------------------------------- + + def create_requirements_index(self, data: ComplianceData) -> list: + """Create the CIS requirements index grouped by dynamic section.""" + elements = [] + + elements.append(Paragraph("Requirements Index", self.styles["h1"])) + elements.append(Spacer(1, 0.1 * inch)) + + sections = self._derive_sections(data) + by_section: dict[str, list[dict]] = defaultdict(list) + for req in data.requirements: + meta = get_requirement_metadata(req.id, data.attributes_by_requirement_id) + section = "Other" + profile_bucket = "Other" + assessment = "" + if meta: + section = getattr(meta, "Section", "Other") or "Other" + profile_bucket = _normalize_profile(getattr(meta, "Profile", None)) + assessment_enum = getattr(meta, "AssessmentStatus", None) + assessment = getattr(assessment_enum, "value", None) or str( + assessment_enum or "" + ) + by_section[section].append( + { + "id": req.id, + "description": truncate_text(req.description, 80), + "profile": _profile_badge_text(profile_bucket), + "assessment": assessment or "-", + "status": (req.status or "").upper(), + } + ) + + columns = [ + ColumnConfig("ID", 0.9 * inch, "id", align="LEFT"), + ColumnConfig("Description", 3.0 * inch, "description", align="LEFT"), + ColumnConfig("Profile", 0.9 * inch, "profile"), + ColumnConfig("Assessment", 1 * inch, "assessment"), + ColumnConfig("Status", 0.9 * inch, "status"), + ] + + for section in sections: + rows = by_section.get(section, []) + if not rows: + continue + elements.append(Paragraph(truncate_text(section, 90), self.styles["h2"])) + elements.append(Spacer(1, 0.05 * inch)) + table = create_data_table( + data=rows, + columns=columns, + header_color=self.config.primary_color, + normal_style=self.styles["normal_center"], + ) + elements.append(table) + elements.append(Spacer(1, 0.15 * inch)) + + return elements + + # ------------------------------------------------------------------------- + # Detailed findings hook — inject CIS-specific rationale / audit content + # ------------------------------------------------------------------------- + + def _render_requirement_detail_extras( + self, req: RequirementData, data: ComplianceData + ) -> list: + """Render CIS rationale, impact, audit, remediation and references.""" + extras = [] + meta = get_requirement_metadata(req.id, data.attributes_by_requirement_id) + if meta is None: + return extras + + field_map = [ + ("Rationale", "RationaleStatement"), + ("Impact", "ImpactStatement"), + ("Audit Procedure", "AuditProcedure"), + ("Remediation", "RemediationProcedure"), + ("References", "References"), + ] + + for label, attr_name in field_map: + value = getattr(meta, attr_name, None) + if not value: + continue + text = str(value).strip() + if not text: + continue + extras.append(Paragraph(f"{label}:", self.styles["h3"])) + extras.append(Paragraph(escape_html(text), self.styles["normal"])) + extras.append(Spacer(1, 0.08 * inch)) + + return extras + + # ------------------------------------------------------------------------- + # Private helpers + # ------------------------------------------------------------------------- + + def _derive_sections(self, data: ComplianceData) -> list[str]: + """Extract ordered unique Section names from loaded compliance data.""" + seen: dict[str, bool] = {} + for req in data.requirements: + meta = get_requirement_metadata(req.id, data.attributes_by_requirement_id) + if meta is None: + continue + section = getattr(meta, "Section", None) or "Other" + if section not in seen: + seen[section] = True + return list(seen.keys()) + + def _compute_statistics(self, data: ComplianceData) -> dict: + """Aggregate all statistics needed for summary and charts. + + Memoized per-``ComplianceData`` instance via ``_stats_cache_*``: the + executive summary and the charts section both need the same numbers, + so they would otherwise re-iterate the requirements twice. We key on + ``id(data)`` because ``ComplianceData`` is a dataclass and its + instances are not hashable. + + Returns a dict with: + - total, passed, failed, manual: int + - overall_compliance: float (percentage) + - profile_counts: {"L1": {"passed", "failed", "manual"}, ...} + - assessment_counts: {"Automated": {...}, "Manual": {...}} + - section_stats: {section_name: {"passed", "failed", "manual"}, ...} + - top_failing_sections: list[(section_name, stats)] (up to 5) + """ + cache_key = id(data) + if self._stats_cache_key == cache_key and self._stats_cache_value is not None: + return self._stats_cache_value + stats = self._compute_statistics_uncached(data) + self._stats_cache_key = cache_key + self._stats_cache_value = stats + return stats + + def _compute_statistics_uncached(self, data: ComplianceData) -> dict: + """Actual aggregation kernel; call ``_compute_statistics`` instead.""" + total = len(data.requirements) + passed = sum(1 for r in data.requirements if r.status == StatusChoices.PASS) + failed = sum(1 for r in data.requirements if r.status == StatusChoices.FAIL) + manual = sum(1 for r in data.requirements if r.status == StatusChoices.MANUAL) + + evaluated = passed + failed + overall_compliance = (passed / evaluated * 100) if evaluated > 0 else 100.0 + + profile_counts: dict[str, dict[str, int]] = { + "L1": {"passed": 0, "failed": 0, "manual": 0}, + "L2": {"passed": 0, "failed": 0, "manual": 0}, + "Other": {"passed": 0, "failed": 0, "manual": 0}, + } + assessment_counts: dict[str, dict[str, int]] = { + "Automated": {"passed": 0, "failed": 0, "manual": 0}, + "Manual": {"passed": 0, "failed": 0, "manual": 0}, + } + section_stats: dict[str, dict[str, int]] = defaultdict( + lambda: {"passed": 0, "failed": 0, "manual": 0} + ) + + for req in data.requirements: + meta = get_requirement_metadata(req.id, data.attributes_by_requirement_id) + if meta is None: + continue + + profile_bucket = _normalize_profile(getattr(meta, "Profile", None)) + assessment_enum = getattr(meta, "AssessmentStatus", None) + assessment_value = getattr(assessment_enum, "value", None) or str( + assessment_enum or "" + ) + assessment_bucket = ( + "Automated" if assessment_value == "Automated" else "Manual" + ) + section = getattr(meta, "Section", None) or "Other" + + status_key = { + StatusChoices.PASS: "passed", + StatusChoices.FAIL: "failed", + StatusChoices.MANUAL: "manual", + }.get(req.status) + if status_key is None: + continue + + profile_counts[profile_bucket][status_key] += 1 + assessment_counts[assessment_bucket][status_key] += 1 + section_stats[section][status_key] += 1 + + # Top 5 sections with lowest pass rate (only sections with evaluated reqs) + def _section_rate(item): + _, stats_ = item + evaluated_ = stats_["passed"] + stats_["failed"] + if evaluated_ == 0: + return 101 # sort evaluated=0 to the bottom + return stats_["passed"] / evaluated_ * 100 + + top_failing_sections = sorted( + ( + item + for item in section_stats.items() + if (item[1]["passed"] + item[1]["failed"]) > 0 + ), + key=_section_rate, + )[:5] + + return { + "total": total, + "passed": passed, + "failed": failed, + "manual": manual, + "overall_compliance": overall_compliance, + "profile_counts": profile_counts, + "assessment_counts": assessment_counts, + "section_stats": dict(section_stats), + "top_failing_sections": top_failing_sections, + } diff --git a/api/src/backend/tasks/jobs/reports/components.py b/api/src/backend/tasks/jobs/reports/components.py index 323c4547e6..049cc043d3 100644 --- a/api/src/backend/tasks/jobs/reports/components.py +++ b/api/src/backend/tasks/jobs/reports/components.py @@ -26,6 +26,52 @@ from .config import ( ) +def truncate_text(text: str, max_len: int) -> str: + """Truncate ``text`` to ``max_len`` characters, appending an ellipsis if cut. + + Used by report generators that need to squeeze long descriptions, section + titles or finding titles into a fixed-width table cell. + + Args: + text: Source string. ``None`` and non-string values are treated as empty. + max_len: Maximum output length including the ellipsis. Values < 4 are + clamped so the result never grows beyond ``max_len``. + + Returns: + The original string if short enough, otherwise ``text[: max_len - 3] + "..."``. + When ``max_len < 4`` a plain substring of length ``max_len`` is returned + so callers never get a string longer than they asked for. + """ + if not text: + return "" + text = str(text) + if len(text) <= max_len: + return text + if max_len < 4: + return text[:max_len] + return text[: max_len - 3] + "..." + + +def escape_html(text: str) -> str: + """Escape the minimal HTML entities required for safe ReportLab Paragraph rendering. + + ReportLab's ``Paragraph`` parses a small HTML subset, so raw ``<``, ``>`` + and ``&`` in user-provided content (rationale, remediation, etc.) would + break layout or be interpreted as tags. This helper mirrors + ``html.escape`` but avoids pulling in the stdlib dependency and keeps the + output deterministic. + + Args: + text: Untrusted source string. + + Returns: + A string safe to embed inside a ReportLab Paragraph. + """ + return ( + str(text or "").replace("&", "&").replace("<", "<").replace(">", ">") + ) + + def get_color_for_risk_level(risk_level: int) -> colors.Color: """ Get color based on risk level. diff --git a/api/src/backend/tasks/jobs/reports/config.py b/api/src/backend/tasks/jobs/reports/config.py index fe0326980d..669f31c287 100644 --- a/api/src/backend/tasks/jobs/reports/config.py +++ b/api/src/backend/tasks/jobs/reports/config.py @@ -313,6 +313,32 @@ FRAMEWORK_REGISTRY: dict[str, FrameworkConfig] = { has_niveles=False, has_weight=False, ), + "cis": FrameworkConfig( + name="cis", + display_name="CIS Benchmark", + logo_filename=None, + primary_color=COLOR_BLUE, + secondary_color=COLOR_LIGHT_BLUE, + bg_color=COLOR_BG_BLUE, + attribute_fields=[ + "Section", + "SubSection", + "Profile", + "AssessmentStatus", + "Description", + "RationaleStatement", + "ImpactStatement", + "RemediationProcedure", + "AuditProcedure", + "References", + ], + sections=None, # Derived dynamically per CIS variant (section names differ across versions/providers) + language="en", + has_risk_levels=False, + has_dimensions=False, + has_niveles=False, + has_weight=False, + ), } @@ -336,5 +362,7 @@ def get_framework_config(compliance_id: str) -> FrameworkConfig | None: return FRAMEWORK_REGISTRY["nis2"] if "csa" in compliance_lower or "ccm" in compliance_lower: return FRAMEWORK_REGISTRY["csa_ccm"] + if compliance_lower.startswith("cis_") or "cis" in compliance_lower: + return FRAMEWORK_REGISTRY["cis"] return None diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index b70ce36a7f..515aea8620 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -10,16 +10,29 @@ from typing import Any import sentry_sdk from celery.utils.log import get_task_logger +from config.django.base import DJANGO_FINDINGS_BATCH_SIZE from config.env import env from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS from django.db import IntegrityError, OperationalError -from django.db.models import Case, Count, IntegerField, Max, Min, Prefetch, Q, Sum, When +from django.db.models import ( + Case, + Count, + Exists, + IntegerField, + Max, + Min, + OuterRef, + Prefetch, + Q, + Sum, + When, +) from django.utils import timezone as django_timezone from tasks.jobs.queries import ( COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL, COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL, ) -from tasks.utils import CustomEncoder +from tasks.utils import CustomEncoder, batched from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE from api.constants import SEVERITY_ORDER @@ -677,6 +690,7 @@ def _process_finding_micro_batch( # Create finding object (don't save yet) check_metadata = finding.get_metadata() + check_metadata["compliance"] = finding.compliance finding_instance = Finding( tenant_id=tenant_id, uid=finding_uid, @@ -751,11 +765,19 @@ def _process_finding_micro_batch( ) if mappings_to_create: - ResourceFindingMapping.objects.bulk_create( + created_mappings = ResourceFindingMapping.objects.bulk_create( mappings_to_create, batch_size=SCAN_DB_BATCH_SIZE, ignore_conflicts=True, + unique_fields=["tenant_id", "resource_id", "finding_id"], ) + inserted = sum(1 for m in created_mappings if m.pk) + if inserted != len(mappings_to_create): + logger.error( + f"scan {scan_instance.id}: expected " + f"{len(mappings_to_create)} ResourceFindingMapping rows, " + f"inserted {inserted}. Rolling back micro-batch." + ) # Update finding denormalized arrays findings_to_update = [] @@ -1188,8 +1210,39 @@ def aggregate_findings(tenant_id: str, scan_id: str): muted_changed=agg["muted_changed"], ) for agg in aggregation + if agg["resources__service"] is not None + and agg["resources__region"] is not None } - ScanSummary.objects.bulk_create(scan_aggregations, batch_size=3000) + # Upsert so re-runs (post-mute reaggregation) don't trip + # `unique_scan_summary`; race-safe under concurrent writers. + ScanSummary.objects.bulk_create( + scan_aggregations, + batch_size=3000, + update_conflicts=True, + unique_fields=[ + "tenant", + "scan", + "check_id", + "service", + "severity", + "region", + ], + update_fields=[ + "_pass", + "fail", + "muted", + "total", + "new", + "changed", + "unchanged", + "fail_new", + "fail_changed", + "pass_new", + "pass_changed", + "muted_new", + "muted_changed", + ], + ) def _aggregate_findings_by_region( @@ -1534,13 +1587,24 @@ def aggregate_attack_surface(tenant_id: str, scan_id: str): ) ) - # Bulk create overview records if overview_objects: with rls_transaction(tenant_id): - AttackSurfaceOverview.objects.bulk_create(overview_objects, batch_size=500) - logger.info( - f"Created {len(overview_objects)} attack surface overview records for scan {scan_id}" + # Upsert so re-runs (post-mute reaggregation) don't trip + # `unique_attack_surface_per_scan`; race-safe under concurrent writers. + AttackSurfaceOverview.objects.bulk_create( + overview_objects, + batch_size=500, + update_conflicts=True, + unique_fields=["tenant_id", "scan_id", "attack_surface_type"], + update_fields=[ + "total_findings", + "failed_findings", + "muted_failed_findings", + ], ) + logger.info( + f"Upserted {len(overview_objects)} attack surface overview records for scan {scan_id}" + ) else: logger.info(f"No attack surface overview records created for scan {scan_id}") @@ -1802,7 +1866,10 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str): output_field=IntegerField(), ) - # Aggregate findings by check_id for this scan + # Aggregate findings by check_id for this scan. + # `pass_count`, `fail_count` and `manual_count` only count non-muted + # findings. Muted findings are tracked separately via the + # `*_muted_count` fields. aggregated = ( Finding.objects.filter( tenant_id=tenant_id, @@ -1813,9 +1880,50 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str): severity_order=Max(severity_case), pass_count=Count("id", filter=Q(status="PASS", muted=False)), fail_count=Count("id", filter=Q(status="FAIL", muted=False)), + manual_count=Count("id", filter=Q(status="MANUAL", muted=False)), + pass_muted_count=Count("id", filter=Q(status="PASS", muted=True)), + fail_muted_count=Count("id", filter=Q(status="FAIL", muted=True)), + manual_muted_count=Count("id", filter=Q(status="MANUAL", muted=True)), muted_count=Count("id", filter=Q(muted=True)), + nonmuted_count=Count("id", filter=Q(muted=False)), new_count=Count("id", filter=Q(delta="new", muted=False)), changed_count=Count("id", filter=Q(delta="changed", muted=False)), + new_fail_count=Count( + "id", filter=Q(delta="new", status="FAIL", muted=False) + ), + new_fail_muted_count=Count( + "id", filter=Q(delta="new", status="FAIL", muted=True) + ), + new_pass_count=Count( + "id", filter=Q(delta="new", status="PASS", muted=False) + ), + new_pass_muted_count=Count( + "id", filter=Q(delta="new", status="PASS", muted=True) + ), + new_manual_count=Count( + "id", filter=Q(delta="new", status="MANUAL", muted=False) + ), + new_manual_muted_count=Count( + "id", filter=Q(delta="new", status="MANUAL", muted=True) + ), + changed_fail_count=Count( + "id", filter=Q(delta="changed", status="FAIL", muted=False) + ), + changed_fail_muted_count=Count( + "id", filter=Q(delta="changed", status="FAIL", muted=True) + ), + changed_pass_count=Count( + "id", filter=Q(delta="changed", status="PASS", muted=False) + ), + changed_pass_muted_count=Count( + "id", filter=Q(delta="changed", status="PASS", muted=True) + ), + changed_manual_count=Count( + "id", filter=Q(delta="changed", status="MANUAL", muted=False) + ), + changed_manual_muted_count=Count( + "id", filter=Q(delta="changed", status="MANUAL", muted=True) + ), resources_total=Count("resources__id", distinct=True), resources_fail=Count( "resources__id", @@ -1823,7 +1931,9 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str): filter=Q(status="FAIL", muted=False), ), # Use prefixed names to avoid conflict with model field names - agg_first_seen_at=Min("first_seen_at"), + agg_first_seen_at=Min( + "first_seen_at", filter=Q(delta="new", muted=False) + ), agg_last_seen_at=Max("inserted_at"), agg_failing_since=Min( "first_seen_at", filter=Q(status="FAIL", muted=False) @@ -1887,13 +1997,31 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str): inserted_at=summary_timestamp, updated_at=updated_at, check_title=metadata.get("checktitle", ""), - check_description=metadata.get("Description", ""), + check_description=metadata.get("description", "") + or metadata.get("Description", ""), severity_order=row["severity_order"] or 1, pass_count=row["pass_count"], fail_count=row["fail_count"], + manual_count=row["manual_count"], + pass_muted_count=row["pass_muted_count"], + fail_muted_count=row["fail_muted_count"], + manual_muted_count=row["manual_muted_count"], muted_count=row["muted_count"], + muted=row["nonmuted_count"] == 0, new_count=row["new_count"], changed_count=row["changed_count"], + new_fail_count=row["new_fail_count"], + new_fail_muted_count=row["new_fail_muted_count"], + new_pass_count=row["new_pass_count"], + new_pass_muted_count=row["new_pass_muted_count"], + new_manual_count=row["new_manual_count"], + new_manual_muted_count=row["new_manual_muted_count"], + changed_fail_count=row["changed_fail_count"], + changed_fail_muted_count=row["changed_fail_muted_count"], + changed_pass_count=row["changed_pass_count"], + changed_pass_muted_count=row["changed_pass_muted_count"], + changed_manual_count=row["changed_manual_count"], + changed_manual_muted_count=row["changed_manual_muted_count"], resources_total=row["resources_total"], resources_fail=row["resources_fail"], first_seen_at=row["agg_first_seen_at"], @@ -1913,9 +2041,26 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str): "severity_order", "pass_count", "fail_count", + "manual_count", + "pass_muted_count", + "fail_muted_count", + "manual_muted_count", "muted_count", + "muted", "new_count", "changed_count", + "new_fail_count", + "new_fail_muted_count", + "new_pass_count", + "new_pass_muted_count", + "new_manual_count", + "new_manual_muted_count", + "changed_fail_count", + "changed_fail_muted_count", + "changed_pass_count", + "changed_pass_muted_count", + "changed_manual_count", + "changed_manual_muted_count", "resources_total", "resources_fail", "first_seen_at", @@ -1937,3 +2082,169 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str): "created": created_count, "updated": updated_count, } + + +def reset_ephemeral_resource_findings_count(tenant_id: str, scan_id: str) -> dict: + """Zero failed_findings_count for resources missing from a completed full-scope scan. + + Resources that exist in the database for the scan's provider but were not + touched by this scan are treated as ephemeral. We keep their historical + findings, but reset the denormalized counter that drives the Resources page + sort so they stop ranking at the top. + + Skipped (no-op) when: + - The scan is not in COMPLETED state. + - The scan ran with any scoping filter in scanner_args (partial scope). + + Query design (must scale to 500k+ resources per provider): + Phase 1 — collect ephemeral IDs with one anti-join read. + Outer filter ``(tenant_id, provider_id, failed_findings_count > 0)`` + uses ``resources_tenant_provider_idx``. The correlated + ``NOT EXISTS`` subquery hits the implicit unique index + ``(tenant_id, scan_id, resource_id)`` on ``ResourceScanSummary``. + ``NOT EXISTS`` (vs ``NOT IN``) is null-safe and lets the planner + choose between hash anti-join and indexed nested-loop anti-join. + ``.iterator(chunk_size=...)`` skips the queryset cache so memory + stays bounded while streaming UUIDs. + Phase 2 — UPDATE in fixed-size batches. + One large UPDATE would hold row-exclusive locks for seconds and + create a WAL spike. Batched UPDATEs by ``id__in`` (~1k rows each) + hit the primary key, keep each lock window ~50ms, bound WAL chunks, + and let other writers proceed between batches. + ``failed_findings_count__gt=0`` in the UPDATE is idempotent under + concurrent scans and skips no-op rewrites. + Reads use the primary DB, not the replica: ``ResourceScanSummary`` rows + were written by the same scan task that triggered this one, so replica + lag could falsely classify scanned resources as ephemeral. + + Scope detection (``Scan.is_full_scope()``) derives the set of scoping + scanner_args from ``prowler.lib.scan.scan.Scan.__init__`` via + introspection, so the API can never drift from the SDK's filter + contract. Imported scans are also rejected by trigger — they may only + cover a partial slice of resources. + """ + with rls_transaction(tenant_id): + scan = Scan.objects.filter(tenant_id=tenant_id, id=scan_id).first() + + if scan is None: + logger.warning(f"Scan {scan_id} not found") + return {"status": "skipped", "reason": "scan not found"} + + if scan.state != StateChoices.COMPLETED: + logger.info(f"Scan {scan_id} not completed; skipping ephemeral reset") + return {"status": "skipped", "reason": "scan not completed"} + + if not scan.is_full_scope(): + logger.info( + f"Scan {scan_id} ran with scoping filters; skipping ephemeral reset" + ) + return {"status": "skipped", "reason": "partial scan scope"} + + # Race protection: if a newer completed full-scope scan exists for this + # provider, our ResourceScanSummary set is stale relative to the resources' + # current failed_findings_count values (which the newer scan already + # refreshed). Wiping based on the older scan would zero counts the newer + # scan just set. Skip and let the newer scan's reset task do the work; if + # this task was delayed in the queue, that's the correct outcome. + # `completed_at__isnull=False` is required: Postgres orders NULL first in + # DESC, so a sibling COMPLETED scan with a missing completed_at would sort + # as "newest" and incorrectly cause us to skip. + with rls_transaction(tenant_id): + latest_full_scope_scan_id = ( + Scan.objects.filter( + tenant_id=tenant_id, + provider_id=scan.provider_id, + state=StateChoices.COMPLETED, + completed_at__isnull=False, + ) + .order_by("-completed_at", "-inserted_at") + .values_list("id", flat=True) + .first() + ) + if latest_full_scope_scan_id != scan.id: + logger.info( + f"Scan {scan_id} is not the latest completed scan for provider " + f"{scan.provider_id}; skipping ephemeral reset" + ) + return {"status": "skipped", "reason": "newer scan exists"} + + # Defensive gate: ResourceScanSummary rows are written by perform_prowler_scan + # via best-effort bulk_create. If those writes failed silently (or the scan + # genuinely produced resources but no summaries were persisted), the + # ~Exists(in_scan) anti-join below would classify EVERY resource for this + # provider as ephemeral and zero their counts. Bail loudly instead. + with rls_transaction(tenant_id): + summaries_present = ResourceScanSummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).exists() + if scan.unique_resource_count > 0 and not summaries_present: + logger.error( + f"Scan {scan_id} reports {scan.unique_resource_count} unique " + f"resources but no ResourceScanSummary rows are persisted; " + f"skipping ephemeral reset to avoid wiping valid counts" + ) + return {"status": "skipped", "reason": "summaries missing"} + + # Stays on the primary DB intentionally. ResourceScanSummary rows are + # written by perform_prowler_scan in the same chain that triggered this + # task, so replica lag could return an empty/partial summary set; a stale + # read here would classify every Resource as ephemeral and wipe valid + # failed_findings_count values on the primary. Same rationale as + # update_provider_compliance_scores below in this module. + # Materializing the ID list (rather than streaming the iterator into + # batched UPDATEs) is intentional: it lets the UPDATEs run in their own + # short rls_transactions instead of one long transaction holding row locks + # on every batch. At 500k UUIDs the peak memory is ~40 MB — acceptable for + # a Celery worker — and is the better trade-off versus a multi-second + # write-lock window blocking concurrent scans. + with rls_transaction(tenant_id): + in_scan = ResourceScanSummary.objects.filter( + tenant_id=tenant_id, + scan_id=scan_id, + resource_id=OuterRef("pk"), + ) + ephemeral_ids = list( + Resource.objects.filter( + tenant_id=tenant_id, + provider_id=scan.provider_id, + failed_findings_count__gt=0, + ) + .filter(~Exists(in_scan)) + .values_list("id", flat=True) + .iterator(chunk_size=DJANGO_FINDINGS_BATCH_SIZE) + ) + + if not ephemeral_ids: + logger.info(f"No ephemeral resources for scan {scan_id}") + return { + "status": "completed", + "scan_id": str(scan_id), + "provider_id": str(scan.provider_id), + "reset": 0, + } + + total_updated = 0 + for batch, _ in batched(ephemeral_ids, DJANGO_FINDINGS_BATCH_SIZE): + # batched() always yields a final tuple, which is empty when the input + # length is an exact multiple of the batch size. Skip it so we don't + # issue a no-op UPDATE ... WHERE id IN (). + if not batch: + continue + with rls_transaction(tenant_id): + total_updated += Resource.objects.filter( + tenant_id=tenant_id, + id__in=batch, + failed_findings_count__gt=0, + ).update(failed_findings_count=0) + + logger.info( + f"Ephemeral resource reset for scan {scan_id}: " + f"{total_updated} resources zeroed for provider {scan.provider_id}" + ) + + return { + "status": "completed", + "scan_id": str(scan_id), + "provider_id": str(scan.provider_id), + "reset": total_updated, + } diff --git a/api/src/backend/tasks/jobs/threatscore_utils.py b/api/src/backend/tasks/jobs/threatscore_utils.py index a17125af46..2ef29484ee 100644 --- a/api/src/backend/tasks/jobs/threatscore_utils.py +++ b/api/src/backend/tasks/jobs/threatscore_utils.py @@ -4,7 +4,7 @@ from django.db.models import Count, Q from api.db_router import READ_REPLICA_ALIAS from api.db_utils import rls_transaction -from api.models import Finding, StatusChoices +from api.models import Finding, Scan, StatusChoices from prowler.lib.outputs.finding import Finding as FindingOutput logger = get_task_logger(__name__) @@ -35,25 +35,26 @@ def _aggregate_requirement_statistics_from_database( } """ requirement_statistics_by_check_id = {} - # TODO: take into account that now the relation is 1 finding == 1 resource, review this when the logic changes + # TODO: review when finding-resource relation changes from 1:1 with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + # Pre-check: skip if the scan's provider is deleted (avoids JOINs in the main query) + if Scan.all_objects.filter(id=scan_id, provider__is_deleted=True).exists(): + return requirement_statistics_by_check_id + aggregated_statistics_queryset = ( Finding.all_objects.filter( tenant_id=tenant_id, scan_id=scan_id, muted=False, - resources__provider__is_deleted=False, ) .values("check_id") .annotate( total_findings=Count( "id", - distinct=True, filter=Q(status__in=[StatusChoices.PASS, StatusChoices.FAIL]), ), passed_findings=Count( "id", - distinct=True, filter=Q(status=StatusChoices.PASS), ), ) diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index 2e31ebc0f0..3d36d711d5 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -11,16 +11,17 @@ from django_celery_beat.models import PeriodicTask from tasks.jobs.attack_paths import ( attack_paths_scan, can_provider_run_attack_paths_scan, - db_utils as attack_paths_db_utils, ) +from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils +from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans from tasks.jobs.backfill import ( backfill_compliance_summaries, backfill_daily_severity_summaries, backfill_finding_group_summaries, backfill_provider_compliance_scores, backfill_resource_scan_summaries, - backfill_scan_category_summaries, - backfill_scan_resource_group_summaries, + aggregate_scan_category_summaries, + aggregate_scan_resource_group_summaries, ) from tasks.jobs.connection import ( check_integration_connection, @@ -45,7 +46,11 @@ from tasks.jobs.lighthouse_providers import ( refresh_lighthouse_provider_models, ) from tasks.jobs.muting import mute_historical_findings -from tasks.jobs.report import generate_compliance_reports_job +from tasks.jobs.report import ( + STALE_TMP_OUTPUT_MAX_AGE_HOURS, + _cleanup_stale_tmp_output_directories, + generate_compliance_reports_job, +) from tasks.jobs.scan import ( aggregate_attack_surface, aggregate_daily_severity, @@ -53,6 +58,7 @@ from tasks.jobs.scan import ( aggregate_findings, create_compliance_requirements, perform_prowler_scan, + reset_ephemeral_resource_findings_count, update_provider_compliance_scores, ) from tasks.utils import ( @@ -72,6 +78,7 @@ from prowler.lib.check.compliance_models import Compliance from prowler.lib.outputs.compliance.generic.generic import GenericCompliance from prowler.lib.outputs.finding import Finding as FindingOutput + logger = get_task_logger(__name__) @@ -153,6 +160,13 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str) generate_outputs_task.si( scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id ), + # post-scan task — runs in the parallel group so a + # failure cannot cascade into reports or integrations. Its only + # prerequisite is that perform_prowler_scan has committed + # ResourceScanSummary, which is true by the time this chain fires. + reset_ephemeral_resource_findings_count_task.si( + tenant_id=tenant_id, scan_id=scan_id + ), ), group( # Use optimized task that generates both reports with shared queries @@ -168,10 +182,25 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str) ).apply_async() if can_provider_run_attack_paths_scan(tenant_id, provider_id): - perform_attack_paths_scan_task.apply_async( + # Row is normally created upstream, so this is a safeguard so we can attach the task id below + attack_paths_scan = attack_paths_db_utils.retrieve_attack_paths_scan( + tenant_id, scan_id + ) + if attack_paths_scan is None: + attack_paths_scan = attack_paths_db_utils.create_attack_paths_scan( + tenant_id, scan_id, provider_id + ) + + # Persist the Celery task id so the periodic cleanup can revoke scans stuck in SCHEDULED + result = perform_attack_paths_scan_task.apply_async( kwargs={"tenant_id": tenant_id, "scan_id": scan_id} ) + if attack_paths_scan and result: + attack_paths_db_utils.set_attack_paths_scan_task_id( + tenant_id, attack_paths_scan.id, result.task_id + ) + @shared_task(base=RLSTask, name="provider-connection-check") @set_tenant @@ -373,7 +402,8 @@ class AttackPathsScanRLSTask(RLSTask): SDK initialization, or Neo4j configuration errors during setup). """ - def on_failure(self, exc, task_id, args, kwargs, _einfo): + def on_failure(self, exc, task_id, args, kwargs, _einfo): # noqa: ARG002 + del args # Required by Celery's Task.on_failure signature; not used. tenant_id = kwargs.get("tenant_id") scan_id = kwargs.get("scan_id") @@ -406,6 +436,11 @@ def perform_attack_paths_scan_task(self, tenant_id: str, scan_id: str): ) +@shared_task(name="attack-paths-cleanup-stale-scans", queue="attack-paths-scans") +def cleanup_stale_attack_paths_scans_task(): + return cleanup_stale_attack_paths_scans() + + @shared_task(name="tenant-deletion", queue="deletion", autoretry_for=(Exception,)) def delete_tenant_task(tenant_id: str): return delete_tenant(pk=tenant_id) @@ -434,6 +469,19 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str): scan_id (str): The scan identifier. provider_id (str): The provider_id id to be used in generating outputs. """ + try: + _cleanup_stale_tmp_output_directories( + DJANGO_TMP_OUTPUT_DIRECTORY, + max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS, + exclude_scan=(tenant_id, scan_id), + ) + except Exception as error: + logger.warning( + "Skipping stale tmp cleanup before output generation for scan %s: %s", + scan_id, + error, + ) + # Check if the scan has findings if not ScanSummary.objects.filter(scan_id=scan_id).exists(): logger.info(f"No findings found for scan {scan_id}") @@ -653,9 +701,9 @@ def backfill_finding_group_summaries_task(tenant_id: str, days: int = None): return backfill_finding_group_summaries(tenant_id=tenant_id, days=days) -@shared_task(name="backfill-scan-category-summaries", queue="backfill") +@shared_task(name="scan-category-summaries", queue="overview") @handle_provider_deletion -def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str): +def aggregate_scan_category_summaries_task(tenant_id: str, scan_id: str): """ Backfill ScanCategorySummary for a completed scan. @@ -665,12 +713,12 @@ def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str): tenant_id (str): The tenant identifier. scan_id (str): The scan identifier. """ - return backfill_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id) + return aggregate_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id) -@shared_task(name="backfill-scan-resource-group-summaries", queue="backfill") +@shared_task(name="scan-resource-group-summaries", queue="overview") @handle_provider_deletion -def backfill_scan_resource_group_summaries_task(tenant_id: str, scan_id: str): +def aggregate_scan_resource_group_summaries_task(tenant_id: str, scan_id: str): """ Backfill ScanGroupSummary for a completed scan. @@ -680,7 +728,7 @@ def backfill_scan_resource_group_summaries_task(tenant_id: str, scan_id: str): tenant_id (str): The tenant identifier. scan_id (str): The scan identifier. """ - return backfill_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id) + return aggregate_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id) @shared_task(name="backfill-provider-compliance-scores", queue="backfill") @@ -752,6 +800,32 @@ def aggregate_daily_severity_task(tenant_id: str, scan_id: str): return aggregate_daily_severity(tenant_id=tenant_id, scan_id=scan_id) +@shared_task(name="scan-reset-ephemeral-resources", queue="overview") +@handle_provider_deletion +def reset_ephemeral_resource_findings_count_task(tenant_id: str, scan_id: str): + """Reset failed_findings_count for resources missing from a completed full-scope scan. + + Failures are swallowed and returned as a status: this task lives inside the + post-scan group, and Celery propagates group-member exceptions into the next + chain step — meaning a crash here would block compliance reports and + integrations. The reset is purely cosmetic (UI sort optimization), so a + bad run is logged and absorbed rather than allowed to cascade. + """ + try: + return reset_ephemeral_resource_findings_count( + tenant_id=tenant_id, scan_id=scan_id + ) + except Exception as exc: # noqa: BLE001 — intentionally broad + logger.exception( + f"reset_ephemeral_resource_findings_count failed for scan {scan_id}: {exc}" + ) + return { + "status": "failed", + "scan_id": str(scan_id), + "reason": str(exc), + } + + @shared_task(base=RLSTask, name="scan-finding-group-summaries", queue="overview") @set_tenant(keep_tenant=True) @handle_provider_deletion @@ -760,6 +834,87 @@ def aggregate_finding_group_summaries_task(tenant_id: str, scan_id: str): return aggregate_finding_group_summaries(tenant_id=tenant_id, scan_id=scan_id) +@shared_task( + base=RLSTask, name="reaggregate-all-finding-group-summaries", queue="overview" +) +@set_tenant(keep_tenant=True) +def reaggregate_all_finding_group_summaries_task(tenant_id: str): + """Reaggregate every pre-aggregated summary table for this tenant. + + Mirrors the unbounded scope of `mute_historical_findings_task`: that task + rewrites every Finding row whose UID matches a mute rule, with no time + limit. To keep the pre-aggregated tables consistent with that update, + this task re-runs the same per-scan aggregation pipeline that scan + completion runs on the latest completed scan of every (provider, day) + pair, rebuilding the tables that power the read endpoints: + + - `ScanSummary` and `DailySeveritySummary` -> `/overviews/findings`, + `/overviews/findings-severity`, `/overviews/services`. + - `FindingGroupDailySummary` -> `/finding-groups` and + `/finding-groups/latest`. + - `ScanGroupSummary` -> `/overviews/resource-groups` (resource + inventory). + - `ScanCategorySummary` -> `/overviews/categories`. + - `AttackSurfaceOverview` -> `/overviews/attack-surfaces`. + + Per-scan pipelines are dispatched in parallel via a Celery group so + wallclock scales with the worker pool. + """ + completed_scans = list( + Scan.objects.filter( + tenant_id=tenant_id, + state=StateChoices.COMPLETED, + completed_at__isnull=False, + ) + .order_by("-completed_at") + .values("id", "completed_at", "provider_id") + ) + + # Keep the latest scan per (provider, day) pair so the daily summary row + # the aggregator writes is the most recent snapshot of that day for that + # provider. Iterating from most recent to oldest means the first scan we + # see for a given key wins. + latest_scans: dict[tuple, str] = {} + for scan in completed_scans: + key = (scan["provider_id"], scan["completed_at"].date()) + if key not in latest_scans: + latest_scans[key] = str(scan["id"]) + + scan_ids = list(latest_scans.values()) + if scan_ids: + logger.info( + "Reaggregating overview/finding summaries for %d scans (provider x day)", + len(scan_ids), + ) + # DailySeveritySummary reads from ScanSummary, so ScanSummary must be + # recomputed first; the other aggregators read Finding directly and + # can run in parallel with the severity step. + group( + chain( + perform_scan_summary_task.si(tenant_id=tenant_id, scan_id=scan_id), + group( + aggregate_daily_severity_task.si( + tenant_id=tenant_id, scan_id=scan_id + ), + aggregate_finding_group_summaries_task.si( + tenant_id=tenant_id, scan_id=scan_id + ), + aggregate_scan_resource_group_summaries_task.si( + tenant_id=tenant_id, scan_id=scan_id + ), + aggregate_scan_category_summaries_task.si( + tenant_id=tenant_id, scan_id=scan_id + ), + aggregate_attack_surface_task.si( + tenant_id=tenant_id, scan_id=scan_id + ), + ), + ) + for scan_id in scan_ids + ).apply_async() + return {"scans_reaggregated": len(scan_ids)} + + @shared_task(base=RLSTask, name="lighthouse-connection-check") @set_tenant def check_lighthouse_connection_task(lighthouse_config_id: str, tenant_id: str = None): @@ -926,13 +1081,17 @@ def jira_integration_task( @handle_provider_deletion def generate_compliance_reports_task(tenant_id: str, scan_id: str, provider_id: str): """ - Optimized task to generate ThreatScore, ENS, NIS2, and CSA CCM reports with shared queries. + Optimized task to generate ThreatScore, ENS, NIS2, CSA CCM and CIS reports with shared queries. This task is more efficient than running separate report tasks because it reuses database queries: - Provider object fetched once (instead of multiple times) - Requirement statistics aggregated once (instead of multiple times) - Can reduce database load by up to 50-70% + CIS emits a single PDF per run: the one matching the highest CIS version + available for the scan's provider, picked dynamically from + ``Compliance.get_bulk`` (no hard-coded provider → version mapping). + Args: tenant_id (str): The tenant identifier. scan_id (str): The scan identifier. @@ -949,6 +1108,7 @@ def generate_compliance_reports_task(tenant_id: str, scan_id: str, provider_id: generate_ens=True, generate_nis2=True, generate_csa=True, + generate_cis=True, ) diff --git a/api/src/backend/tasks/tests/test_attack_paths_scan.py b/api/src/backend/tasks/tests/test_attack_paths_scan.py index 8132b7dad3..918e54ff6c 100644 --- a/api/src/backend/tasks/tests/test_attack_paths_scan.py +++ b/api/src/backend/tasks/tests/test_attack_paths_scan.py @@ -1,14 +1,14 @@ from contextlib import nullcontext +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from unittest.mock import MagicMock, call, patch import pytest +from django_celery_results.models import TaskResult from tasks.jobs.attack_paths import findings as findings_module +from tasks.jobs.attack_paths import indexes as indexes_module from tasks.jobs.attack_paths import internet as internet_module from tasks.jobs.attack_paths import sync as sync_module -from tasks.jobs.attack_paths.config import ( - get_deprecated_provider_resource_label, -) from tasks.jobs.attack_paths.scan import run as attack_paths_run from api.models import ( @@ -20,6 +20,7 @@ from api.models import ( Scan, StateChoices, StatusChoices, + Task, ) from prowler.lib.check.models import Severity @@ -37,12 +38,15 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.sync.sync_graph") - @patch("tasks.jobs.attack_paths.scan.graph_database.drop_subgraph") - @patch("tasks.jobs.attack_paths.scan.sync.create_sync_indexes") + @patch( + "tasks.jobs.attack_paths.scan.sync.sync_graph", + return_value={"nodes": 0, "relationships": 0}, + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.drop_subgraph", return_value=0) + @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") @patch("tasks.jobs.attack_paths.scan.internet.analysis") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") - @patch("tasks.jobs.attack_paths.scan.findings.create_findings_indexes") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) + @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run") @@ -131,7 +135,7 @@ class TestAttackPathsRun: assert result == ingestion_result mock_retrieve_scan.assert_called_once_with(str(tenant.id), str(scan.id)) mock_starting.assert_called_once() - config = mock_starting.call_args[0][2] + config = mock_starting.call_args[0][1] assert config.neo4j_database == "tenant-db" mock_get_db_name.assert_has_calls( [call(attack_paths_scan.id, temporary=True), call(provider.tenant_id)] @@ -154,6 +158,7 @@ class TestAttackPathsRun: mock_sync.assert_called_once_with( source_database="db-scan-id", target_database="tenant-db", + tenant_id=str(provider.tenant_id), provider_id=str(provider.id), ) mock_get_ingestion.assert_called_once_with(provider.provider) @@ -186,9 +191,9 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.internet.analysis") - @patch("tasks.jobs.attack_paths.scan.findings.create_findings_indexes") + @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run") @patch("tasks.jobs.attack_paths.scan.graph_database.create_database") @@ -271,6 +276,106 @@ class TestAttackPathsRun: assert failure_args[1] == StateChoices.FAILED assert failure_args[2] == {"global_error": "Cartography failed: ingestion boom"} + @patch( + "tasks.jobs.attack_paths.scan.utils.stringify_exception", + return_value="Cartography failed: ingestion boom", + ) + @patch( + "tasks.jobs.attack_paths.scan.utils.call_within_event_loop", + side_effect=lambda fn, *a, **kw: fn(*a, **kw), + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.drop_database") + @patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan") + @patch("tasks.jobs.attack_paths.scan.db_utils.set_graph_data_ready") + @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") + @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") + @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) + @patch("tasks.jobs.attack_paths.scan.internet.analysis") + @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") + @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") + @patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run") + @patch("tasks.jobs.attack_paths.scan.graph_database.create_database") + @patch( + "tasks.jobs.attack_paths.scan.graph_database.get_database_name", + return_value="db-scan-id", + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.get_uri") + @patch( + "tasks.jobs.attack_paths.scan.initialize_prowler_provider", + return_value=MagicMock(_enabled_regions=["us-east-1"]), + ) + @patch( + "tasks.jobs.attack_paths.scan.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + def test_failure_before_gate_does_not_flip_graph_data_ready_true( + self, + mock_init_provider, + mock_get_uri, + mock_get_db_name, + mock_create_db, + mock_cartography_indexes, + mock_cartography_analysis, + mock_findings_indexes, + mock_internet_analysis, + mock_findings_analysis, + mock_starting, + mock_update_progress, + mock_set_provider_graph_data_ready, + mock_set_graph_data_ready, + mock_finish, + mock_drop_db, + mock_event_loop, + mock_stringify, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + """Failure during ingestion (before set_provider_graph_data_ready(False)) + must NOT flip graph_data_ready to True for providers that never had data.""" + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + scan = scans_fixture[0] + scan.provider = provider + scan.save() + + attack_paths_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=scan, + state=StateChoices.SCHEDULED, + ) + + mock_session = MagicMock() + session_ctx = MagicMock() + session_ctx.__enter__.return_value = mock_session + session_ctx.__exit__.return_value = False + ingestion_fn = MagicMock(side_effect=RuntimeError("ingestion boom")) + + with ( + patch( + "tasks.jobs.attack_paths.scan.graph_database.get_session", + return_value=session_ctx, + ), + patch( + "tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan", + return_value=attack_paths_scan, + ), + patch( + "tasks.jobs.attack_paths.scan.get_cartography_ingestion_function", + return_value=ingestion_fn, + ), + ): + with pytest.raises(RuntimeError, match="ingestion boom"): + attack_paths_run(str(tenant.id), str(scan.id), "task-456") + + # Gate was never applied, so recovery must not flip anything to True + mock_set_provider_graph_data_ready.assert_not_called() + mock_set_graph_data_ready.assert_not_called() + @patch( "tasks.jobs.attack_paths.scan.utils.stringify_exception", return_value="Cartography failed: ingestion boom", @@ -288,9 +393,9 @@ class TestAttackPathsRun: @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") - @patch("tasks.jobs.attack_paths.scan.findings.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) @patch("tasks.jobs.attack_paths.scan.internet.analysis") - @patch("tasks.jobs.attack_paths.scan.findings.create_findings_indexes") + @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") @patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run") @patch("tasks.jobs.attack_paths.scan.graph_database.create_database") @@ -373,6 +478,474 @@ class TestAttackPathsRun: assert failure_args[1] == StateChoices.FAILED assert failure_args[2] == {"global_error": "Cartography failed: ingestion boom"} + @patch( + "tasks.jobs.attack_paths.scan.utils.stringify_exception", + return_value="Attack Paths scan failed: drop failed", + ) + @patch( + "tasks.jobs.attack_paths.scan.utils.call_within_event_loop", + side_effect=lambda fn, *a, **kw: fn(*a, **kw), + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.drop_database") + @patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan") + @patch("tasks.jobs.attack_paths.scan.db_utils.set_graph_data_ready") + @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") + @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") + @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") + @patch( + "tasks.jobs.attack_paths.scan.sync.sync_graph", + return_value={"nodes": 0, "relationships": 0}, + ) + @patch( + "tasks.jobs.attack_paths.scan.graph_database.drop_subgraph", + side_effect=RuntimeError("drop failed"), + ) + @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") + @patch("tasks.jobs.attack_paths.scan.internet.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) + @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") + @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") + @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") + @patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run") + @patch("tasks.jobs.attack_paths.scan.graph_database.clear_cache") + @patch("tasks.jobs.attack_paths.scan.graph_database.create_database") + @patch( + "tasks.jobs.attack_paths.scan.graph_database.get_uri", + return_value="bolt://neo4j", + ) + @patch( + "tasks.jobs.attack_paths.scan.initialize_prowler_provider", + return_value=MagicMock(_enabled_regions=["us-east-1"]), + ) + @patch( + "tasks.jobs.attack_paths.scan.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + def test_failure_after_gate_before_drop_restores_graph_data_ready( + self, + mock_init_provider, + mock_get_uri, + mock_create_db, + mock_clear_cache, + mock_cartography_indexes, + mock_cartography_analysis, + mock_cartography_ontology, + mock_findings_indexes, + mock_findings_analysis, + mock_internet_analysis, + mock_sync_indexes, + mock_drop_subgraph, + mock_sync, + mock_starting, + mock_update_progress, + mock_set_provider_graph_data_ready, + mock_set_graph_data_ready, + mock_finish, + mock_drop_db, + mock_event_loop, + mock_stringify, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + scan = scans_fixture[0] + scan.provider = provider + scan.save() + + attack_paths_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=scan, + state=StateChoices.SCHEDULED, + ) + + mock_session = MagicMock() + session_ctx = MagicMock() + session_ctx.__enter__.return_value = mock_session + session_ctx.__exit__.return_value = False + + with ( + patch( + "tasks.jobs.attack_paths.scan.graph_database.get_database_name", + side_effect=["db-scan-id", "tenant-db"], + ), + patch( + "tasks.jobs.attack_paths.scan.graph_database.get_session", + return_value=session_ctx, + ), + patch( + "tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan", + return_value=attack_paths_scan, + ), + patch( + "tasks.jobs.attack_paths.scan.get_cartography_ingestion_function", + return_value=MagicMock(return_value={}), + ), + ): + with pytest.raises(RuntimeError, match="drop failed"): + attack_paths_run(str(tenant.id), str(scan.id), "task-456") + + assert mock_set_provider_graph_data_ready.call_args_list == [ + call(attack_paths_scan, False), + call(attack_paths_scan, True), + ] + + @patch( + "tasks.jobs.attack_paths.scan.utils.stringify_exception", + return_value="Attack Paths scan failed: sync failed", + ) + @patch( + "tasks.jobs.attack_paths.scan.utils.call_within_event_loop", + side_effect=lambda fn, *a, **kw: fn(*a, **kw), + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.drop_database") + @patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan") + @patch("tasks.jobs.attack_paths.scan.db_utils.set_graph_data_ready") + @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") + @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") + @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") + @patch( + "tasks.jobs.attack_paths.scan.sync.sync_graph", + side_effect=RuntimeError("sync failed"), + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.drop_subgraph") + @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") + @patch("tasks.jobs.attack_paths.scan.internet.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) + @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") + @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") + @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") + @patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run") + @patch("tasks.jobs.attack_paths.scan.graph_database.clear_cache") + @patch("tasks.jobs.attack_paths.scan.graph_database.create_database") + @patch( + "tasks.jobs.attack_paths.scan.graph_database.get_uri", + return_value="bolt://neo4j", + ) + @patch( + "tasks.jobs.attack_paths.scan.initialize_prowler_provider", + return_value=MagicMock(_enabled_regions=["us-east-1"]), + ) + @patch( + "tasks.jobs.attack_paths.scan.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + def test_failure_after_drop_before_sync_leaves_graph_data_ready_false( + self, + mock_init_provider, + mock_get_uri, + mock_create_db, + mock_clear_cache, + mock_cartography_indexes, + mock_cartography_analysis, + mock_cartography_ontology, + mock_findings_indexes, + mock_findings_analysis, + mock_internet_analysis, + mock_sync_indexes, + mock_drop_subgraph, + mock_sync, + mock_starting, + mock_update_progress, + mock_set_provider_graph_data_ready, + mock_set_graph_data_ready, + mock_finish, + mock_drop_db, + mock_event_loop, + mock_stringify, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + scan = scans_fixture[0] + scan.provider = provider + scan.save() + + attack_paths_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=scan, + state=StateChoices.SCHEDULED, + ) + + mock_session = MagicMock() + session_ctx = MagicMock() + session_ctx.__enter__.return_value = mock_session + session_ctx.__exit__.return_value = False + + with ( + patch( + "tasks.jobs.attack_paths.scan.graph_database.get_database_name", + side_effect=["db-scan-id", "tenant-db"], + ), + patch( + "tasks.jobs.attack_paths.scan.graph_database.get_session", + return_value=session_ctx, + ), + patch( + "tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan", + return_value=attack_paths_scan, + ), + patch( + "tasks.jobs.attack_paths.scan.get_cartography_ingestion_function", + return_value=MagicMock(return_value={}), + ), + ): + with pytest.raises(RuntimeError, match="sync failed"): + attack_paths_run(str(tenant.id), str(scan.id), "task-456") + + # Only called with False (gate), never with True (no recovery for partial data) + mock_set_provider_graph_data_ready.assert_called_once_with( + attack_paths_scan, False + ) + + @patch( + "tasks.jobs.attack_paths.scan.utils.stringify_exception", + return_value="Attack Paths scan failed: flag failed", + ) + @patch( + "tasks.jobs.attack_paths.scan.utils.call_within_event_loop", + side_effect=lambda fn, *a, **kw: fn(*a, **kw), + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.drop_database") + @patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan") + @patch( + "tasks.jobs.attack_paths.scan.db_utils.set_graph_data_ready", + side_effect=[RuntimeError("flag failed"), None], + ) + @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") + @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") + @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") + @patch( + "tasks.jobs.attack_paths.scan.sync.sync_graph", + return_value={"nodes": 0, "relationships": 0}, + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.drop_subgraph") + @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") + @patch("tasks.jobs.attack_paths.scan.internet.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) + @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") + @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") + @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") + @patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run") + @patch("tasks.jobs.attack_paths.scan.graph_database.clear_cache") + @patch("tasks.jobs.attack_paths.scan.graph_database.create_database") + @patch( + "tasks.jobs.attack_paths.scan.graph_database.get_uri", + return_value="bolt://neo4j", + ) + @patch( + "tasks.jobs.attack_paths.scan.initialize_prowler_provider", + return_value=MagicMock(_enabled_regions=["us-east-1"]), + ) + @patch( + "tasks.jobs.attack_paths.scan.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + def test_failure_after_sync_restores_graph_data_ready( + self, + mock_init_provider, + mock_get_uri, + mock_create_db, + mock_clear_cache, + mock_cartography_indexes, + mock_cartography_analysis, + mock_cartography_ontology, + mock_findings_indexes, + mock_findings_analysis, + mock_internet_analysis, + mock_sync_indexes, + mock_drop_subgraph, + mock_sync, + mock_starting, + mock_update_progress, + mock_set_provider_graph_data_ready, + mock_set_graph_data_ready, + mock_finish, + mock_drop_db, + mock_event_loop, + mock_stringify, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + scan = scans_fixture[0] + scan.provider = provider + scan.save() + + attack_paths_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=scan, + state=StateChoices.SCHEDULED, + ) + + mock_session = MagicMock() + session_ctx = MagicMock() + session_ctx.__enter__.return_value = mock_session + session_ctx.__exit__.return_value = False + + with ( + patch( + "tasks.jobs.attack_paths.scan.graph_database.get_database_name", + side_effect=["db-scan-id", "tenant-db"], + ), + patch( + "tasks.jobs.attack_paths.scan.graph_database.get_session", + return_value=session_ctx, + ), + patch( + "tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan", + return_value=attack_paths_scan, + ), + patch( + "tasks.jobs.attack_paths.scan.get_cartography_ingestion_function", + return_value=MagicMock(return_value={}), + ), + ): + with pytest.raises(RuntimeError, match="flag failed"): + attack_paths_run(str(tenant.id), str(scan.id), "task-456") + + # sync completed: first call (normal path) raised, recovery retried and succeeded + assert mock_set_graph_data_ready.call_args_list == [ + call(attack_paths_scan, True), + call(attack_paths_scan, True), + ] + # set_provider_graph_data_ready only called once with False (the gate) + mock_set_provider_graph_data_ready.assert_called_once_with( + attack_paths_scan, False + ) + + @patch( + "tasks.jobs.attack_paths.scan.utils.stringify_exception", + return_value="Attack Paths scan failed: drop failed", + ) + @patch( + "tasks.jobs.attack_paths.scan.utils.call_within_event_loop", + side_effect=lambda fn, *a, **kw: fn(*a, **kw), + ) + @patch("tasks.jobs.attack_paths.scan.graph_database.drop_database") + @patch("tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan") + @patch("tasks.jobs.attack_paths.scan.db_utils.set_graph_data_ready") + @patch("tasks.jobs.attack_paths.scan.db_utils.set_provider_graph_data_ready") + @patch("tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress") + @patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan") + @patch( + "tasks.jobs.attack_paths.scan.sync.sync_graph", + return_value={"nodes": 0, "relationships": 0}, + ) + @patch( + "tasks.jobs.attack_paths.scan.graph_database.drop_subgraph", + side_effect=RuntimeError("drop failed"), + ) + @patch("tasks.jobs.attack_paths.scan.indexes.create_sync_indexes") + @patch("tasks.jobs.attack_paths.scan.internet.analysis") + @patch("tasks.jobs.attack_paths.scan.findings.analysis", return_value=(0, 0)) + @patch("tasks.jobs.attack_paths.scan.indexes.create_findings_indexes") + @patch("tasks.jobs.attack_paths.scan.cartography_ontology.run") + @patch("tasks.jobs.attack_paths.scan.cartography_analysis.run") + @patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run") + @patch("tasks.jobs.attack_paths.scan.graph_database.clear_cache") + @patch("tasks.jobs.attack_paths.scan.graph_database.create_database") + @patch( + "tasks.jobs.attack_paths.scan.graph_database.get_uri", + return_value="bolt://neo4j", + ) + @patch( + "tasks.jobs.attack_paths.scan.initialize_prowler_provider", + return_value=MagicMock(_enabled_regions=["us-east-1"]), + ) + @patch( + "tasks.jobs.attack_paths.scan.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + def test_recovery_failure_does_not_suppress_original_exception( + self, + mock_init_provider, + mock_get_uri, + mock_create_db, + mock_clear_cache, + mock_cartography_indexes, + mock_cartography_analysis, + mock_cartography_ontology, + mock_findings_indexes, + mock_findings_analysis, + mock_internet_analysis, + mock_sync_indexes, + mock_drop_subgraph, + mock_sync, + mock_starting, + mock_update_progress, + mock_set_provider_graph_data_ready, + mock_set_graph_data_ready, + mock_finish, + mock_drop_db, + mock_event_loop, + mock_stringify, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + scan = scans_fixture[0] + scan.provider = provider + scan.save() + + attack_paths_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=scan, + state=StateChoices.SCHEDULED, + ) + + # Recovery itself fails on the second call (True) + mock_set_provider_graph_data_ready.side_effect = [ + None, + RuntimeError("recovery boom"), + ] + + mock_session = MagicMock() + session_ctx = MagicMock() + session_ctx.__enter__.return_value = mock_session + session_ctx.__exit__.return_value = False + + with ( + patch( + "tasks.jobs.attack_paths.scan.graph_database.get_database_name", + side_effect=["db-scan-id", "tenant-db"], + ), + patch( + "tasks.jobs.attack_paths.scan.graph_database.get_session", + return_value=session_ctx, + ), + patch( + "tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan", + return_value=attack_paths_scan, + ), + patch( + "tasks.jobs.attack_paths.scan.get_cartography_ingestion_function", + return_value=MagicMock(return_value={}), + ), + ): + # Original exception propagates despite recovery failure + with pytest.raises(RuntimeError, match="drop failed"): + attack_paths_run(str(tenant.id), str(scan.id), "task-456") + def test_run_returns_early_for_unsupported_provider(self, tenants_fixture): tenant = tenants_fixture[0] provider = Provider.objects.create( @@ -421,9 +994,7 @@ class TestFailAttackPathsScan: def test_marks_executing_scan_as_failed( self, tenants_fixture, providers_fixture, scans_fixture ): - from tasks.jobs.attack_paths.db_utils import ( - fail_attack_paths_scan, - ) + from tasks.jobs.attack_paths.db_utils import fail_attack_paths_scan tenant = tenants_fixture[0] provider = providers_fixture[0] @@ -448,27 +1019,24 @@ class TestFailAttackPathsScan: patch( "tasks.jobs.attack_paths.db_utils.graph_database.drop_database" ) as mock_drop_db, - patch( - "tasks.jobs.attack_paths.db_utils.finish_attack_paths_scan" - ) as mock_finish, + patch("tasks.jobs.attack_paths.db_utils.recover_graph_data_ready"), ): fail_attack_paths_scan(str(tenant.id), str(scan.id), "setup exploded") mock_retrieve.assert_called_once_with(str(tenant.id), str(scan.id)) expected_tmp_db = f"db-tmp-scan-{str(attack_paths_scan.id).lower()}" mock_drop_db.assert_called_once_with(expected_tmp_db) - mock_finish.assert_called_once_with( - attack_paths_scan, - StateChoices.FAILED, - {"global_error": "setup exploded"}, - ) + + attack_paths_scan.refresh_from_db() + assert attack_paths_scan.state == StateChoices.FAILED + assert attack_paths_scan.ingestion_exceptions == { + "global_error": "setup exploded" + } def test_drops_temp_database_even_when_drop_fails( self, tenants_fixture, providers_fixture, scans_fixture ): - from tasks.jobs.attack_paths.db_utils import ( - fail_attack_paths_scan, - ) + from tasks.jobs.attack_paths.db_utils import fail_attack_paths_scan tenant = tenants_fixture[0] provider = providers_fixture[0] @@ -494,24 +1062,17 @@ class TestFailAttackPathsScan: "tasks.jobs.attack_paths.db_utils.graph_database.drop_database", side_effect=Exception("Neo4j unreachable"), ), - patch( - "tasks.jobs.attack_paths.db_utils.finish_attack_paths_scan" - ) as mock_finish, + patch("tasks.jobs.attack_paths.db_utils.recover_graph_data_ready"), ): fail_attack_paths_scan(str(tenant.id), str(scan.id), "setup exploded") - mock_finish.assert_called_once_with( - attack_paths_scan, - StateChoices.FAILED, - {"global_error": "setup exploded"}, - ) + attack_paths_scan.refresh_from_db() + assert attack_paths_scan.state == StateChoices.FAILED def test_skips_already_failed_scan( self, tenants_fixture, providers_fixture, scans_fixture ): - from tasks.jobs.attack_paths.db_utils import ( - fail_attack_paths_scan, - ) + from tasks.jobs.attack_paths.db_utils import fail_attack_paths_scan tenant = tenants_fixture[0] provider = providers_fixture[0] @@ -536,34 +1097,127 @@ class TestFailAttackPathsScan: patch( "tasks.jobs.attack_paths.db_utils.graph_database.drop_database" ) as mock_drop_db, - patch( - "tasks.jobs.attack_paths.db_utils.finish_attack_paths_scan" - ) as mock_finish, ): fail_attack_paths_scan(str(tenant.id), str(scan.id), "setup exploded") - mock_drop_db.assert_not_called() - mock_finish.assert_not_called() + mock_drop_db.assert_called_once() + + attack_paths_scan.refresh_from_db() + assert attack_paths_scan.state == StateChoices.FAILED def test_skips_when_no_scan_found(self, tenants_fixture): - from tasks.jobs.attack_paths.db_utils import ( - fail_attack_paths_scan, - ) + from tasks.jobs.attack_paths.db_utils import fail_attack_paths_scan tenant = tenants_fixture[0] + with patch( + "tasks.jobs.attack_paths.db_utils.retrieve_attack_paths_scan", + return_value=None, + ): + fail_attack_paths_scan(str(tenant.id), "nonexistent", "setup exploded") + + def test_fail_recovers_graph_data_ready_when_data_exists( + self, tenants_fixture, providers_fixture, scans_fixture + ): + from tasks.jobs.attack_paths.db_utils import fail_attack_paths_scan + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + scan = scans_fixture[0] + scan.provider = provider + scan.save() + + attack_paths_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=scan, + state=StateChoices.EXECUTING, + ) + with ( patch( "tasks.jobs.attack_paths.db_utils.retrieve_attack_paths_scan", - return_value=None, + return_value=attack_paths_scan, + ), + patch("tasks.jobs.attack_paths.db_utils.graph_database.drop_database"), + patch( + "tasks.jobs.attack_paths.db_utils.graph_database.has_provider_data", + return_value=True, ), patch( - "tasks.jobs.attack_paths.db_utils.finish_attack_paths_scan" - ) as mock_finish, + "tasks.jobs.attack_paths.db_utils.set_provider_graph_data_ready" + ) as mock_set_ready, ): - fail_attack_paths_scan(str(tenant.id), "nonexistent", "setup exploded") + fail_attack_paths_scan(str(tenant.id), str(scan.id), "worker died") - mock_finish.assert_not_called() + mock_set_ready.assert_called_once_with(attack_paths_scan, True) + + def test_fail_leaves_graph_data_ready_false_when_no_data( + self, tenants_fixture, providers_fixture, scans_fixture + ): + from tasks.jobs.attack_paths.db_utils import fail_attack_paths_scan + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + scan = scans_fixture[0] + scan.provider = provider + scan.save() + + attack_paths_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=scan, + state=StateChoices.EXECUTING, + ) + + with ( + patch( + "tasks.jobs.attack_paths.db_utils.retrieve_attack_paths_scan", + return_value=attack_paths_scan, + ), + patch("tasks.jobs.attack_paths.db_utils.graph_database.drop_database"), + patch( + "tasks.jobs.attack_paths.db_utils.graph_database.has_provider_data", + return_value=False, + ), + patch( + "tasks.jobs.attack_paths.db_utils.set_provider_graph_data_ready" + ) as mock_set_ready, + ): + fail_attack_paths_scan(str(tenant.id), str(scan.id), "worker died") + + mock_set_ready.assert_not_called() + + def test_recover_graph_data_ready_never_raises( + self, tenants_fixture, providers_fixture, scans_fixture + ): + from tasks.jobs.attack_paths.db_utils import recover_graph_data_ready + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + scan = scans_fixture[0] + scan.provider = provider + scan.save() + + attack_paths_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=scan, + state=StateChoices.EXECUTING, + ) + + with patch( + "tasks.jobs.attack_paths.db_utils.graph_database.has_provider_data", + side_effect=Exception("Neo4j unreachable"), + ): + # Should not raise + recover_graph_data_ready(attack_paths_scan) class TestAttackPathsScanRLSTaskOnFailure: @@ -609,7 +1263,7 @@ class TestAttackPathsFindingsHelpers: def test_create_findings_indexes_executes_all_statements(self): mock_session = MagicMock() with patch("tasks.jobs.attack_paths.indexes.run_write_query") as mock_run_write: - findings_module.create_findings_indexes(mock_session) + indexes_module.create_findings_indexes(mock_session) from tasks.jobs.attack_paths.indexes import FINDINGS_INDEX_STATEMENTS @@ -623,33 +1277,30 @@ class TestAttackPathsFindingsHelpers: provider.provider = Provider.ProviderChoices.AWS provider.save() - # Create mock Finding objects with to_dict() method - mock_finding_1 = MagicMock() - mock_finding_1.to_dict.return_value = {"id": "1", "resource_uid": "r-1"} - mock_finding_2 = MagicMock() - mock_finding_2.to_dict.return_value = {"id": "2", "resource_uid": "r-2"} - - # Create a generator that yields two batches of Finding instances + # Create a generator that yields two batches of dicts (pre-converted) def findings_generator(): - yield [mock_finding_1] - yield [mock_finding_2] + yield [{"id": "1", "resource_uid": "r-1"}] + yield [{"id": "2", "resource_uid": "r-2"}] config = SimpleNamespace(update_tag=12345) mock_session = MagicMock() + first_result = MagicMock() + first_result.single.return_value = {"merged_count": 1, "dropped_count": 0} + second_result = MagicMock() + second_result.single.return_value = {"merged_count": 0, "dropped_count": 1} + mock_session.run.side_effect = [first_result, second_result] + with ( - patch( - "tasks.jobs.attack_paths.findings.get_root_node_label", - return_value="AWSAccount", - ), patch( "tasks.jobs.attack_paths.findings.get_node_uid_field", return_value="arn", ), patch( "tasks.jobs.attack_paths.findings.get_provider_resource_label", - return_value="AWSResource", + return_value="_AWSResource", ), + patch("tasks.jobs.attack_paths.findings.logger") as mock_logger, ): findings_module.load_findings( mock_session, findings_generator(), provider, config @@ -658,27 +1309,16 @@ class TestAttackPathsFindingsHelpers: assert mock_session.run.call_count == 2 for call_args in mock_session.run.call_args_list: params = call_args.args[1] - assert params["provider_uid"] == str(provider.uid) assert params["last_updated"] == config.update_tag assert "findings_data" in params - def test_cleanup_findings_runs_batches(self, providers_fixture): - provider = providers_fixture[0] - config = SimpleNamespace(update_tag=1024) - mock_session = MagicMock() - - first_batch = MagicMock() - first_batch.single.return_value = {"deleted_findings_count": 3} - second_batch = MagicMock() - second_batch.single.return_value = {"deleted_findings_count": 0} - mock_session.run.side_effect = [first_batch, second_batch] - - findings_module.cleanup_findings(mock_session, provider, config) - - assert mock_session.run.call_count == 2 - params = mock_session.run.call_args.args[1] - assert params["provider_uid"] == str(provider.uid) - assert params["last_updated"] == config.update_tag + summary_log = next( + call_args.args[0] + for call_args in mock_logger.info.call_args_list + if call_args.args and "Finished loading" in call_args.args[0] + ) + assert "edges_merged=1" in summary_log + assert "edges_dropped=1" in summary_log def test_stream_findings_with_resources_returns_latest_scan_data( self, @@ -779,17 +1419,17 @@ class TestAttackPathsFindingsHelpers: assert len(findings_data) == 1 finding_result = findings_data[0] - assert finding_result.id == str(finding.id) - assert finding_result.resource_uid == resource.uid - assert finding_result.check_title == "Check title" - assert finding_result.scan_id == str(latest_scan.id) + assert finding_result["id"] == str(finding.id) + assert finding_result["resource_uid"] == resource.uid + assert finding_result["check_title"] == "Check title" + assert finding_result["scan_id"] == str(latest_scan.id) def test_enrich_batch_with_resources_single_resource( self, tenants_fixture, providers_fixture, ): - """One finding + one resource = one output Finding instance""" + """One finding + one resource = one output dict""" tenant = tenants_fixture[0] provider = providers_fixture[0] provider.provider = Provider.ProviderChoices.AWS @@ -859,20 +1499,21 @@ class TestAttackPathsFindingsHelpers: "default", ): result = findings_module._enrich_batch_with_resources( - [finding_dict], str(tenant.id) + [finding_dict], str(tenant.id), lambda uid: f"short:{uid}" ) assert len(result) == 1 - assert result[0].resource_uid == resource.uid - assert result[0].id == str(finding.id) - assert result[0].status == "FAIL" + assert result[0]["resource_uid"] == resource.uid + assert result[0]["resource_short_uid"] == f"short:{resource.uid}" + assert result[0]["id"] == str(finding.id) + assert result[0]["status"] == "FAIL" def test_enrich_batch_with_resources_multiple_resources( self, tenants_fixture, providers_fixture, ): - """One finding + three resources = three output Finding instances""" + """One finding + three resources = three output dicts""" tenant = tenants_fixture[0] provider = providers_fixture[0] provider.provider = Provider.ProviderChoices.AWS @@ -947,17 +1588,17 @@ class TestAttackPathsFindingsHelpers: "default", ): result = findings_module._enrich_batch_with_resources( - [finding_dict], str(tenant.id) + [finding_dict], str(tenant.id), lambda uid: uid ) assert len(result) == 3 - result_resource_uids = {r.resource_uid for r in result} + result_resource_uids = {r["resource_uid"] for r in result} assert result_resource_uids == {r.uid for r in resources} # All should have same finding data for r in result: - assert r.id == str(finding.id) - assert r.status == "FAIL" + assert r["id"] == str(finding.id) + assert r["status"] == "FAIL" def test_enrich_batch_with_resources_no_resources_skips( self, @@ -1021,7 +1662,7 @@ class TestAttackPathsFindingsHelpers: patch("tasks.jobs.attack_paths.findings.logger") as mock_logger, ): result = findings_module._enrich_batch_with_resources( - [finding_dict], str(tenant.id) + [finding_dict], str(tenant.id), lambda uid: uid ) assert len(result) == 0 @@ -1034,16 +1675,12 @@ class TestAttackPathsFindingsHelpers: provider.save() scan_id = "some-scan-id" - with ( - patch("tasks.jobs.attack_paths.findings.rls_transaction") as mock_rls, - patch("tasks.jobs.attack_paths.findings.Finding") as mock_finding, - ): + with patch("tasks.jobs.attack_paths.findings.rls_transaction") as mock_rls: # Create generator but don't iterate findings_module.stream_findings_with_resources(provider, scan_id) # Nothing should be called yet mock_rls.assert_not_called() - mock_finding.objects.filter.assert_not_called() def test_load_findings_empty_generator(self, providers_fixture): """Empty generator should not call neo4j""" @@ -1059,37 +1696,79 @@ class TestAttackPathsFindingsHelpers: yield # Make it a generator with ( - patch( - "tasks.jobs.attack_paths.findings.get_root_node_label", - return_value="AWSAccount", - ), patch( "tasks.jobs.attack_paths.findings.get_node_uid_field", return_value="arn", ), patch( "tasks.jobs.attack_paths.findings.get_provider_resource_label", - return_value="AWSResource", + return_value="_AWSResource", ), ): findings_module.load_findings(mock_session, empty_gen(), provider, config) mock_session.run.assert_not_called() + @pytest.mark.parametrize( + "uid, expected", + [ + ( + "arn:aws:ec2:us-east-1:552455647653:instance/i-05075b63eb51baacb", + "i-05075b63eb51baacb", + ), + ( + "arn:aws:ec2:us-east-1:123456789012:volume/vol-0abcd1234ef567890", + "vol-0abcd1234ef567890", + ), + ( + "arn:aws:ec2:us-east-1:123456789012:security-group/sg-0123abcd", + "sg-0123abcd", + ), + ("arn:aws:s3:::my-bucket-name", "my-bucket-name"), + ("arn:aws:iam::123456789012:role/MyRole", "MyRole"), + ( + "arn:aws:lambda:us-east-1:123456789012:function:my-function", + "my-function", + ), + ("i-05075b63eb51baacb", "i-05075b63eb51baacb"), + ], + ) + def test_extract_short_uid_aws_variants(self, uid, expected): + from tasks.jobs.attack_paths.aws import extract_short_uid -class TestProviderConfigAccessors: - def test_get_deprecated_provider_resource_label_known_provider(self): - assert get_deprecated_provider_resource_label("aws") == "AWSResource" + assert extract_short_uid(uid) == expected - def test_get_deprecated_provider_resource_label_unknown_provider(self): - assert ( - get_deprecated_provider_resource_label("unknown") - == "UnknownProviderResource" + def test_insert_finding_template_has_short_id_fallback(self): + from tasks.jobs.attack_paths.queries import ( + INSERT_FINDING_TEMPLATE, + render_cypher_template, ) + rendered = render_cypher_template( + INSERT_FINDING_TEMPLATE, + { + "__NODE_UID_FIELD__": "arn", + "__RESOURCE_LABEL__": "_AWSResource", + }, + ) + + assert ( + "resource_by_uid:_AWSResource {arn: finding_data.resource_uid}" in rendered + ) + assert "resource_by_id:_AWSResource {id: finding_data.resource_uid}" in rendered + assert ( + "resource_by_short:_AWSResource {id: finding_data.resource_short_uid}" + in rendered + ) + assert "head(collect(resource_by_short)) AS resource_by_short" in rendered + assert ( + "COALESCE(resource_by_uid, resource_by_id, resource_by_short)" in rendered + ) + assert "RETURN merged_count, dropped_count" in rendered + class TestAddResourceLabel: - def test_add_resource_label_applies_both_labels(self): + def test_add_resource_label_applies_private_label(self): mock_session = MagicMock() first_result = MagicMock() @@ -1104,40 +1783,228 @@ class TestAddResourceLabel: assert mock_session.run.call_count == 2 query = mock_session.run.call_args_list[0].args[0] assert "_AWSResource" in query - assert "AWSResource" in query + assert "AWSResource" not in query.replace("_AWSResource", "") + + +def _make_session_ctx(session, call_order=None, name=None): + """Create a mock context manager wrapping a mock session.""" + ctx = MagicMock() + if call_order is not None and name is not None: + ctx.__enter__ = MagicMock( + side_effect=lambda: (call_order.append(f"{name}:enter"), session)[1] + ) + ctx.__exit__ = MagicMock( + side_effect=lambda *a: (call_order.append(f"{name}:exit"), False)[1] + ) + else: + ctx.__enter__ = MagicMock(return_value=session) + ctx.__exit__ = MagicMock(return_value=False) + return ctx class TestSyncNodes: - def test_sync_nodes_adds_both_labels(self): - mock_source_session = MagicMock() - mock_target_session = MagicMock() - + def test_sync_nodes_adds_private_label(self): row = { "internal_id": 1, "element_id": "elem-1", "labels": ["SomeLabel"], "props": {"key": "value"}, } - mock_source_session.run.side_effect = [[row], []] - source_ctx = MagicMock() - source_ctx.__enter__ = MagicMock(return_value=mock_source_session) - source_ctx.__exit__ = MagicMock(return_value=False) - - target_ctx = MagicMock() - target_ctx.__enter__ = MagicMock(return_value=mock_target_session) - target_ctx.__exit__ = MagicMock(return_value=False) + mock_source_1 = MagicMock() + mock_source_1.run.return_value = [row] + mock_target = MagicMock() + mock_source_2 = MagicMock() + mock_source_2.run.return_value = [] with patch( "tasks.jobs.attack_paths.sync.graph_database.get_session", - side_effect=[source_ctx, target_ctx], + side_effect=[ + _make_session_ctx(mock_source_1), + _make_session_ctx(mock_target), + _make_session_ctx(mock_source_2), + ], ): - total = sync_module.sync_nodes("source-db", "target-db", "prov-1") + total = sync_module.sync_nodes( + "source-db", "target-db", "tenant-1", "prov-1" + ) assert total == 1 - query = mock_target_session.run.call_args.args[0] + query = mock_target.run.call_args.args[0] assert "_ProviderResource" in query - assert "ProviderResource" in query + assert "_Tenant_tenant1" in query + assert "_Provider_prov1" in query + + def test_sync_nodes_source_closes_before_target_opens(self): + row = { + "internal_id": 1, + "element_id": "elem-1", + "labels": ["SomeLabel"], + "props": {"key": "value"}, + } + + call_order = [] + + src_1 = MagicMock() + src_1.run.return_value = [row] + tgt = MagicMock() + src_2 = MagicMock() + src_2.run.return_value = [] + + with patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[ + _make_session_ctx(src_1, call_order, "source1"), + _make_session_ctx(tgt, call_order, "target"), + _make_session_ctx(src_2, call_order, "source2"), + ], + ): + sync_module.sync_nodes("src-db", "tgt-db", "t-1", "p-1") + + assert call_order.index("source1:exit") < call_order.index("target:enter") + + def test_sync_nodes_pagination_with_batch_size_1(self): + row_a = { + "internal_id": 1, + "element_id": "elem-1", + "labels": ["LabelA"], + "props": {"a": 1}, + } + row_b = { + "internal_id": 2, + "element_id": "elem-2", + "labels": ["LabelB"], + "props": {"b": 2}, + } + + src_1 = MagicMock() + src_1.run.return_value = [row_a] + src_2 = MagicMock() + src_2.run.return_value = [row_b] + src_3 = MagicMock() + src_3.run.return_value = [] + tgt_1 = MagicMock() + tgt_2 = MagicMock() + + with ( + patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[ + _make_session_ctx(src_1), + _make_session_ctx(tgt_1), + _make_session_ctx(src_2), + _make_session_ctx(tgt_2), + _make_session_ctx(src_3), + ], + ), + patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 1), + ): + total = sync_module.sync_nodes("src", "tgt", "t-1", "p-1") + + assert total == 2 + assert src_1.run.call_args.args[1]["last_id"] == -1 + assert src_2.run.call_args.args[1]["last_id"] == 1 + + def test_sync_nodes_empty_source_returns_zero(self): + src = MagicMock() + src.run.return_value = [] + + with patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[_make_session_ctx(src)], + ) as mock_get_session: + total = sync_module.sync_nodes("src", "tgt", "t-1", "p-1") + + assert total == 0 + assert mock_get_session.call_count == 1 + + +class TestSyncRelationships: + def test_sync_relationships_source_closes_before_target_opens(self): + row = { + "internal_id": 1, + "rel_type": "HAS", + "start_element_id": "s-1", + "end_element_id": "e-1", + "props": {}, + } + + call_order = [] + + src_1 = MagicMock() + src_1.run.return_value = [row] + tgt = MagicMock() + src_2 = MagicMock() + src_2.run.return_value = [] + + with patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[ + _make_session_ctx(src_1, call_order, "source1"), + _make_session_ctx(tgt, call_order, "target"), + _make_session_ctx(src_2, call_order, "source2"), + ], + ): + sync_module.sync_relationships("src", "tgt", "p-1") + + assert call_order.index("source1:exit") < call_order.index("target:enter") + + def test_sync_relationships_pagination_with_batch_size_1(self): + row_a = { + "internal_id": 1, + "rel_type": "HAS", + "start_element_id": "s-1", + "end_element_id": "e-1", + "props": {"a": 1}, + } + row_b = { + "internal_id": 2, + "rel_type": "CONNECTS", + "start_element_id": "s-2", + "end_element_id": "e-2", + "props": {"b": 2}, + } + + src_1 = MagicMock() + src_1.run.return_value = [row_a] + src_2 = MagicMock() + src_2.run.return_value = [row_b] + src_3 = MagicMock() + src_3.run.return_value = [] + tgt_1 = MagicMock() + tgt_2 = MagicMock() + + with ( + patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[ + _make_session_ctx(src_1), + _make_session_ctx(tgt_1), + _make_session_ctx(src_2), + _make_session_ctx(tgt_2), + _make_session_ctx(src_3), + ], + ), + patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 1), + ): + total = sync_module.sync_relationships("src", "tgt", "p-1") + + assert total == 2 + assert src_1.run.call_args.args[1]["last_id"] == -1 + assert src_2.run.call_args.args[1]["last_id"] == 1 + + def test_sync_relationships_empty_source_returns_zero(self): + src = MagicMock() + src.run.return_value = [] + + with patch( + "tasks.jobs.attack_paths.sync.graph_database.get_session", + side_effect=[_make_session_ctx(src)], + ) as mock_get_session: + total = sync_module.sync_relationships("src", "tgt", "p-1") + + assert total == 0 + assert mock_get_session.call_count == 1 class TestInternetAnalysis: @@ -1494,3 +2361,514 @@ class TestAttackPathsDbUtilsGraphDataReady: ap_scan_b.refresh_from_db() assert ap_scan_a.graph_data_ready is False assert ap_scan_b.graph_data_ready is True + + +@pytest.mark.django_db +class TestCleanupStaleAttackPathsScans: + def _create_executing_scan( + self, tenant, provider, scan=None, started_at=None, worker=None + ): + """Helper to create an EXECUTING AttackPathsScan with optional Task+TaskResult.""" + ap_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=scan, + state=StateChoices.EXECUTING, + started_at=started_at or datetime.now(tz=timezone.utc), + ) + + task_result = None + if worker is not None: + task_result = TaskResult.objects.create( + task_id=str(ap_scan.id), + task_name="attack-paths-scan-perform", + status="STARTED", + worker=worker, + ) + task = Task.objects.create( + id=task_result.task_id, + task_runner_task=task_result, + tenant_id=tenant.id, + ) + ap_scan.task = task + ap_scan.save(update_fields=["task_id"]) + + return ap_scan, task_result + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._is_worker_alive", return_value=False) + def test_cleans_up_scan_with_dead_worker( + self, + mock_alive, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + # Recent scan — should still be cleaned up because worker is dead + ap_scan, task_result = self._create_executing_scan( + tenant, provider, worker="dead-worker@host" + ) + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 1 + assert str(ap_scan.id) in result["scan_ids"] + mock_drop_db.assert_called_once() + mock_recover.assert_called_once() + + ap_scan.refresh_from_db() + assert ap_scan.state == StateChoices.FAILED + assert ap_scan.progress == 100 + assert ap_scan.completed_at is not None + assert ap_scan.ingestion_exceptions == { + "global_error": "Worker dead — cleaned up by periodic task" + } + + task_result.refresh_from_db() + assert task_result.status == "FAILURE" + assert task_result.date_done is not None + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._revoke_task") + @patch("tasks.jobs.attack_paths.cleanup._is_worker_alive", return_value=True) + def test_revokes_and_cleans_scan_exceeding_threshold_on_live_worker( + self, + mock_alive, + mock_revoke, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + old_start = datetime.now(tz=timezone.utc) - timedelta(hours=49) + ap_scan, task_result = self._create_executing_scan( + tenant, provider, started_at=old_start, worker="live-worker@host" + ) + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 1 + mock_revoke.assert_called_once_with(task_result) + mock_recover.assert_called_once() + + ap_scan.refresh_from_db() + assert ap_scan.state == StateChoices.FAILED + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._is_worker_alive", return_value=True) + def test_ignores_recent_executing_scans_on_live_worker( + self, + mock_alive, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + # Recent scan on live worker — should be skipped + self._create_executing_scan(tenant, provider, worker="live-worker@host") + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 0 + mock_drop_db.assert_not_called() + mock_recover.assert_not_called() + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + def test_ignores_completed_and_failed_scans( + self, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + state=StateChoices.COMPLETED, + ) + AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + state=StateChoices.FAILED, + ) + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 0 + mock_drop_db.assert_not_called() + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch( + "tasks.jobs.attack_paths.cleanup.graph_database.drop_database", + side_effect=Exception("Neo4j unreachable"), + ) + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._is_worker_alive", return_value=False) + def test_handles_drop_database_failure_gracefully( + self, + mock_alive, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + self._create_executing_scan(tenant, provider, worker="dead-worker@host") + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 1 + mock_drop_db.assert_called_once() + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._is_worker_alive", return_value=False) + def test_cross_tenant_cleanup( + self, + mock_alive, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant1 = tenants_fixture[0] + tenant2 = tenants_fixture[1] + provider1 = providers_fixture[0] + provider1.provider = Provider.ProviderChoices.AWS + provider1.save() + + provider2 = Provider.objects.create( + provider="aws", + uid="999888777666", + alias="aws_tenant2", + tenant_id=tenant2.id, + ) + + ap_scan1, _ = self._create_executing_scan( + tenant1, provider1, worker="dead-worker-1@host" + ) + ap_scan2, _ = self._create_executing_scan( + tenant2, provider2, worker="dead-worker-2@host" + ) + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 2 + assert mock_recover.call_count == 2 + + ap_scan1.refresh_from_db() + ap_scan2.refresh_from_db() + assert ap_scan1.state == StateChoices.FAILED + assert ap_scan2.state == StateChoices.FAILED + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._is_worker_alive", return_value=False) + def test_recovers_graph_data_ready_for_stale_scan( + self, + mock_alive, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + ap_scan, _ = self._create_executing_scan( + tenant, provider, worker="dead-worker@host" + ) + + cleanup_stale_attack_paths_scans() + + mock_recover.assert_called_once() + recovered_scan = mock_recover.call_args[0][0] + assert recovered_scan.id == ap_scan.id + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + def test_fallback_to_time_heuristic_when_no_worker_field( + self, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + # Old scan with no Task/TaskResult + old_start = datetime.now(tz=timezone.utc) - timedelta(hours=49) + ap_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + state=StateChoices.EXECUTING, + started_at=old_start, + ) + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 1 + + ap_scan.refresh_from_db() + assert ap_scan.state == StateChoices.FAILED + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._is_worker_alive", return_value=False) + def test_shared_worker_is_pinged_only_once( + self, + mock_alive, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + # Two scans on the same dead worker + self._create_executing_scan(tenant, provider, worker="shared-worker@host") + self._create_executing_scan(tenant, provider, worker="shared-worker@host") + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 2 + # Worker should be pinged exactly once — cache prevents second ping + mock_alive.assert_called_once_with("shared-worker@host") + + # `SCHEDULED` state cleanup + def _create_scheduled_scan( + self, + tenant, + provider, + *, + age_minutes, + parent_state, + with_task=True, + ): + """Create a SCHEDULED AttackPathsScan with a parent Scan in `parent_state`. + + `age_minutes` controls how far in the past `started_at` is set, so + callers can place rows safely past the cleanup cutoff. + """ + parent_scan = Scan.objects.create( + name="Parent Prowler scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=parent_state, + tenant_id=tenant.id, + ) + + ap_scan = AttackPathsScan.objects.create( + tenant_id=tenant.id, + provider=provider, + scan=parent_scan, + state=StateChoices.SCHEDULED, + started_at=datetime.now(tz=timezone.utc) - timedelta(minutes=age_minutes), + ) + + task_result = None + if with_task: + task_result = TaskResult.objects.create( + task_id=str(ap_scan.id), + task_name="attack-paths-scan-perform", + status="PENDING", + ) + task = Task.objects.create( + id=task_result.task_id, + task_runner_task=task_result, + tenant_id=tenant.id, + ) + ap_scan.task = task + ap_scan.save(update_fields=["task_id"]) + + return ap_scan, task_result + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._revoke_task") + def test_cleans_up_scheduled_scan_when_parent_is_terminal( + self, + mock_revoke, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + ap_scan, task_result = self._create_scheduled_scan( + tenant, + provider, + age_minutes=24 * 60 * 3, # 3 days, safely past any threshold + parent_state=StateChoices.FAILED, + ) + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 1 + assert str(ap_scan.id) in result["scan_ids"] + + ap_scan.refresh_from_db() + assert ap_scan.state == StateChoices.FAILED + assert ap_scan.progress == 100 + assert ap_scan.completed_at is not None + assert ap_scan.ingestion_exceptions == { + "global_error": "Scan never started — cleaned up by periodic task" + } + + # SCHEDULED revoke must NOT terminate a running worker + mock_revoke.assert_called_once() + assert mock_revoke.call_args.kwargs == {"terminate": False} + + # Temp DB never created for SCHEDULED, so no drop attempted + mock_drop_db.assert_not_called() + # Tenant Neo4j data is untouched in this path + mock_recover.assert_not_called() + + task_result.refresh_from_db() + assert task_result.status == "FAILURE" + assert task_result.date_done is not None + + @patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready") + @patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database") + @patch( + "tasks.jobs.attack_paths.cleanup.rls_transaction", + new=lambda *args, **kwargs: nullcontext(), + ) + @patch("tasks.jobs.attack_paths.cleanup._revoke_task") + def test_skips_scheduled_scan_when_parent_still_in_flight( + self, + mock_revoke, + mock_drop_db, + mock_recover, + tenants_fixture, + providers_fixture, + ): + from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans + + tenant = tenants_fixture[0] + provider = providers_fixture[0] + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + ap_scan, _ = self._create_scheduled_scan( + tenant, + provider, + age_minutes=24 * 60 * 3, + parent_state=StateChoices.EXECUTING, + ) + + result = cleanup_stale_attack_paths_scans() + + assert result["cleaned_up_count"] == 0 + + ap_scan.refresh_from_db() + assert ap_scan.state == StateChoices.SCHEDULED + mock_revoke.assert_not_called() diff --git a/api/src/backend/tasks/tests/test_backfill.py b/api/src/backend/tasks/tests/test_backfill.py index 469b0a393b..3ab49a15e6 100644 --- a/api/src/backend/tasks/tests/test_backfill.py +++ b/api/src/backend/tasks/tests/test_backfill.py @@ -7,8 +7,8 @@ from tasks.jobs.backfill import ( backfill_compliance_summaries, backfill_provider_compliance_scores, backfill_resource_scan_summaries, - backfill_scan_category_summaries, - backfill_scan_resource_group_summaries, + aggregate_scan_category_summaries, + aggregate_scan_resource_group_summaries, ) from api.models import ( @@ -183,6 +183,10 @@ class TestBackfillComplianceSummaries: def test_backfill_creates_compliance_summaries( self, tenants_fixture, scans_fixture, compliance_requirements_overviews_fixture ): + # Fixture seeds compliance rows the backfill aggregates over; pytest + # injects it by parameter name, so we reference it explicitly here + # to keep static analysers from flagging it as unused. + del compliance_requirements_overviews_fixture tenant = tenants_fixture[0] scan = scans_fixture[0] @@ -227,22 +231,86 @@ class TestBackfillComplianceSummaries: @pytest.mark.django_db class TestBackfillScanCategorySummaries: - def test_already_backfilled(self, scan_category_summary_fixture): + def test_rerun_with_no_findings_is_noop(self, scan_category_summary_fixture): + """When the scan has no findings, the backfill is a no-op: it + reports `no categories to backfill` and leaves the table + untouched. The upsert path cannot drop rows it does not produce, + so any pre-existing row survives (matching the scan-completion + writer that used `ignore_conflicts=True`).""" tenant_id = scan_category_summary_fixture.tenant_id scan_id = scan_category_summary_fixture.scan_id - result = backfill_scan_category_summaries(str(tenant_id), str(scan_id)) + result = aggregate_scan_category_summaries(str(tenant_id), str(scan_id)) - assert result == {"status": "already backfilled"} + assert result == {"status": "no categories to backfill"} + assert ScanCategorySummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id, category="existing-category" + ).exists() + + def test_rerun_upserts_without_duplicating(self, findings_with_categories_fixture): + """Calling the backfill twice upserts rather than raising on + `unique_category_severity_per_scan`; rows are updated in place + (same primary keys).""" + finding = findings_with_categories_fixture + tenant_id = str(finding.tenant_id) + scan_id = str(finding.scan_id) + + aggregate_scan_category_summaries(tenant_id, scan_id) + first_ids = set( + ScanCategorySummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).values_list("id", flat=True) + ) + + aggregate_scan_category_summaries(tenant_id, scan_id) + second_ids = set( + ScanCategorySummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).values_list("id", flat=True) + ) + + assert first_ids == second_ids + assert len(first_ids) == 2 # 2 categories x 1 severity + + def test_rerun_reflects_mute_between_runs(self, findings_with_categories_fixture): + """Muting a finding between two backfill runs must move counters: + `failed_findings` and `new_failed_findings` drop to zero (muted + findings are excluded from those totals). Guards against a + regression where the upsert keeps stale counts from the first run.""" + finding = findings_with_categories_fixture + tenant_id = str(finding.tenant_id) + scan_id = str(finding.scan_id) + + aggregate_scan_category_summaries(tenant_id, scan_id) + before = list( + ScanCategorySummary.objects.filter(tenant_id=tenant_id, scan_id=scan_id) + ) + assert all(s.failed_findings == 1 for s in before) + assert all(s.new_failed_findings == 1 for s in before) + assert all(s.total_findings == 1 for s in before) + + Finding.all_objects.filter(pk=finding.pk).update(muted=True) + + aggregate_scan_category_summaries(tenant_id, scan_id) + after = list( + ScanCategorySummary.objects.filter(tenant_id=tenant_id, scan_id=scan_id) + ) + + assert {s.id for s in after} == {s.id for s in before} + assert all(s.failed_findings == 0 for s in after) + assert all(s.new_failed_findings == 0 for s in after) + assert all(s.total_findings == 0 for s in after) def test_not_completed_scan(self, get_not_completed_scans): for scan in get_not_completed_scans: - result = backfill_scan_category_summaries(str(scan.tenant_id), str(scan.id)) + result = aggregate_scan_category_summaries( + str(scan.tenant_id), str(scan.id) + ) assert result == {"status": "scan is not completed"} def test_no_categories_to_backfill(self, scans_fixture): scan = scans_fixture[1] # Failed scan with no findings - result = backfill_scan_category_summaries(str(scan.tenant_id), str(scan.id)) + result = aggregate_scan_category_summaries(str(scan.tenant_id), str(scan.id)) assert result == {"status": "no categories to backfill"} def test_successful_backfill(self, findings_with_categories_fixture): @@ -250,7 +318,7 @@ class TestBackfillScanCategorySummaries: tenant_id = str(finding.tenant_id) scan_id = str(finding.scan_id) - result = backfill_scan_category_summaries(tenant_id, scan_id) + result = aggregate_scan_category_summaries(tenant_id, scan_id) # 2 categories × 1 severity = 2 rows assert result == {"status": "backfilled", "categories_count": 2} @@ -311,24 +379,87 @@ def scan_resource_group_summary_fixture(scans_fixture): @pytest.mark.django_db class TestBackfillScanGroupSummaries: - def test_already_backfilled(self, scan_resource_group_summary_fixture): + def test_rerun_with_no_findings_is_noop(self, scan_resource_group_summary_fixture): + """When the scan has no findings, the backfill is a no-op: it + reports `no resource groups to backfill` and leaves the table + untouched. The upsert path cannot drop rows it does not produce, + so any pre-existing row survives (matching the scan-completion + writer that used `ignore_conflicts=True`).""" tenant_id = scan_resource_group_summary_fixture.tenant_id scan_id = scan_resource_group_summary_fixture.scan_id - result = backfill_scan_resource_group_summaries(str(tenant_id), str(scan_id)) + result = aggregate_scan_resource_group_summaries(str(tenant_id), str(scan_id)) - assert result == {"status": "already backfilled"} + assert result == {"status": "no resource groups to backfill"} + assert ScanGroupSummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id, resource_group="existing-group" + ).exists() + + def test_rerun_upserts_without_duplicating(self, findings_with_group_fixture): + """Calling the backfill twice upserts rather than raising on + `unique_resource_group_severity_per_scan`; rows are updated in + place (same primary keys).""" + finding = findings_with_group_fixture + tenant_id = str(finding.tenant_id) + scan_id = str(finding.scan_id) + + aggregate_scan_resource_group_summaries(tenant_id, scan_id) + first_ids = set( + ScanGroupSummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).values_list("id", flat=True) + ) + + aggregate_scan_resource_group_summaries(tenant_id, scan_id) + second_ids = set( + ScanGroupSummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).values_list("id", flat=True) + ) + + assert first_ids == second_ids + assert len(first_ids) == 1 # 1 resource group x 1 severity + + def test_rerun_reflects_mute_between_runs(self, findings_with_group_fixture): + """Muting a finding between two backfill runs must move counters: + `failed_findings` and `new_failed_findings` drop to zero (muted + findings are excluded from those totals). Guards against a + regression where the upsert keeps stale counts from the first run.""" + finding = findings_with_group_fixture + tenant_id = str(finding.tenant_id) + scan_id = str(finding.scan_id) + + aggregate_scan_resource_group_summaries(tenant_id, scan_id) + before = list( + ScanGroupSummary.objects.filter(tenant_id=tenant_id, scan_id=scan_id) + ) + assert len(before) == 1 + assert before[0].failed_findings == 1 + assert before[0].new_failed_findings == 1 + assert before[0].total_findings == 1 + + Finding.all_objects.filter(pk=finding.pk).update(muted=True) + + aggregate_scan_resource_group_summaries(tenant_id, scan_id) + after = list( + ScanGroupSummary.objects.filter(tenant_id=tenant_id, scan_id=scan_id) + ) + + assert {s.id for s in after} == {s.id for s in before} + assert after[0].failed_findings == 0 + assert after[0].new_failed_findings == 0 + assert after[0].total_findings == 0 def test_not_completed_scan(self, get_not_completed_scans): for scan in get_not_completed_scans: - result = backfill_scan_resource_group_summaries( + result = aggregate_scan_resource_group_summaries( str(scan.tenant_id), str(scan.id) ) assert result == {"status": "scan is not completed"} def test_no_resource_groups_to_backfill(self, scans_fixture): scan = scans_fixture[1] # Failed scan with no findings - result = backfill_scan_resource_group_summaries( + result = aggregate_scan_resource_group_summaries( str(scan.tenant_id), str(scan.id) ) assert result == {"status": "no resource groups to backfill"} @@ -338,7 +469,7 @@ class TestBackfillScanGroupSummaries: tenant_id = str(finding.tenant_id) scan_id = str(finding.scan_id) - result = backfill_scan_resource_group_summaries(tenant_id, scan_id) + result = aggregate_scan_resource_group_summaries(tenant_id, scan_id) # 1 resource group × 1 severity = 1 row assert result == {"status": "backfilled", "resource_groups_count": 1} diff --git a/api/src/backend/tasks/tests/test_reports.py b/api/src/backend/tasks/tests/test_reports.py index a6a80b0891..b5cb0cc38e 100644 --- a/api/src/backend/tasks/tests/test_reports.py +++ b/api/src/backend/tasks/tests/test_reports.py @@ -1,10 +1,21 @@ +import os +import time import uuid from unittest.mock import Mock, patch import matplotlib import pytest from reportlab.lib import colors -from tasks.jobs.report import generate_compliance_reports, generate_threatscore_report +from tasks.jobs.report import ( + STALE_TMP_OUTPUT_MAX_AGE_HOURS, + STALE_TMP_OUTPUT_LOCK_FILE_NAME, + _cleanup_stale_tmp_output_directories, + _is_scan_directory_protected, + _pick_latest_cis_variant, + _should_run_stale_cleanup, + generate_compliance_reports, + generate_threatscore_report, +) from tasks.jobs.reports import ( CHART_COLOR_GREEN_1, CHART_COLOR_GREEN_2, @@ -29,7 +40,13 @@ from tasks.jobs.threatscore_utils import ( _load_findings_for_requirement_checks, ) -from api.models import Finding, Resource, ResourceFindingMapping, StatusChoices +from api.models import ( + Finding, + Resource, + ResourceFindingMapping, + StateChoices, + StatusChoices, +) from prowler.lib.check.models import Severity matplotlib.use("Agg") # Use non-interactive backend for tests @@ -169,35 +186,27 @@ class TestAggregateRequirementStatistics: assert result["check_1"]["passed"] == 1 assert result["check_1"]["total"] == 1 - def test_excludes_findings_without_resources(self, tenants_fixture, scans_fixture): - """Verify findings without resources are excluded from aggregation.""" + def test_skips_aggregation_for_deleted_provider( + self, tenants_fixture, scans_fixture + ): + """Verify aggregation returns empty when the scan's provider is soft-deleted.""" tenant = tenants_fixture[0] scan = scans_fixture[0] - # Finding WITH resource → should be counted self._create_finding_with_resource( tenant, scan, "finding-1", "check_1", StatusChoices.PASS ) - # Finding WITHOUT resource → should be EXCLUDED - Finding.objects.create( - tenant_id=tenant.id, - scan=scan, - uid="finding-2", - check_id="check_1", - status=StatusChoices.FAIL, - severity=Severity.high, - impact=Severity.high, - check_metadata={}, - raw_result={}, - ) + # Soft-delete the provider + provider = scan.provider + provider.is_deleted = True + provider.save(update_fields=["is_deleted"]) result = _aggregate_requirement_statistics_from_database( str(tenant.id), str(scan.id) ) - assert result["check_1"]["passed"] == 1 - assert result["check_1"]["total"] == 1 + assert result == {} def test_multiple_resources_no_double_count(self, tenants_fixture, scans_fixture): """Verify a finding with multiple resources is only counted once.""" @@ -359,6 +368,366 @@ class TestLoadFindingsForChecks: assert result == {} +class TestCleanupStaleTmpOutputDirectories: + """Unit tests for opportunistic stale cleanup under tmp output root.""" + + def test_removes_only_scan_dirs_older_than_ttl(self, tmp_path, monkeypatch): + """Should remove stale scan directories and keep recent ones.""" + root_dir = tmp_path / "prowler_api_output" + + old_scan_dir = root_dir / "tenant-a" / "scan-old" + old_scan_dir.mkdir(parents=True) + (old_scan_dir / "artifact.txt").write_text("old") + + recent_scan_dir = root_dir / "tenant-a" / "scan-recent" + recent_scan_dir.mkdir(parents=True) + (recent_scan_dir / "artifact.txt").write_text("recent") + + now = time.time() + stale_ts = now - ((STALE_TMP_OUTPUT_MAX_AGE_HOURS + 1) * 60 * 60) + os.utime(old_scan_dir, (stale_ts, stale_ts)) + + monkeypatch.setattr( + "tasks.jobs.report.STALE_TMP_OUTPUT_SAFE_ROOT", root_dir.resolve() + ) + monkeypatch.setattr( + "tasks.jobs.report._should_run_stale_cleanup", lambda *_: True + ) + monkeypatch.setattr( + "tasks.jobs.report._is_scan_directory_protected", lambda **_: False + ) + + removed = _cleanup_stale_tmp_output_directories( + str(root_dir), max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS + ) + + assert removed == 1 + assert not old_scan_dir.exists() + assert recent_scan_dir.exists() + + def test_skips_current_scan_even_when_stale(self, tmp_path, monkeypatch): + """Should not delete stale directory for the currently processed scan.""" + root_dir = tmp_path / "prowler_api_output" + + current_scan_dir = root_dir / "tenant-current" / "scan-current" + current_scan_dir.mkdir(parents=True) + (current_scan_dir / "artifact.txt").write_text("current") + + other_stale_scan_dir = root_dir / "tenant-other" / "scan-old" + other_stale_scan_dir.mkdir(parents=True) + (other_stale_scan_dir / "artifact.txt").write_text("other") + + now = time.time() + stale_ts = now - ((STALE_TMP_OUTPUT_MAX_AGE_HOURS + 1) * 60 * 60) + os.utime(current_scan_dir, (stale_ts, stale_ts)) + os.utime(other_stale_scan_dir, (stale_ts, stale_ts)) + + monkeypatch.setattr( + "tasks.jobs.report.STALE_TMP_OUTPUT_SAFE_ROOT", root_dir.resolve() + ) + monkeypatch.setattr( + "tasks.jobs.report._should_run_stale_cleanup", lambda *_: True + ) + monkeypatch.setattr( + "tasks.jobs.report._is_scan_directory_protected", lambda **_: False + ) + + removed = _cleanup_stale_tmp_output_directories( + str(root_dir), + max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS, + exclude_scan=("tenant-current", "scan-current"), + ) + + assert removed == 1 + assert current_scan_dir.exists() + assert not other_stale_scan_dir.exists() + + def test_respects_max_deletions_per_run(self, tmp_path, monkeypatch): + """Cleanup should stop deleting when max_deletions_per_run is reached.""" + root_dir = tmp_path / "prowler_api_output" + + stale_dir_1 = root_dir / "tenant-a" / "scan-old-1" + stale_dir_2 = root_dir / "tenant-a" / "scan-old-2" + stale_dir_1.mkdir(parents=True) + stale_dir_2.mkdir(parents=True) + (stale_dir_1 / "artifact.txt").write_text("old-1") + (stale_dir_2 / "artifact.txt").write_text("old-2") + + now = time.time() + stale_ts = now - ((STALE_TMP_OUTPUT_MAX_AGE_HOURS + 1) * 60 * 60) + os.utime(stale_dir_1, (stale_ts, stale_ts)) + os.utime(stale_dir_2, (stale_ts, stale_ts)) + + monkeypatch.setattr( + "tasks.jobs.report.STALE_TMP_OUTPUT_SAFE_ROOT", root_dir.resolve() + ) + monkeypatch.setattr( + "tasks.jobs.report._should_run_stale_cleanup", lambda *_: True + ) + monkeypatch.setattr( + "tasks.jobs.report._is_scan_directory_protected", lambda **_: False + ) + + removed = _cleanup_stale_tmp_output_directories( + str(root_dir), + max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS, + max_deletions_per_run=1, + ) + + assert removed == 1 + remaining = sum( + 1 for scan_dir in (stale_dir_1, stale_dir_2) if scan_dir.exists() + ) + assert remaining == 1 + + def test_rejects_non_safe_root(self, tmp_path, monkeypatch): + """Cleanup must no-op when called with a root outside the allowed safe root.""" + root_dir = tmp_path / "prowler_api_output" + root_dir.mkdir(parents=True) + + monkeypatch.setattr( + "tasks.jobs.report.STALE_TMP_OUTPUT_SAFE_ROOT", + (tmp_path / "another-root").resolve(), + ) + + def _fail_should_run(*_args, **_kwargs): + raise AssertionError("_should_run_stale_cleanup should not be called") + + monkeypatch.setattr( + "tasks.jobs.report._should_run_stale_cleanup", _fail_should_run + ) + + removed = _cleanup_stale_tmp_output_directories(str(root_dir), max_age_hours=48) + + assert removed == 0 + + def test_ignores_symlink_scan_directories(self, tmp_path, monkeypatch): + """Symlinked scan directories must never be deleted by cleanup.""" + root_dir = tmp_path / "prowler_api_output" + stale_real_scan_dir = root_dir / "tenant-a" / "scan-old-real" + stale_real_scan_dir.mkdir(parents=True) + (stale_real_scan_dir / "artifact.txt").write_text("old") + + symlink_target = tmp_path / "symlink-target" + symlink_target.mkdir(parents=True) + (symlink_target / "artifact.txt").write_text("target") + symlink_scan_dir = root_dir / "tenant-a" / "scan-link" + symlink_scan_dir.symlink_to(symlink_target, target_is_directory=True) + + now = time.time() + stale_ts = now - ((STALE_TMP_OUTPUT_MAX_AGE_HOURS + 1) * 60 * 60) + os.utime(stale_real_scan_dir, (stale_ts, stale_ts)) + + monkeypatch.setattr( + "tasks.jobs.report.STALE_TMP_OUTPUT_SAFE_ROOT", root_dir.resolve() + ) + monkeypatch.setattr( + "tasks.jobs.report._should_run_stale_cleanup", lambda *_: True + ) + monkeypatch.setattr( + "tasks.jobs.report._is_scan_directory_protected", lambda **_: False + ) + + removed = _cleanup_stale_tmp_output_directories( + str(root_dir), max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS + ) + + assert removed == 1 + assert not stale_real_scan_dir.exists() + assert symlink_scan_dir.exists() + assert symlink_target.exists() + + def test_handles_internal_exception_without_propagating( + self, tmp_path, monkeypatch + ): + """Cleanup errors must be swallowed so callers are not interrupted.""" + root_dir = tmp_path / "prowler_api_output" + stale_scan_dir = root_dir / "tenant-a" / "scan-old" + stale_scan_dir.mkdir(parents=True) + + now = time.time() + stale_ts = now - ((STALE_TMP_OUTPUT_MAX_AGE_HOURS + 1) * 60 * 60) + os.utime(stale_scan_dir, (stale_ts, stale_ts)) + + monkeypatch.setattr( + "tasks.jobs.report.STALE_TMP_OUTPUT_SAFE_ROOT", root_dir.resolve() + ) + monkeypatch.setattr( + "tasks.jobs.report._should_run_stale_cleanup", lambda *_: True + ) + + def _raise(*_args, **_kwargs): + raise RuntimeError("db timeout") + + monkeypatch.setattr("tasks.jobs.report._is_scan_directory_protected", _raise) + + removed = _cleanup_stale_tmp_output_directories( + str(root_dir), max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS + ) + + assert removed == 0 + assert stale_scan_dir.exists() + + def test_safe_root_follows_custom_tmp_output_directory(self, tmp_path, monkeypatch): + """Custom DJANGO_TMP_OUTPUT_DIRECTORY must be honored as the safe root.""" + from tasks.jobs import report as report_module + + custom_root = tmp_path / "custom_tmp_output" + custom_root.mkdir(parents=True) + + monkeypatch.setattr( + report_module, "DJANGO_TMP_OUTPUT_DIRECTORY", str(custom_root) + ) + + resolved_root = report_module._resolve_stale_tmp_safe_root() + assert resolved_root == custom_root.resolve() + + stale_scan_dir = custom_root / "tenant-a" / "scan-old" + stale_scan_dir.mkdir(parents=True) + (stale_scan_dir / "artifact.txt").write_text("old") + + stale_ts = time.time() - ((STALE_TMP_OUTPUT_MAX_AGE_HOURS + 1) * 60 * 60) + os.utime(stale_scan_dir, (stale_ts, stale_ts)) + + monkeypatch.setattr(report_module, "STALE_TMP_OUTPUT_SAFE_ROOT", resolved_root) + monkeypatch.setattr( + "tasks.jobs.report._should_run_stale_cleanup", lambda *_: True + ) + monkeypatch.setattr( + "tasks.jobs.report._is_scan_directory_protected", lambda **_: False + ) + + removed = _cleanup_stale_tmp_output_directories( + str(custom_root), max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS + ) + + assert removed == 1 + assert not stale_scan_dir.exists() + + @pytest.mark.parametrize( + "forbidden_root", + ["/", "/tmp", "/var", "/var/tmp", "/home", "/root", "/etc", "/usr"], + ) + def test_safe_root_rejects_forbidden_system_roots( + self, forbidden_root, monkeypatch + ): + """Cleanup must refuse to operate against shared system roots.""" + from tasks.jobs import report as report_module + + monkeypatch.setattr( + report_module, "DJANGO_TMP_OUTPUT_DIRECTORY", forbidden_root + ) + + assert report_module._resolve_stale_tmp_safe_root() is None + + def test_skips_cleanup_when_safe_root_is_none(self, tmp_path, monkeypatch): + """A None safe root (forbidden config) must short-circuit the cleanup.""" + root_dir = tmp_path / "prowler_api_output" + root_dir.mkdir(parents=True) + + monkeypatch.setattr("tasks.jobs.report.STALE_TMP_OUTPUT_SAFE_ROOT", None) + + def _fail_should_run(*_args, **_kwargs): + raise AssertionError("_should_run_stale_cleanup should not be called") + + monkeypatch.setattr( + "tasks.jobs.report._should_run_stale_cleanup", _fail_should_run + ) + + removed = _cleanup_stale_tmp_output_directories( + str(root_dir), max_age_hours=STALE_TMP_OUTPUT_MAX_AGE_HOURS + ) + + assert removed == 0 + + +class TestStaleCleanupProtectionHelpers: + """Unit tests for stale cleanup helper guard logic.""" + + def test_should_run_cleanup_is_throttled(self, tmp_path): + root_dir = tmp_path / "prowler_api_output" + root_dir.mkdir(parents=True) + + assert _should_run_stale_cleanup(root_dir, throttle_seconds=3600) is True + assert _should_run_stale_cleanup(root_dir, throttle_seconds=3600) is False + + lock_file = root_dir / STALE_TMP_OUTPUT_LOCK_FILE_NAME + lock_file.write_text(str(int(time.time()) - 7200), encoding="ascii") + + assert _should_run_stale_cleanup(root_dir, throttle_seconds=3600) is True + + @patch("tasks.jobs.report.fcntl.flock", side_effect=BlockingIOError) + def test_should_run_cleanup_returns_false_when_lock_is_busy( + self, _mock_flock, tmp_path + ): + root_dir = tmp_path / "prowler_api_output" + root_dir.mkdir(parents=True) + + assert _should_run_stale_cleanup(root_dir, throttle_seconds=3600) is False + + @patch("tasks.jobs.report.Scan.all_objects.using") + def test_is_scan_directory_protected_for_executing_scan( + self, mock_scan_using, tmp_path + ): + scan_id = str(uuid.uuid4()) + scan_path = tmp_path / scan_id + scan_path.mkdir(parents=True) + mock_scan_using.return_value.filter.return_value.only.return_value.first.return_value = Mock( + state=StateChoices.EXECUTING, output_location=None + ) + + assert ( + _is_scan_directory_protected( + tenant_id="tenant-a", + scan_id=scan_id, + scan_path=scan_path, + ) + is True + ) + + @patch("tasks.jobs.report.Scan.all_objects.using") + def test_is_scan_directory_protected_for_local_output( + self, mock_scan_using, tmp_path + ): + scan_id = str(uuid.uuid4()) + scan_path = tmp_path / scan_id + scan_path.mkdir(parents=True) + local_output_path = scan_path / "outputs.zip" + mock_scan_using.return_value.filter.return_value.only.return_value.first.return_value = Mock( + state=StateChoices.COMPLETED, output_location=str(local_output_path) + ) + + assert ( + _is_scan_directory_protected( + tenant_id="tenant-a", + scan_id=scan_id, + scan_path=scan_path.resolve(), + ) + is True + ) + + @patch("tasks.jobs.report.Scan.all_objects.using") + def test_is_scan_directory_not_protected_for_s3_output( + self, mock_scan_using, tmp_path + ): + scan_id = str(uuid.uuid4()) + scan_path = tmp_path / scan_id + scan_path.mkdir(parents=True) + mock_scan_using.return_value.filter.return_value.only.return_value.first.return_value = Mock( + state=StateChoices.COMPLETED, + output_location="s3://bucket/path/report.zip", + ) + + assert ( + _is_scan_directory_protected( + tenant_id="tenant-a", + scan_id=scan_id, + scan_path=scan_path, + ) + is False + ) + + @pytest.mark.django_db class TestGenerateThreatscoreReportFunction: """Test suite for generate_threatscore_report function.""" @@ -430,6 +799,425 @@ class TestGenerateComplianceReportsOptimized: mock_ens.assert_not_called() mock_nis2.assert_not_called() + @patch( + "tasks.jobs.report._cleanup_stale_tmp_output_directories", + side_effect=RuntimeError("cleanup boom"), + ) + def test_cleanup_exception_does_not_break_no_findings_flow(self, _mock_cleanup): + """Unexpected cleanup failures must not abort report generation.""" + random_tenant = str(uuid.uuid4()) + random_scan = str(uuid.uuid4()) + random_provider = str(uuid.uuid4()) + + with patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter: + mock_filter.return_value.exists.return_value = False + result = generate_compliance_reports( + tenant_id=random_tenant, + scan_id=random_scan, + provider_id=random_provider, + generate_threatscore=True, + generate_ens=False, + generate_nis2=False, + generate_csa=False, + generate_cis=False, + ) + + assert result["threatscore"] == {"upload": False, "path": ""} + + @patch("tasks.jobs.report._upload_to_s3") + @patch("tasks.jobs.report.generate_cis_report") + def test_no_findings_returns_flat_cis_entry( + self, + mock_cis, + mock_upload, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Scan with no findings and ``generate_cis=True`` must yield a flat + ``{"upload": False, "path": ""}`` entry, consistent with the other + frameworks (no nested dict, no sentinel keys).""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + result = generate_compliance_reports( + tenant_id=str(tenant.id), + scan_id=str(scan.id), + provider_id=str(provider.id), + generate_threatscore=False, + generate_ens=False, + generate_nis2=False, + generate_csa=False, + generate_cis=True, + ) + + assert result["cis"] == {"upload": False, "path": ""} + mock_cis.assert_not_called() + + @patch("tasks.jobs.report.rmtree") + @patch("tasks.jobs.report._upload_to_s3") + @patch("tasks.jobs.report.generate_threatscore_report") + @patch("tasks.jobs.report._generate_compliance_output_directory") + @patch("tasks.jobs.report._aggregate_requirement_statistics_from_database") + @patch("tasks.jobs.report.Compliance.get_bulk") + @patch("tasks.jobs.report.Provider.objects.get") + @patch("tasks.jobs.report.ScanSummary.objects.filter") + def test_cleanup_runs_when_supported_reports_upload_successfully( + self, + mock_scan_summary_filter, + mock_provider_get, + mock_get_bulk, + mock_aggregate_stats, + mock_generate_output_dir, + mock_threatscore, + mock_upload_to_s3, + mock_rmtree, + ): + """Cleanup must run when all generated (supported) reports are uploaded.""" + mock_scan_summary_filter.return_value.exists.return_value = True + mock_provider_get.return_value = Mock(uid="provider-uid", provider="m365") + mock_get_bulk.return_value = {} + mock_aggregate_stats.return_value = {} + mock_generate_output_dir.return_value = ( + "/tmp/tenant/scan/threatscore/prowler-output-provider-20240101000000" + ) + mock_upload_to_s3.return_value = ( + "s3://bucket/tenant/scan/threatscore/report.pdf" + ) + + result = generate_compliance_reports( + tenant_id=str(uuid.uuid4()), + scan_id=str(uuid.uuid4()), + provider_id=str(uuid.uuid4()), + generate_threatscore=True, + generate_ens=True, + generate_nis2=True, + generate_csa=True, + generate_cis=True, + ) + + assert result["threatscore"]["upload"] is True + assert result["ens"]["upload"] is False + assert result["nis2"]["upload"] is False + assert result["csa"]["upload"] is False + assert result["cis"] == {"upload": False, "path": ""} + mock_generate_output_dir.assert_called_once() + mock_threatscore.assert_called_once() + mock_rmtree.assert_called_once() + + @patch("tasks.jobs.report.rmtree") + @patch("tasks.jobs.report._upload_to_s3") + @patch("tasks.jobs.report.generate_threatscore_report") + @patch("tasks.jobs.report._generate_compliance_output_directory") + @patch("tasks.jobs.report._aggregate_requirement_statistics_from_database") + @patch("tasks.jobs.report.Compliance.get_bulk") + @patch("tasks.jobs.report.Provider.objects.get") + @patch("tasks.jobs.report.ScanSummary.objects.filter") + def test_cleanup_skipped_when_supported_upload_fails( + self, + mock_scan_summary_filter, + mock_provider_get, + mock_get_bulk, + mock_aggregate_stats, + mock_generate_output_dir, + mock_threatscore, + mock_upload_to_s3, + mock_rmtree, + ): + """Cleanup must not run when a generated report upload fails.""" + mock_scan_summary_filter.return_value.exists.return_value = True + mock_provider_get.return_value = Mock(uid="provider-uid", provider="m365") + mock_get_bulk.return_value = {} + mock_aggregate_stats.return_value = {} + mock_generate_output_dir.return_value = ( + "/tmp/tenant/scan/threatscore/prowler-output-provider-20240101000000" + ) + mock_upload_to_s3.return_value = None + + result = generate_compliance_reports( + tenant_id=str(uuid.uuid4()), + scan_id=str(uuid.uuid4()), + provider_id=str(uuid.uuid4()), + generate_threatscore=True, + generate_ens=True, + generate_nis2=True, + generate_csa=True, + generate_cis=True, + ) + + assert result["threatscore"]["upload"] is False + assert result["cis"] == {"upload": False, "path": ""} + mock_generate_output_dir.assert_called_once() + mock_threatscore.assert_called_once() + mock_rmtree.assert_not_called() + + +@pytest.mark.django_db +class TestGenerateComplianceReportsCIS: + """Test suite covering the CIS branch of generate_compliance_reports.""" + + def _force_scan_has_findings(self, monkeypatch): + """Bypass the ScanSummary.exists() early-return guard.""" + + class _FakeManager: + def filter(self, **kwargs): + class _Q: + def exists(self): + return True + + return _Q() + + monkeypatch.setattr("tasks.jobs.report.ScanSummary.objects", _FakeManager()) + + @patch("tasks.jobs.report._aggregate_requirement_statistics_from_database") + @patch("tasks.jobs.report._upload_to_s3") + @patch("tasks.jobs.report.generate_cis_report") + @patch("tasks.jobs.report.Compliance.get_bulk") + def test_cis_picks_latest_version( + self, + mock_get_bulk, + mock_cis, + mock_upload, + mock_stats, + monkeypatch, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """CIS branch should generate a single PDF for the highest version. + + The returned ``results["cis"]`` must have the same flat shape as the + other single-version frameworks (``{"upload", "path"}``) — the picked + variant is an internal detail and is not exposed in the result. + """ + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + self._force_scan_has_findings(monkeypatch) + + mock_stats.return_value = {} + # Multiple CIS variants + a non-CIS framework that must be ignored. + # Includes 1.10 to verify the selection is not lexicographic. + mock_get_bulk.return_value = { + "cis_1.4_aws": Mock(), + "cis_1.10_aws": Mock(), + "cis_2.0_aws": Mock(), + "cis_5.0_aws": Mock(), + "ens_rd2022_aws": Mock(), + } + mock_upload.return_value = "s3://bucket/path" + + result = generate_compliance_reports( + tenant_id=str(tenant.id), + scan_id=str(scan.id), + provider_id=str(provider.id), + generate_threatscore=False, + generate_ens=False, + generate_nis2=False, + generate_csa=False, + generate_cis=True, + ) + + # Exactly one call for the latest version, never for older variants + # or non-CIS frameworks. + assert mock_cis.call_count == 1 + assert mock_cis.call_args.kwargs["compliance_id"] == "cis_5.0_aws" + + assert result["cis"]["upload"] is True + assert result["cis"]["path"] == "s3://bucket/path" + assert "compliance_id" not in result["cis"] + + @patch("tasks.jobs.report._aggregate_requirement_statistics_from_database") + @patch("tasks.jobs.report._upload_to_s3") + @patch("tasks.jobs.report.generate_cis_report") + @patch("tasks.jobs.report.Compliance.get_bulk") + def test_cis_latest_variant_failure_captured_in_results( + self, + mock_get_bulk, + mock_cis, + mock_upload, + mock_stats, + monkeypatch, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """A failure in the latest CIS variant must be surfaced in the flat results entry.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + self._force_scan_has_findings(monkeypatch) + + mock_stats.return_value = {} + mock_get_bulk.return_value = { + "cis_1.4_aws": Mock(), + "cis_5.0_aws": Mock(), + } + mock_cis.side_effect = RuntimeError("boom") + + result = generate_compliance_reports( + tenant_id=str(tenant.id), + scan_id=str(scan.id), + provider_id=str(provider.id), + generate_threatscore=False, + generate_ens=False, + generate_nis2=False, + generate_csa=False, + generate_cis=True, + ) + + # Only the latest variant is attempted; its failure lands in a flat + # entry keyed under "cis" with the same shape as sibling frameworks. + assert mock_cis.call_count == 1 + assert result["cis"]["upload"] is False + assert result["cis"]["error"] == "boom" + assert "compliance_id" not in result["cis"] + + @patch("tasks.jobs.report._aggregate_requirement_statistics_from_database") + @patch("tasks.jobs.report._upload_to_s3") + @patch("tasks.jobs.report.generate_cis_report") + @patch("tasks.jobs.report.Compliance.get_bulk") + def test_cis_provider_without_cis_skipped_cleanly( + self, + mock_get_bulk, + mock_cis, + mock_upload, + mock_stats, + monkeypatch, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """When ``Compliance.get_bulk`` returns no CIS entry the CIS branch + must skip cleanly and record a flat ``{"upload": False, "path": ""}`` + entry — no hard-coded provider whitelist is consulted.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + self._force_scan_has_findings(monkeypatch) + mock_stats.return_value = {} + # No ``cis_*`` keys in the bulk → no variant picked. + mock_get_bulk.return_value = {"ens_rd2022_aws": Mock()} + + result = generate_compliance_reports( + tenant_id=str(tenant.id), + scan_id=str(scan.id), + provider_id=str(provider.id), + generate_threatscore=False, + generate_ens=False, + generate_nis2=False, + generate_csa=False, + generate_cis=True, + ) + + assert result["cis"] == {"upload": False, "path": ""} + mock_cis.assert_not_called() + + @patch("tasks.jobs.report._aggregate_requirement_statistics_from_database") + @patch("tasks.jobs.report._generate_compliance_output_directory") + @patch("tasks.jobs.report.Compliance.get_bulk") + def test_cis_output_directory_failure_is_captured( + self, + mock_get_bulk, + mock_generate_output_dir, + mock_stats, + monkeypatch, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """CIS output dir errors must be captured in results (not raised).""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + self._force_scan_has_findings(monkeypatch) + mock_stats.return_value = {} + mock_get_bulk.return_value = {"cis_5.0_aws": Mock()} + mock_generate_output_dir.side_effect = RuntimeError("dir boom") + + result = generate_compliance_reports( + tenant_id=str(tenant.id), + scan_id=str(scan.id), + provider_id=str(provider.id), + generate_threatscore=False, + generate_ens=False, + generate_nis2=False, + generate_csa=False, + generate_cis=True, + ) + + assert result["cis"]["upload"] is False + assert result["cis"]["error"] == "dir boom" + + +class TestPickLatestCisVariant: + """Unit tests for `_pick_latest_cis_variant` helper.""" + + def test_empty_returns_none(self): + assert _pick_latest_cis_variant([]) is None + + def test_single_variant(self): + assert _pick_latest_cis_variant(["cis_5.0_aws"]) == "cis_5.0_aws" + + def test_numeric_not_lexicographic(self): + """1.10 must beat 1.2 (lex sort would pick 1.2).""" + variants = ["cis_1.2_kubernetes", "cis_1.10_kubernetes"] + assert _pick_latest_cis_variant(variants) == "cis_1.10_kubernetes" + + def test_major_version_wins(self): + variants = ["cis_1.4_aws", "cis_2.0_aws", "cis_5.0_aws", "cis_6.0_aws"] + assert _pick_latest_cis_variant(variants) == "cis_6.0_aws" + + def test_minor_version_breaks_tie(self): + variants = ["cis_3.0_aws", "cis_3.1_aws", "cis_2.9_aws"] + assert _pick_latest_cis_variant(variants) == "cis_3.1_aws" + + def test_three_part_version(self): + """Versions like 3.0.1 must win over 3.0.""" + variants = ["cis_3.0_aws", "cis_3.0.1_aws"] + assert _pick_latest_cis_variant(variants) == "cis_3.0.1_aws" + + def test_malformed_names_ignored(self): + variants = ["notcis_1.0_aws", "cis_abc_aws", "cis_5.0_aws"] + assert _pick_latest_cis_variant(variants) == "cis_5.0_aws" + + def test_only_malformed_returns_none(self): + variants = ["notcis_1.0_aws", "cis_abc_aws"] + assert _pick_latest_cis_variant(variants) is None + + def test_multidigit_provider_name(self): + """Provider name with underscores (e.g. googleworkspace) must parse.""" + variants = ["cis_1.3_googleworkspace"] + assert _pick_latest_cis_variant(variants) == "cis_1.3_googleworkspace" + + def test_accepts_iterator(self): + """The helper must accept any iterable, not just lists.""" + + def _gen(): + yield "cis_1.4_aws" + yield "cis_5.0_aws" + + assert _pick_latest_cis_variant(_gen()) == "cis_5.0_aws" + + def test_rejects_single_integer_version(self): + """The regex requires at least one dotted component. ``cis_5_aws`` + without a minor version is malformed per the backend contract.""" + assert _pick_latest_cis_variant(["cis_5_aws"]) is None + + def test_rejects_trailing_dot(self): + """Inputs like ``cis_5._aws`` must be rejected at the regex stage + instead of silently normalising to ``(5, 0)``.""" + assert _pick_latest_cis_variant(["cis_5._aws", "cis_1.0_aws"]) == "cis_1.0_aws" + + def test_rejects_lone_dot_version(self): + """``cis_._aws`` has no numeric component and must be skipped.""" + assert _pick_latest_cis_variant(["cis_._aws", "cis_1.0_aws"]) == "cis_1.0_aws" + class TestOptimizationImprovements: """Test suite for optimization-related functionality.""" diff --git a/api/src/backend/tasks/tests/test_reports_cis.py b/api/src/backend/tasks/tests/test_reports_cis.py new file mode 100644 index 0000000000..2d4528c82d --- /dev/null +++ b/api/src/backend/tasks/tests/test_reports_cis.py @@ -0,0 +1,532 @@ +from unittest.mock import Mock, patch + +import pytest +from reportlab.platypus import Image, LongTable, Paragraph, Table +from tasks.jobs.reports import FRAMEWORK_REGISTRY, ComplianceData, RequirementData +from tasks.jobs.reports.cis import ( + CISReportGenerator, + _normalize_profile, + _profile_badge_text, +) + +from api.models import StatusChoices + +# ============================================================================= +# Fixtures +# ============================================================================= + + +@pytest.fixture +def cis_generator(): + """Create a CISReportGenerator instance for testing.""" + config = FRAMEWORK_REGISTRY["cis"] + return CISReportGenerator(config) + + +def _make_attr( + section: str, + profile_value: str = "Level 1", + assessment_value: str = "Automated", + sub_section: str = "", + **extras, +) -> Mock: + """Build a mock CIS_Requirement_Attribute with duck-typed fields.""" + attr = Mock() + attr.Section = section + attr.SubSection = sub_section + # CIS enums have `.value`. Use a simple Mock that exposes `.value`. + attr.Profile = Mock(value=profile_value) + attr.AssessmentStatus = Mock(value=assessment_value) + attr.Description = extras.get("description", "desc") + attr.RationaleStatement = extras.get("rationale", "the rationale") + attr.ImpactStatement = extras.get("impact", "the impact") + attr.RemediationProcedure = extras.get("remediation", "the remediation") + attr.AuditProcedure = extras.get("audit", "the audit") + attr.AdditionalInformation = "" + attr.DefaultValue = "" + attr.References = extras.get("references", "https://example.com") + return attr + + +@pytest.fixture +def basic_cis_compliance_data(): + """Create basic ComplianceData for CIS testing (no requirements).""" + return ComplianceData( + tenant_id="tenant-123", + scan_id="scan-456", + provider_id="provider-789", + compliance_id="cis_5.0_aws", + framework="CIS", + name="CIS Amazon Web Services Foundations Benchmark v5.0.0", + version="5.0", + description="Center for Internet Security AWS Foundations Benchmark", + ) + + +@pytest.fixture +def populated_cis_compliance_data(basic_cis_compliance_data): + """CIS data with mixed requirements across 2 sections, Profile L1/L2, Pass/Fail/Manual.""" + data = basic_cis_compliance_data + data.requirements = [ + RequirementData( + id="1.1", + description="Maintain current contact details", + status=StatusChoices.PASS, + passed_findings=5, + failed_findings=0, + total_findings=5, + checks=["aws_check_1"], + ), + RequirementData( + id="1.2", + description="Ensure root account has no access keys", + status=StatusChoices.FAIL, + passed_findings=0, + failed_findings=3, + total_findings=3, + checks=["aws_check_2"], + ), + RequirementData( + id="1.3", + description="Ensure MFA is enabled for all IAM users", + status=StatusChoices.MANUAL, + checks=[], + ), + RequirementData( + id="2.1", + description="Ensure S3 Buckets are logging", + status=StatusChoices.PASS, + passed_findings=2, + failed_findings=0, + total_findings=2, + checks=["aws_check_3"], + ), + RequirementData( + id="2.2", + description="Ensure encryption at rest is enabled", + status=StatusChoices.FAIL, + passed_findings=0, + failed_findings=4, + total_findings=4, + checks=["aws_check_4"], + ), + ] + data.attributes_by_requirement_id = { + "1.1": { + "attributes": { + "req_attributes": [ + _make_attr( + "1 Identity and Access Management", + profile_value="Level 1", + assessment_value="Automated", + ) + ], + "checks": ["aws_check_1"], + } + }, + "1.2": { + "attributes": { + "req_attributes": [ + _make_attr( + "1 Identity and Access Management", + profile_value="Level 1", + assessment_value="Automated", + ) + ], + "checks": ["aws_check_2"], + } + }, + "1.3": { + "attributes": { + "req_attributes": [ + _make_attr( + "1 Identity and Access Management", + profile_value="Level 2", + assessment_value="Manual", + ) + ], + "checks": [], + } + }, + "2.1": { + "attributes": { + "req_attributes": [ + _make_attr( + "2 Storage", + profile_value="Level 2", + assessment_value="Automated", + ) + ], + "checks": ["aws_check_3"], + } + }, + "2.2": { + "attributes": { + "req_attributes": [ + _make_attr( + "2 Storage", + profile_value="Level 1", + assessment_value="Automated", + ) + ], + "checks": ["aws_check_4"], + } + }, + } + return data + + +# ============================================================================= +# Helper function tests +# ============================================================================= + + +class TestNormalizeProfile: + """Test suite for _normalize_profile helper.""" + + def test_level_1_string(self): + assert _normalize_profile(Mock(value="Level 1")) == "L1" + + def test_level_2_string(self): + assert _normalize_profile(Mock(value="Level 2")) == "L2" + + def test_e3_level_1(self): + assert _normalize_profile(Mock(value="E3 Level 1")) == "L1" + + def test_e5_level_2(self): + assert _normalize_profile(Mock(value="E5 Level 2")) == "L2" + + def test_none_returns_other(self): + assert _normalize_profile(None) == "Other" + + def test_substring_trap_rejected(self): + """Unrelated tokens containing the literal ``L2`` must NOT map to L2.""" + # A future enum value like "CL2 Kubernetes Worker" would be silently + # misclassified by a naive substring check. + assert _normalize_profile(Mock(value="CL2 Worker")) == "Other" + assert _normalize_profile(Mock(value="HL2 Legacy")) == "Other" + + def test_raw_string_level_1(self): + # Mock without .value falls back to str(profile); use a real string + class NoValue: + def __str__(self): + return "Level 1" + + assert _normalize_profile(NoValue()) == "L1" + + def test_unknown_profile_returns_other(self): + assert _normalize_profile(Mock(value="Custom Profile")) == "Other" + + +class TestProfileBadgeText: + def test_l1_label(self): + assert _profile_badge_text("L1") == "Level 1" + + def test_l2_label(self): + assert _profile_badge_text("L2") == "Level 2" + + def test_other_label(self): + assert _profile_badge_text("Other") == "Other" + + +# ============================================================================= +# Generator initialization +# ============================================================================= + + +class TestCISGeneratorInitialization: + def test_generator_created(self, cis_generator): + assert cis_generator is not None + assert cis_generator.config.name == "cis" + + def test_generator_language(self, cis_generator): + assert cis_generator.config.language == "en" + + def test_generator_sections_dynamic(self, cis_generator): + # CIS sections differ per variant so config.sections MUST be None + assert cis_generator.config.sections is None + + def test_attribute_fields_contain_cis_specific(self, cis_generator): + for field in ("Profile", "AssessmentStatus", "RationaleStatement"): + assert field in cis_generator.config.attribute_fields + + +# ============================================================================= +# _derive_sections +# ============================================================================= + + +class TestDeriveSections: + def test_preserves_first_seen_order( + self, cis_generator, populated_cis_compliance_data + ): + sections = cis_generator._derive_sections(populated_cis_compliance_data) + assert sections == [ + "1 Identity and Access Management", + "2 Storage", + ] + + def test_deduplicates_sections(self, cis_generator, basic_cis_compliance_data): + basic_cis_compliance_data.requirements = [ + RequirementData(id="1.1", description="a", status=StatusChoices.PASS), + RequirementData(id="1.2", description="b", status=StatusChoices.PASS), + ] + attr = _make_attr("1 IAM") + basic_cis_compliance_data.attributes_by_requirement_id = { + "1.1": {"attributes": {"req_attributes": [attr], "checks": []}}, + "1.2": {"attributes": {"req_attributes": [attr], "checks": []}}, + } + assert cis_generator._derive_sections(basic_cis_compliance_data) == ["1 IAM"] + + def test_empty_data_returns_empty(self, cis_generator, basic_cis_compliance_data): + basic_cis_compliance_data.requirements = [] + basic_cis_compliance_data.attributes_by_requirement_id = {} + assert cis_generator._derive_sections(basic_cis_compliance_data) == [] + + +# ============================================================================= +# _compute_statistics +# ============================================================================= + + +class TestComputeStatistics: + def test_totals(self, cis_generator, populated_cis_compliance_data): + stats = cis_generator._compute_statistics(populated_cis_compliance_data) + assert stats["total"] == 5 + assert stats["passed"] == 2 + assert stats["failed"] == 2 + assert stats["manual"] == 1 + + def test_overall_compliance_excludes_manual( + self, cis_generator, populated_cis_compliance_data + ): + stats = cis_generator._compute_statistics(populated_cis_compliance_data) + # 2 passed / 4 evaluated (pass + fail) = 50% + assert stats["overall_compliance"] == pytest.approx(50.0) + + def test_overall_compliance_all_manual( + self, cis_generator, basic_cis_compliance_data + ): + basic_cis_compliance_data.requirements = [ + RequirementData(id="x", description="d", status=StatusChoices.MANUAL), + ] + attr = _make_attr("1 IAM", profile_value="Level 1", assessment_value="Manual") + basic_cis_compliance_data.attributes_by_requirement_id = { + "x": {"attributes": {"req_attributes": [attr], "checks": []}}, + } + stats = cis_generator._compute_statistics(basic_cis_compliance_data) + # No evaluated → defaults to 100% + assert stats["overall_compliance"] == 100.0 + + def test_profile_counts(self, cis_generator, populated_cis_compliance_data): + stats = cis_generator._compute_statistics(populated_cis_compliance_data) + profile = stats["profile_counts"] + # From fixture: + # L1: 1.1 (PASS, Auto), 1.2 (FAIL, Auto), 2.2 (FAIL, Auto) → pass=1, fail=2, manual=0 + # L2: 1.3 (MANUAL, Manual), 2.1 (PASS, Auto) → pass=1, fail=0, manual=1 + assert profile["L1"] == {"passed": 1, "failed": 2, "manual": 0} + assert profile["L2"] == {"passed": 1, "failed": 0, "manual": 1} + + def test_assessment_counts(self, cis_generator, populated_cis_compliance_data): + stats = cis_generator._compute_statistics(populated_cis_compliance_data) + assessment = stats["assessment_counts"] + # Automated: 1.1 PASS, 1.2 FAIL, 2.1 PASS, 2.2 FAIL → pass=2, fail=2, manual=0 + # Manual: 1.3 MANUAL → pass=0, fail=0, manual=1 + assert assessment["Automated"] == {"passed": 2, "failed": 2, "manual": 0} + assert assessment["Manual"] == {"passed": 0, "failed": 0, "manual": 1} + + def test_top_failing_sections_includes_all_evaluated( + self, cis_generator, populated_cis_compliance_data + ): + stats = cis_generator._compute_statistics(populated_cis_compliance_data) + top = stats["top_failing_sections"] + # Both sections have 1 PASS + 1 FAIL evaluated → tied at 50%. The + # sort is stable, so both must appear and both must be capped at + # 5 entries. + assert len(top) == 2 + section_names = {name for name, _ in top} + assert section_names == { + "1 Identity and Access Management", + "2 Storage", + } + + def test_compute_statistics_is_memoized( + self, cis_generator, populated_cis_compliance_data + ): + """Calling ``_compute_statistics`` twice with the same data must + reuse the cached value and not re-run the uncached kernel.""" + with patch.object( + CISReportGenerator, + "_compute_statistics_uncached", + wraps=cis_generator._compute_statistics_uncached, + ) as spy: + cis_generator._compute_statistics(populated_cis_compliance_data) + cis_generator._compute_statistics(populated_cis_compliance_data) + assert spy.call_count == 1 + + +# ============================================================================= +# Executive summary +# ============================================================================= + + +class TestCISExecutiveSummary: + def test_title_present(self, cis_generator, populated_cis_compliance_data): + elements = cis_generator.create_executive_summary(populated_cis_compliance_data) + paragraphs = [e for e in elements if isinstance(e, Paragraph)] + text = " ".join(str(p.text) for p in paragraphs) + assert "Executive Summary" in text + + def test_tables_rendered(self, cis_generator, populated_cis_compliance_data): + elements = cis_generator.create_executive_summary(populated_cis_compliance_data) + tables = [e for e in elements if isinstance(e, Table)] + # Exact count: Summary, Profile, Assessment, Top Failing Sections = 4. + assert len(tables) == 4 + + def test_no_requirements(self, cis_generator, basic_cis_compliance_data): + basic_cis_compliance_data.requirements = [] + basic_cis_compliance_data.attributes_by_requirement_id = {} + elements = cis_generator.create_executive_summary(basic_cis_compliance_data) + # With no requirements: Summary table always renders, and both Profile + # and Assessment breakdown tables render with a 0-filled default row, + # but Top Failing Sections is suppressed → exactly 3 tables. + tables = [e for e in elements if isinstance(e, Table)] + assert len(tables) == 3 + + +# ============================================================================= +# Charts section +# ============================================================================= + + +class TestCISChartsSection: + def test_charts_rendered(self, cis_generator, populated_cis_compliance_data): + elements = cis_generator.create_charts_section(populated_cis_compliance_data) + # At least 1 image for the pie + 1 for section bar + 1 for stacked + images = [e for e in elements if isinstance(e, Image)] + assert len(images) >= 1 + + def test_charts_no_data_no_crash(self, cis_generator, basic_cis_compliance_data): + basic_cis_compliance_data.requirements = [] + basic_cis_compliance_data.attributes_by_requirement_id = {} + elements = cis_generator.create_charts_section(basic_cis_compliance_data) + # Must not raise; may or may not have any Image + assert isinstance(elements, list) + + +# ============================================================================= +# Requirements index +# ============================================================================= + + +class TestCISRequirementsIndex: + def test_title_present(self, cis_generator, populated_cis_compliance_data): + elements = cis_generator.create_requirements_index( + populated_cis_compliance_data + ) + paragraphs = [e for e in elements if isinstance(e, Paragraph)] + text = " ".join(str(p.text) for p in paragraphs) + assert "Requirements Index" in text + + def test_groups_by_section(self, cis_generator, populated_cis_compliance_data): + elements = cis_generator.create_requirements_index( + populated_cis_compliance_data + ) + paragraphs = [e for e in elements if isinstance(e, Paragraph)] + text = " ".join(str(p.text) for p in paragraphs) + assert "1 Identity and Access Management" in text + assert "2 Storage" in text + + def test_renders_tables_per_section( + self, cis_generator, populated_cis_compliance_data + ): + elements = cis_generator.create_requirements_index( + populated_cis_compliance_data + ) + # One table per section with requirements. ``create_data_table`` + # returns a LongTable when the row count exceeds its threshold and a + # plain Table otherwise — both are valid. + tables = [e for e in elements if isinstance(e, (Table, LongTable))] + assert len(tables) == 2 + + +# ============================================================================= +# Detailed findings extras hook +# ============================================================================= + + +class TestRenderRequirementDetailExtras: + def test_inserts_all_fields(self, cis_generator, populated_cis_compliance_data): + req = populated_cis_compliance_data.requirements[1] # 1.2 FAIL + extras = cis_generator._render_requirement_detail_extras( + req, populated_cis_compliance_data + ) + text = " ".join(str(p.text) for p in extras if isinstance(p, Paragraph)) + assert "Rationale" in text + assert "Impact" in text + assert "Audit Procedure" in text + assert "Remediation" in text + assert "References" in text + + def test_missing_metadata_returns_empty( + self, cis_generator, basic_cis_compliance_data + ): + basic_cis_compliance_data.attributes_by_requirement_id = {} + req = RequirementData(id="99", description="unknown", status=StatusChoices.FAIL) + extras = cis_generator._render_requirement_detail_extras( + req, basic_cis_compliance_data + ) + assert extras == [] + + def test_escapes_html_chars(self, cis_generator, basic_cis_compliance_data): + attr = _make_attr( + "1 IAM", + rationale="", + ) + basic_cis_compliance_data.attributes_by_requirement_id = { + "1.1": {"attributes": {"req_attributes": [attr], "checks": []}} + } + req = RequirementData(id="1.1", description="d", status=StatusChoices.FAIL) + extras = cis_generator._render_requirement_detail_extras( + req, basic_cis_compliance_data + ) + text = " ".join(str(p.text) for p in extras if isinstance(p, Paragraph)) + assert "", + "resource%00id", + "", + ])("rejects malicious or invalid resourceId: %s", async (maliciousId) => { + // When + const result = await getResourceEvents(maliciousId); + + // Then + expect(result).toEqual({ error: "Invalid resource ID format." }); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/actions/resources/resources.ts b/ui/actions/resources/resources.ts index d65505704f..8af2576852 100644 --- a/ui/actions/resources/resources.ts +++ b/ui/actions/resources/resources.ts @@ -2,9 +2,12 @@ import { redirect } from "next/navigation"; -import { apiBaseUrl, getAuthHeaders } from "@/lib"; +import { getLatestFindings } from "@/actions/findings"; +import { listOrganizationsSafe } from "@/actions/organizations/organizations"; +import { apiBaseUrl, FINDINGS_FILTERED_SORT, getAuthHeaders } from "@/lib"; import { appendSanitizedProviderTypeFilters } from "@/lib/provider-filters"; import { handleApiResponse } from "@/lib/server-actions-helper"; +import { OrganizationResource } from "@/types/organizations"; export const getResources = async ({ page = 1, @@ -160,6 +163,64 @@ export const getLatestMetadataInfo = async ({ } }; +const SAFE_RESOURCE_ID_PATTERN = /^[a-zA-Z0-9_-]+$/; + +export const getResourceEvents = async ( + resourceId: string, + { + includeReadEvents = false, + lookbackDays = 90, + pageSize = 50, + }: { + includeReadEvents?: boolean; + lookbackDays?: number; + pageSize?: number; + } = {}, +) => { + if (!SAFE_RESOURCE_ID_PATTERN.test(resourceId)) { + return { error: "Invalid resource ID format." }; + } + + const headers = await getAuthHeaders({ contentType: false }); + + const url = new URL(`${apiBaseUrl}/resources/${resourceId}/events`); + url.searchParams.append("include_read_events", String(includeReadEvents)); + url.searchParams.append("lookback_days", String(lookbackDays)); + url.searchParams.append("page[size]", String(pageSize)); + + try { + const response = await fetch(url.toString(), { headers }); + + if (!response.ok) { + const rawText = await response.text().catch(() => ""); + const defaultError = "An error occurred while fetching events."; + try { + const errorData = rawText ? JSON.parse(rawText) : null; + return { + error: + errorData?.errors?.[0]?.detail || + errorData?.error || + errorData?.message || + rawText || + response.statusText || + defaultError, + status: response.status, + }; + } catch { + return { + error: rawText || response.statusText || defaultError, + status: response.status, + }; + } + } + + return handleApiResponse(response); + } catch (error) { + console.error("Error fetching resource events:", error); + return { error: "An unexpected error occurred." }; + } +}; + export const getResourceById = async ( id: string, { @@ -197,3 +258,57 @@ export const getResourceById = async ( return undefined; } }; + +export const getResourceDrawerData = async ({ + resourceId, + resourceUid, + providerId, + providerType, + page = 1, + pageSize = 10, + query = "", +}: { + resourceId: string; + resourceUid: string; + providerId: string; + providerType: string; + page?: number; + pageSize?: number; + query?: string; +}) => { + const isCloudEnv = process.env.NEXT_PUBLIC_IS_CLOUD_ENV === "true"; + + const [resourceData, findingsResponse, organizationsResponse] = + await Promise.all([ + getResourceById(resourceId, { fields: ["tags"] }), + getLatestFindings({ + page, + pageSize, + query, + sort: FINDINGS_FILTERED_SORT, + filters: { + "filter[resource_uid]": resourceUid, + "filter[status]": "FAIL", + }, + }), + isCloudEnv && providerType === "aws" + ? listOrganizationsSafe() + : Promise.resolve({ data: [] }), + ]); + + const providerOrg = + providerType === "aws" + ? (organizationsResponse.data.find((organization: OrganizationResource) => + organization.relationships?.providers?.data?.some( + (provider: { id: string }) => provider.id === providerId, + ), + ) ?? null) + : null; + + return { + findings: findingsResponse?.data ?? [], + findingsMeta: findingsResponse?.meta ?? null, + providerOrg, + resourceTags: resourceData?.data?.attributes.tags ?? {}, + }; +}; diff --git a/ui/actions/scans/scans.ts b/ui/actions/scans/scans.ts index 8716c5c1c6..08b0e8b451 100644 --- a/ui/actions/scans/scans.ts +++ b/ui/actions/scans/scans.ts @@ -76,11 +76,18 @@ export const getScansByState = async () => { } }; -export const getScan = async (scanId: string) => { +export const getScan = async ( + scanId: string, + { include }: { include?: string } = {}, +) => { const headers = await getAuthHeaders({ contentType: false }); const url = new URL(`${apiBaseUrl}/scans/${scanId}`); + if (include) { + url.searchParams.set("include", include); + } + try { const response = await fetch(url.toString(), { headers, @@ -311,55 +318,87 @@ export const getExportsZip = async (scanId: string) => { } }; -export const getComplianceCsv = async ( - scanId: string, - complianceId: string, -) => { - const headers = await getAuthHeaders({ contentType: false }); +/** + * Discriminated union returned by {@link _fetchScanBinary}. + * + * Exported so `ui/lib/helper.ts::downloadFile` can type-narrow on the + * `success` / `pending` / `error` tags without resorting to `any`. + */ +export type ScanBinaryResult = + | { success: true; data: string; filename: string } + | { pending: true; state: string | undefined; taskId: string | undefined } + | { error: string }; - const url = new URL( - `${apiBaseUrl}/scans/${scanId}/compliance/${complianceId}`, - ); +/** + * Shared binary-report fetcher used by CSV and PDF report downloads. + * + * All report endpoints (`/scans/{id}/compliance/{name}`, + * `/scans/{id}/{reportType}`) speak the same protocol: Bearer auth, 202 + * ACCEPTED while the generation task is still running, 2xx with a binary + * body when the artifact is ready, JSON error body otherwise. This helper + * encapsulates all of that so the public wrappers only have to build the + * URL and pick a filename. + * + * @param urlPath Path segment under `{apiBaseUrl}/scans/{scanId}/`. + * @param filename Download filename to surface to the user. + * @param errorLabel Friendly label used when the backend error body is empty. + * @returns A ``{ success, data, filename }`` object on 2xx, a + * ``{ pending, state, taskId }`` object on 202, or + * ``{ error }`` on any failure. + */ +const _fetchScanBinary = async ( + scanId: string, + urlPath: string, + filename: string, + errorLabel: string, +): Promise => { + const headers = await getAuthHeaders({ contentType: false }); + const url = new URL(`${apiBaseUrl}/scans/${scanId}/${urlPath}`); try { const response = await fetch(url.toString(), { headers }); if (response.status === 202) { const json = await response.json(); - const taskId = json?.data?.id; - const state = json?.data?.attributes?.state; return { pending: true, - state, - taskId, + state: json?.data?.attributes?.state, + taskId: json?.data?.id, }; } if (!response.ok) { - const errorData = await response.json(); + const errorData = await response.json().catch(() => ({})); throw new Error( errorData?.errors?.detail || - "Unable to retrieve compliance report. Contact support if the issue continues.", + `Unable to retrieve ${errorLabel}. Contact support if the issue continues.`, ); } const arrayBuffer = await response.arrayBuffer(); const base64 = Buffer.from(arrayBuffer).toString("base64"); - return { - success: true, - data: base64, - filename: `scan-${scanId}-compliance-${complianceId}.csv`, - }; + return { success: true, data: base64, filename }; } catch (error) { - return { - error: getErrorMessage(error), - }; + return { error: getErrorMessage(error) }; } }; +export const getComplianceCsv = async (scanId: string, complianceId: string) => + _fetchScanBinary( + scanId, + `compliance/${complianceId}`, + `scan-${scanId}-compliance-${complianceId}.csv`, + "compliance report", + ); + /** - * Generic function to get a compliance PDF report (ThreatScore, ENS, etc.) + * Get a compliance PDF report for any supported framework. + * + * For frameworks with multiple variants per provider (currently CIS) the + * backend generates a single PDF for the highest available version, so + * callers only need to pass the generic report type. + * * @param scanId - The scan ID * @param reportType - Type of report (from COMPLIANCE_REPORT_TYPES) * @returns Promise with the PDF data or error @@ -368,44 +407,11 @@ export const getCompliancePdfReport = async ( scanId: string, reportType: ComplianceReportType, ) => { - const headers = await getAuthHeaders({ contentType: false }); - - const url = new URL(`${apiBaseUrl}/scans/${scanId}/${reportType}`); - - try { - const response = await fetch(url.toString(), { headers }); - - if (response.status === 202) { - const json = await response.json(); - const taskId = json?.data?.id; - const state = json?.data?.attributes?.state; - return { - pending: true, - state, - taskId, - }; - } - - if (!response.ok) { - const errorData = await response.json(); - const reportName = COMPLIANCE_REPORT_DISPLAY_NAMES[reportType]; - throw new Error( - errorData?.errors?.detail || - `Unable to retrieve ${reportName} PDF report. Contact support if the issue continues.`, - ); - } - - const arrayBuffer = await response.arrayBuffer(); - const base64 = Buffer.from(arrayBuffer).toString("base64"); - - return { - success: true, - data: base64, - filename: `scan-${scanId}-${reportType}.pdf`, - }; - } catch (error) { - return { - error: getErrorMessage(error), - }; - } + const reportName = COMPLIANCE_REPORT_DISPLAY_NAMES[reportType]; + return _fetchScanBinary( + scanId, + reportType, + `scan-${scanId}-${reportType}.pdf`, + `${reportName} PDF report`, + ); }; diff --git a/ui/actions/users/tenants.ts b/ui/actions/users/tenants.ts index f730a02b7f..b43b467d93 100644 --- a/ui/actions/users/tenants.ts +++ b/ui/actions/users/tenants.ts @@ -1,5 +1,6 @@ "use server"; +import { revalidatePath } from "next/cache"; import { z } from "zod"; import { apiBaseUrl, getAuthHeaders } from "@/lib/helper"; @@ -37,7 +38,15 @@ const editTenantFormSchema = z path: ["name"], }); -export async function updateTenantName(_prevState: any, formData: FormData) { +export type UpdateTenantNameState = + | { errors: { name?: string } } + | { success: string } + | { error: string }; + +export async function updateTenantName( + _prevState: UpdateTenantNameState | null, + formData: FormData, +): Promise { const headers = await getAuthHeaders({ contentType: true }); const formDataObject = Object.fromEntries(formData); const validatedData = editTenantFormSchema.safeParse(formDataObject); @@ -82,3 +91,311 @@ export async function updateTenantName(_prevState: any, formData: FormData) { return handleApiError(error); } } + +const switchTenantSchema = z.object({ + tenantId: z.uuid(), +}); + +interface SwitchTenantSuccess { + success: true; + accessToken: string; + refreshToken: string; +} + +interface SwitchTenantError { + error: string; +} + +export type SwitchTenantState = SwitchTenantSuccess | SwitchTenantError; + +export async function switchTenant( + _prevState: SwitchTenantState | null, + formData: FormData, +): Promise { + const formDataObject = Object.fromEntries(formData); + const validatedData = switchTenantSchema.safeParse(formDataObject); + + if (!validatedData.success) { + return { error: "Invalid tenant ID" }; + } + + const { tenantId } = validatedData.data; + const headers = await getAuthHeaders({ contentType: true }); + + const payload = { + data: { + type: "tokens-switch-tenant", + attributes: { + tenant_id: tenantId, + }, + }, + }; + + try { + const url = new URL(`${apiBaseUrl}/tokens/switch`); + const response = await fetch(url.toString(), { + method: "POST", + headers, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => null); + const errorDetail = + errorData?.errors?.[0]?.detail || + `Failed to switch tenant: ${response.statusText}`; + throw new Error(errorDetail); + } + + const data = await response.json(); + const accessToken = data?.data?.attributes?.access; + const refreshToken = data?.data?.attributes?.refresh; + + if (!accessToken || !refreshToken) { + throw new Error("Missing tokens in switch tenant response"); + } + + return { success: true, accessToken, refreshToken }; + } catch (error) { + return handleApiError(error); + } +} + +const createTenantSchema = z.object({ + name: z + .string() + .trim() + .min(1, { message: "Name is required" }) + .max(100, { message: "Name must be 100 characters or less" }), +}); + +interface CreateTenantSuccess { + success: true; + tenantId: string; +} + +interface CreateTenantError { + error: string; +} + +export type CreateTenantState = CreateTenantSuccess | CreateTenantError; + +export async function createTenant( + _prevState: CreateTenantState | null, + formData: FormData, +): Promise { + const formDataObject = Object.fromEntries(formData); + const validatedData = createTenantSchema.safeParse(formDataObject); + + if (!validatedData.success) { + const fieldErrors = validatedData.error.flatten().fieldErrors; + return { error: fieldErrors?.name?.[0] || "Invalid input" }; + } + + const { name } = validatedData.data; + const headers = await getAuthHeaders({ contentType: true }); + + const payload = { + data: { + type: "tenants", + attributes: { name }, + }, + }; + + try { + const url = new URL(`${apiBaseUrl}/tenants`); + const response = await fetch(url.toString(), { + method: "POST", + headers, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => null); + const errorDetail = + errorData?.errors?.[0]?.detail || + `Failed to create tenant: ${response.statusText}`; + throw new Error(errorDetail); + } + + const data = await response.json(); + const tenantId = data?.data?.id; + + if (!tenantId) { + throw new Error("Missing tenant ID in create response"); + } + + revalidatePath("/profile"); + return { success: true, tenantId }; + } catch (error) { + return handleApiError(error); + } +} + +const deleteTenantSchema = z.object({ + tenantId: z.uuid(), +}); + +const switchThenDeleteTenantSchema = z.object({ + tenantId: z.uuid(), + targetTenantId: z.uuid(), +}); + +interface DeleteTenantSuccess { + success: true; +} + +interface DeleteTenantError { + error: string; +} + +export type DeleteTenantState = DeleteTenantSuccess | DeleteTenantError; + +export async function deleteTenant( + _prevState: DeleteTenantState | null, + formData: FormData, +): Promise { + const formDataObject = Object.fromEntries(formData); + const validatedData = deleteTenantSchema.safeParse(formDataObject); + + if (!validatedData.success) { + return { error: "Invalid tenant ID" }; + } + + const { tenantId } = validatedData.data; + const headers = await getAuthHeaders({ contentType: false }); + + try { + const url = new URL(`${apiBaseUrl}/tenants/${tenantId}`); + const response = await fetch(url.toString(), { + method: "DELETE", + headers, + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => null); + const errorDetail = + errorData?.errors?.[0]?.detail || + `Failed to delete tenant: ${response.statusText}`; + throw new Error(errorDetail); + } + + revalidatePath("/profile"); + return { success: true }; + } catch (error) { + return handleApiError(error); + } +} + +interface SwitchThenDeleteSuccess { + success: true; + accessToken: string; + refreshToken: string; +} + +interface SwitchThenDeleteError { + error: string; + accessToken?: string; + refreshToken?: string; +} + +export type SwitchThenDeleteTenantState = + | SwitchThenDeleteSuccess + | SwitchThenDeleteError; + +export async function switchThenDeleteTenant( + _prevState: SwitchThenDeleteTenantState | null, + formData: FormData, +): Promise { + const formDataObject = Object.fromEntries(formData); + const validatedData = switchThenDeleteTenantSchema.safeParse(formDataObject); + + if (!validatedData.success) { + return { error: "Invalid tenant or target tenant ID" }; + } + + const { tenantId, targetTenantId } = validatedData.data; + const headers = await getAuthHeaders({ contentType: true }); + + // Step 1: Switch to the target tenant (current token is still valid) + const switchPayload = { + data: { + type: "tokens-switch-tenant", + attributes: { + tenant_id: targetTenantId, + }, + }, + }; + + let newAccessToken: string; + let newRefreshToken: string; + + try { + const switchUrl = new URL(`${apiBaseUrl}/tokens/switch`); + const switchResponse = await fetch(switchUrl.toString(), { + method: "POST", + headers, + body: JSON.stringify(switchPayload), + }); + + if (!switchResponse.ok) { + const errorData = await switchResponse.json().catch(() => null); + const errorDetail = + errorData?.errors?.[0]?.detail || + `Failed to switch tenant: ${switchResponse.statusText}`; + throw new Error(errorDetail); + } + + const switchData = await switchResponse.json(); + newAccessToken = switchData?.data?.attributes?.access; + newRefreshToken = switchData?.data?.attributes?.refresh; + + if (!newAccessToken || !newRefreshToken) { + throw new Error("Missing tokens in switch tenant response"); + } + } catch (error) { + return handleApiError(error); + } + + // Step 2: Delete the old tenant using the NEW token + const deleteHeaders: Record = { + Accept: "application/vnd.api+json", + Authorization: `Bearer ${newAccessToken}`, + }; + + try { + const deleteUrl = new URL(`${apiBaseUrl}/tenants/${tenantId}`); + const deleteResponse = await fetch(deleteUrl.toString(), { + method: "DELETE", + headers: deleteHeaders, + }); + + if (!deleteResponse.ok) { + const errorData = await deleteResponse.json().catch(() => null); + const errorDetail = + errorData?.errors?.[0]?.detail || + `Failed to delete tenant: ${deleteResponse.statusText}`; + // Switch succeeded but delete failed — return tokens so client can still update session + return { + error: errorDetail, + accessToken: newAccessToken, + refreshToken: newRefreshToken, + }; + } + + revalidatePath("/profile"); + return { + success: true, + accessToken: newAccessToken, + refreshToken: newRefreshToken, + }; + } catch (error) { + // Switch succeeded but delete threw — return tokens so client can still update session + const errorResult = handleApiError(error); + return { + ...errorResult, + accessToken: newAccessToken, + refreshToken: newRefreshToken, + }; + } +} diff --git a/ui/actions/users/users.ts b/ui/actions/users/users.ts index 717606594d..446fa81812 100644 --- a/ui/actions/users/users.ts +++ b/ui/actions/users/users.ts @@ -2,17 +2,64 @@ import { revalidatePath } from "next/cache"; import { redirect } from "next/navigation"; +import { z } from "zod"; +import { auth } from "@/auth.config"; import { apiBaseUrl, getAuthHeaders } from "@/lib"; import { handleApiError, handleApiResponse } from "@/lib/server-actions-helper"; +import { + TENANT_MEMBERSHIP_ROLE, + type TenantMembershipRole, +} from "@/types/users"; + +const getUsersSchema = z.object({ + page: z.coerce.number().int().min(1).default(1), + query: z.string().default(""), + sort: z.string().optional().default(""), + filters: z + .record( + z.string(), + z.union([z.string(), z.array(z.string()), z.number()]).optional(), + ) + .default({}), + pageSize: z.coerce.number().int().min(1).default(10), +}); + +const updateUserSchema = z.object({ + userId: z.uuid(), + name: z.string().min(1).optional(), + email: z.email().optional(), + company_name: z.string().optional(), + password: z.string().min(1).optional(), +}); + +const deleteUserSchema = z.object({ + userId: z.uuid(), +}); + +const removeUserFromTenantSchema = z.object({ + userId: z.uuid(), + tenantId: z.uuid(), +}); + +const updateUserRoleSchema = z.object({ + userId: z.uuid(), + roleId: z.uuid(), +}); + +type GetUsersInput = z.input; +type UpdateUserData = z.infer; +type UserAttributes = Omit; +type MembershipResource = { id: string }; + +export const getUsers = async (rawParams: Partial = {}) => { + const parsed = getUsersSchema.safeParse(rawParams); + if (!parsed.success) { + console.error("Invalid getUsers params:", parsed.error.flatten()); + return undefined; + } + const { page, query, sort, filters, pageSize } = parsed.data; -export const getUsers = async ({ - page = 1, - query = "", - sort = "", - filters = {}, - pageSize = 10, -}) => { const headers = await getAuthHeaders({ contentType: false }); if (isNaN(Number(page)) || page < 1) redirect("/users?include=roles"); @@ -46,22 +93,29 @@ export const getUsers = async ({ export const updateUser = async (formData: FormData) => { const headers = await getAuthHeaders({ contentType: true }); - const userId = formData.get("userId") as string; // Ensure userId is a string - const userName = formData.get("name") as string | null; - const userPassword = formData.get("password") as string | null; - const userEmail = formData.get("email") as string | null; - const userCompanyName = formData.get("company_name") as string | null; + const rawData = { + userId: formData.get("userId"), + name: formData.get("name") ?? undefined, + email: formData.get("email") ?? undefined, + company_name: formData.get("company_name") ?? undefined, + password: formData.get("password") ?? undefined, + }; + const parsed = updateUserSchema.safeParse(rawData); + if (!parsed.success) { + return { error: "Invalid user data" }; + } + const { userId, name, email, company_name, password } = parsed.data; const url = new URL(`${apiBaseUrl}/users/${userId}`); // Prepare attributes to send based on changes - const attributes: Record = {}; + const attributes: UserAttributes = {}; // Add only changed fields - if (userName !== null) attributes.name = userName; - if (userEmail !== null) attributes.email = userEmail; - if (userCompanyName !== null) attributes.company_name = userCompanyName; - if (userPassword !== null) attributes.password = userPassword; + if (name !== undefined) attributes.name = name; + if (email !== undefined) attributes.email = email; + if (company_name !== undefined) attributes.company_name = company_name; + if (password !== undefined) attributes.password = password; // If no fields have changed, don't send the request if (Object.keys(attributes).length === 0) { @@ -90,13 +144,14 @@ export const updateUser = async (formData: FormData) => { export const updateUserRole = async (formData: FormData) => { const headers = await getAuthHeaders({ contentType: true }); - const userId = formData.get("userId") as string; - const roleId = formData.get("roleId") as string; - - // Validate required fields - if (!userId || !roleId) { + const parsed = updateUserRoleSchema.safeParse({ + userId: formData.get("userId"), + roleId: formData.get("roleId"), + }); + if (!parsed.success) { return { error: "userId and roleId are required" }; } + const { userId, roleId } = parsed.data; const url = new URL(`${apiBaseUrl}/users/${userId}/relationships/roles`); @@ -124,11 +179,11 @@ export const updateUserRole = async (formData: FormData) => { export const deleteUser = async (formData: FormData) => { const headers = await getAuthHeaders({ contentType: false }); - const userId = formData.get("userId"); - - if (!userId) { + const parsed = deleteUserSchema.safeParse({ userId: formData.get("userId") }); + if (!parsed.success) { return { error: "User ID is required" }; } + const { userId } = parsed.data; const url = new URL(`${apiBaseUrl}/users/${userId}`); @@ -158,6 +213,156 @@ export const deleteUser = async (formData: FormData) => { } }; +interface ServerActionErrorDetail { + detail: string; + code?: string; +} + +interface ServerActionErrorResponse { + errors: ServerActionErrorDetail[]; +} + +interface RemoveUserFromTenantSuccess { + success: true; +} + +type RemoveUserFromTenantResult = + | RemoveUserFromTenantSuccess + | ServerActionErrorResponse; + +const toErrorResponse = (detail: string): ServerActionErrorResponse => ({ + errors: [{ detail }], +}); + +export const removeUserFromTenant = async ( + formData: FormData, +): Promise => { + const headers = await getAuthHeaders({ contentType: false }); + const parsed = removeUserFromTenantSchema.safeParse({ + userId: formData.get("userId"), + tenantId: formData.get("tenantId"), + }); + if (!parsed.success) { + return toErrorResponse("userId and tenantId are required"); + } + const { userId, tenantId } = parsed.data; + + // Resolve the target user's membership id for the current tenant on the + // server so the client form can open instantly without a prefetch. + // + // We cannot use `/users/{userId}/memberships` here: that endpoint ignores + // the path user id and always returns the authenticated user's memberships, + // which would make us try to delete the caller's own membership. + const listUrl = new URL(`${apiBaseUrl}/tenants/${tenantId}/memberships`); + listUrl.searchParams.append("filter[user]", userId); + listUrl.searchParams.append("page[size]", "1"); + + let targetMembershipId: string | null = null; + try { + const listResponse = await fetch(listUrl.toString(), { headers }); + if (!listResponse.ok) { + const errorData = await listResponse.json().catch(() => ({})); + return { + errors: errorData.errors ?? [ + { detail: "Failed to resolve the user's membership" }, + ], + }; + } + const listData = (await listResponse.json()) as { + data?: MembershipResource[]; + }; + targetMembershipId = listData?.data?.[0]?.id ?? null; + } catch (error) { + const handled = handleApiError(error); + return toErrorResponse( + handled?.error ?? "Failed to resolve the user's membership", + ); + } + + if (!targetMembershipId) { + return toErrorResponse( + "This user is not a member of the current organization.", + ); + } + + const url = new URL( + `${apiBaseUrl}/tenants/${tenantId}/memberships/${targetMembershipId}`, + ); + + try { + const response = await fetch(url.toString(), { + method: "DELETE", + headers, + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + return { + errors: errorData.errors ?? [ + { detail: "Failed to expel the user from the organization" }, + ], + }; + } + + revalidatePath("/users"); + return { success: true }; + } catch (error) { + const handled = handleApiError(error); + return toErrorResponse( + handled?.error ?? "Failed to expel the user from the organization", + ); + } +}; + +interface MembershipAttributesResource { + id: string; + attributes?: { + role?: string; + }; +} + +/** + * Resolve the active user's role inside the current tenant by querying the + * tenant memberships list with `filter[user]`. Returns `null` if the role + * cannot be determined (missing session, API error, or no match), so the + * caller can default-deny the destructive UI action. + */ +export const getCurrentUserTenantRole = + async (): Promise => { + const session = await auth(); + const userId = session?.userId; + const tenantId = session?.tenantId; + if (!userId || !tenantId) { + return null; + } + + const headers = await getAuthHeaders({ contentType: false }); + const url = new URL(`${apiBaseUrl}/tenants/${tenantId}/memberships`); + url.searchParams.append("filter[user]", userId); + url.searchParams.append("page[size]", "1"); + + try { + const response = await fetch(url.toString(), { headers }); + if (!response.ok) { + return null; + } + const body = (await response.json()) as { + data?: MembershipAttributesResource[]; + }; + const role = body?.data?.[0]?.attributes?.role; + if ( + role === TENANT_MEMBERSHIP_ROLE.Owner || + role === TENANT_MEMBERSHIP_ROLE.Member + ) { + return role; + } + return null; + } catch (error) { + console.error("Error resolving current user's tenant role:", error); + return null; + } + }; + export const getUserInfo = async () => { const headers = await getAuthHeaders({ contentType: false }); const url = new URL( diff --git a/ui/app/(auth)/(guest-only)/layout.tsx b/ui/app/(auth)/(guest-only)/layout.tsx new file mode 100644 index 0000000000..3ff93d836b --- /dev/null +++ b/ui/app/(auth)/(guest-only)/layout.tsx @@ -0,0 +1,18 @@ +import { redirect } from "next/navigation"; +import { ReactNode } from "react"; + +import { auth } from "@/auth.config"; + +export default async function GuestOnlyLayout({ + children, +}: { + children: ReactNode; +}) { + const session = await auth(); + + if (session?.user) { + redirect("/"); + } + + return <>{children}; +} diff --git a/ui/app/(auth)/sign-in/page.tsx b/ui/app/(auth)/(guest-only)/sign-in/page.tsx similarity index 100% rename from ui/app/(auth)/sign-in/page.tsx rename to ui/app/(auth)/(guest-only)/sign-in/page.tsx diff --git a/ui/app/(auth)/sign-up/page.tsx b/ui/app/(auth)/(guest-only)/sign-up/page.tsx similarity index 100% rename from ui/app/(auth)/sign-up/page.tsx rename to ui/app/(auth)/(guest-only)/sign-up/page.tsx diff --git a/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx b/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx new file mode 100644 index 0000000000..f334053b48 --- /dev/null +++ b/ui/app/(auth)/invitation/accept/accept-invitation-client.tsx @@ -0,0 +1,215 @@ +"use client"; + +import { Icon } from "@iconify/react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { signOut } from "next-auth/react"; +import { useEffect, useRef, useState } from "react"; + +import { acceptInvitation } from "@/actions/invitations"; +import { Button } from "@/components/shadcn"; + +type AcceptState = + | { kind: "no-token" } + | { kind: "accepting" } + | { kind: "error"; message: string; canRetry: boolean; needsSignOut: boolean } + | { kind: "choose" }; + +function mapApiError(status: number | undefined): { + message: string; + canRetry: boolean; + needsSignOut: boolean; +} { + switch (status) { + case 410: + return { + message: + "This invitation has expired. Please contact your administrator for a new one.", + canRetry: false, + needsSignOut: false, + }; + case 400: + return { + message: "This invitation has already been used.", + canRetry: false, + needsSignOut: false, + }; + case 404: + return { + message: + "This invitation was sent to a different email address. Please sign in with the correct account.", + canRetry: false, + needsSignOut: true, + }; + default: + return { + message: "Something went wrong while accepting the invitation.", + canRetry: true, + needsSignOut: false, + }; + } +} + +export function AcceptInvitationClient({ + isAuthenticated, + token, +}: { + isAuthenticated: boolean; + token: string | null; +}) { + const router = useRouter(); + const [state, setState] = useState(() => { + if (!token) return { kind: "no-token" }; + if (!isAuthenticated) return { kind: "choose" }; + return { kind: "accepting" }; + }); + const hasStartedRef = useRef(false); + + async function doAccept() { + if (!token) return; + setState({ kind: "accepting" }); + + const result = await acceptInvitation(token); + + if (result?.error) { + const { message, canRetry, needsSignOut } = mapApiError(result.status); + setState({ kind: "error", message, canRetry, needsSignOut }); + } else { + router.push("/"); + } + } + + async function handleSignOutAndRedirect() { + if (!token) return; + const callbackPath = `/invitation/accept?invitation_token=${encodeURIComponent(token)}`; + await signOut({ redirect: false }); + router.push(`/sign-in?callbackUrl=${encodeURIComponent(callbackPath)}`); + } + + useEffect(() => { + if (hasStartedRef.current) return; + hasStartedRef.current = true; + + if (!token) { + setState({ kind: "no-token" }); + return; + } + + if (isAuthenticated) { + doAccept(); + } else { + setState({ kind: "choose" }); + } + }, [token, isAuthenticated]); // eslint-disable-line react-hooks/exhaustive-deps + + return ( +

+
+ {/* No token */} + {state.kind === "no-token" && ( +
+ +

Invalid Invitation Link

+

+ No invitation token was provided. Please check the link you + received. +

+ +
+ )} + + {/* Accepting */} + {state.kind === "accepting" && ( +
+ +

Accepting Invitation...

+

+ Please wait while we process your invitation. +

+
+ )} + + {/* Error */} + {state.kind === "error" && ( +
+ +

+ Could Not Accept Invitation +

+

{state.message}

+
+ {state.canRetry && } + {state.needsSignOut ? ( + + ) : ( + + )} +
+
+ )} + + {/* Choice page for unauthenticated users */} + {state.kind === "choose" && ( +
+ +
+

+ You've Been Invited +

+

+ You've been invited to join a tenant on Prowler. How would + you like to continue? +

+
+
+ + +
+
+ )} +
+
+ ); +} diff --git a/ui/app/(auth)/invitation/accept/page.tsx b/ui/app/(auth)/invitation/accept/page.tsx new file mode 100644 index 0000000000..9bfc5e0110 --- /dev/null +++ b/ui/app/(auth)/invitation/accept/page.tsx @@ -0,0 +1,22 @@ +import { auth } from "@/auth.config"; +import { SearchParamsProps } from "@/types"; + +import { AcceptInvitationClient } from "./accept-invitation-client"; + +export default async function AcceptInvitationPage({ + searchParams, +}: { + searchParams: Promise; +}) { + const session = await auth(); + const resolvedSearchParams = await searchParams; + + const token = + typeof resolvedSearchParams?.invitation_token === "string" + ? resolvedSearchParams.invitation_token + : null; + + return ( + + ); +} diff --git a/ui/app/(auth)/layout.tsx b/ui/app/(auth)/layout.tsx index ab6b1e9bfa..07fe3a60c3 100644 --- a/ui/app/(auth)/layout.tsx +++ b/ui/app/(auth)/layout.tsx @@ -2,10 +2,8 @@ import "@/styles/globals.css"; import { GoogleTagManager } from "@next/third-parties/google"; import { Metadata, Viewport } from "next"; -import { redirect } from "next/navigation"; -import { ReactNode } from "react"; +import { ReactNode, Suspense } from "react"; -import { auth } from "@/auth.config"; import { NavigationProgress, Toaster } from "@/components/ui"; import { fontSans } from "@/config/fonts"; import { siteConfig } from "@/config/site"; @@ -31,17 +29,7 @@ export const viewport: Viewport = { ], }; -export default async function RootLayout({ - children, -}: { - children: ReactNode; -}) { - const session = await auth(); - - if (session?.user) { - redirect("/"); - } - +export default function AuthLayout({ children }: { children: ReactNode }) { return ( @@ -53,7 +41,9 @@ export default async function RootLayout({ )} > - + + + {children} ({ + useSearchParams: () => new URLSearchParams(), +})); + +vi.mock("@/hooks/use-url-filters", () => ({ + useUrlFilters: () => ({ + navigateWithParams: vi.fn(), + }), +})); + +vi.mock("@/components/icons/providers-badge", () => ({ + AWSProviderBadge: () => AWS, + AzureProviderBadge: () => Azure, + GCPProviderBadge: () => GCP, + CloudflareProviderBadge: () => Cloudflare, + GitHubProviderBadge: () => GitHub, + GoogleWorkspaceProviderBadge: () => Google Workspace, + IacProviderBadge: () => IaC, + ImageProviderBadge: () => Image, + KS8ProviderBadge: () => Kubernetes, + M365ProviderBadge: () => M365, + MongoDBAtlasProviderBadge: () => MongoDB Atlas, + OpenStackProviderBadge: () => OpenStack, + OracleCloudProviderBadge: () => Oracle Cloud, + AlibabaCloudProviderBadge: () => Alibaba Cloud, + VercelProviderBadge: () => Vercel, +})); + +vi.mock("@/components/shadcn/select/multiselect", () => ({ + MultiSelect: ({ children }: { children: React.ReactNode }) => ( +
{children}
+ ), + MultiSelectTrigger: ({ children }: { children: React.ReactNode }) => ( +
{children}
+ ), + MultiSelectValue: ({ placeholder }: { placeholder: string }) => ( + {placeholder} + ), + MultiSelectContent: ({ + children, + search, + }: { + children: React.ReactNode; + search?: unknown; + }) => { + multiSelectContentSpy(search); + return
{children}
; + }, + MultiSelectItem: ({ + children, + value, + keywords, + }: { + children: React.ReactNode; + value: string; + keywords?: string[]; + }) => ( +
+ {children} +
+ ), +})); + +const providers = [ + { + id: "provider-1", + type: "providers" as const, + attributes: { + provider: "aws" as const, + uid: "123456789012", + alias: "Production AWS", + status: "completed" as const, + resources: 0, + connection: { + connected: true, + last_checked_at: "2026-04-13T00:00:00Z", + }, + scanner_args: { + only_logs: false, + excluded_checks: [], + aws_retries_max_attempts: 3, + }, + inserted_at: "2026-04-13T00:00:00Z", + updated_at: "2026-04-13T00:00:00Z", + created_by: { + object: "user", + id: "user-1", + }, + }, + relationships: { + secret: { + data: null, + }, + provider_groups: { + meta: { + count: 0, + }, + data: [], + }, + }, + }, +]; + +describe("AccountsSelector", () => { + it("passes searchable dropdown defaults to MultiSelectContent", () => { + render(); + + expect(multiSelectContentSpy).toHaveBeenCalledWith({ + placeholder: "Search accounts...", + emptyMessage: "No accounts found.", + }); + expect(screen.getByText("Production AWS")).toBeInTheDocument(); + }); + + it("allows disabling search explicitly", () => { + render(); + + expect(multiSelectContentSpy).toHaveBeenLastCalledWith(false); + }); + + it("passes visible account labels as search keywords instead of only the internal id", () => { + render(); + + expect( + screen.getByText("Production AWS").closest("[data-value]"), + ).toHaveAttribute( + "data-keywords", + expect.stringContaining("Production AWS"), + ); + expect( + screen.getByText("Production AWS").closest("[data-value]"), + ).toHaveAttribute("data-keywords", expect.stringContaining("123456789012")); + }); +}); diff --git a/ui/app/(prowler)/_overview/_components/accounts-selector.tsx b/ui/app/(prowler)/_overview/_components/accounts-selector.tsx index 0e3845787e..e134625d98 100644 --- a/ui/app/(prowler)/_overview/_components/accounts-selector.tsx +++ b/ui/app/(prowler)/_overview/_components/accounts-selector.tsx @@ -10,22 +10,30 @@ import { CloudflareProviderBadge, GCPProviderBadge, GitHubProviderBadge, + GoogleWorkspaceProviderBadge, IacProviderBadge, + ImageProviderBadge, KS8ProviderBadge, M365ProviderBadge, MongoDBAtlasProviderBadge, OpenStackProviderBadge, OracleCloudProviderBadge, + VercelProviderBadge, } from "@/components/icons/providers-badge"; import { MultiSelect, MultiSelectContent, MultiSelectItem, + type MultiSelectSearchProp, MultiSelectTrigger, MultiSelectValue, } from "@/components/shadcn/select/multiselect"; import { useUrlFilters } from "@/hooks/use-url-filters"; -import type { ProviderProps, ProviderType } from "@/types/providers"; +import { + getProviderDisplayName, + type ProviderProps, + type ProviderType, +} from "@/types/providers"; const PROVIDER_ICON: Record = { aws: , @@ -34,68 +42,89 @@ const PROVIDER_ICON: Record = { kubernetes: , m365: , github: , + googleworkspace: , iac: , + image: , oraclecloud: , mongodbatlas: , alibabacloud: , cloudflare: , openstack: , + vercel: , }; -interface AccountsSelectorProps { +/** Common props shared by both batch and instant modes. */ +interface AccountsSelectorBaseProps { providers: ProviderProps[]; + search?: MultiSelectSearchProp; + /** + * Currently selected provider types (from the pending ProviderTypeSelector state). + * Used only for contextual description/empty-state messaging — does NOT narrow + * the list of available accounts, which remains independent of provider selection. + */ + selectedProviderTypes?: string[]; } -export function AccountsSelector({ providers }: AccountsSelectorProps) { +/** Batch mode: caller controls both pending state and notification callback (all-or-nothing). */ +interface AccountsSelectorBatchProps extends AccountsSelectorBaseProps { + /** + * Called instead of navigating immediately. + * Use this on pages that batch filter changes (e.g. Findings). + * + * @param filterKey - The raw filter key without "filter[]" wrapper, e.g. "provider_id__in" + * @param values - The selected values array + */ + onBatchChange: (filterKey: string, values: string[]) => void; + /** + * Pending selected values controlled by the parent. + * Reflects pending state before Apply is clicked. + */ + selectedValues: string[]; +} + +/** Instant mode: URL-driven — neither callback nor controlled value. */ +interface AccountsSelectorInstantProps extends AccountsSelectorBaseProps { + onBatchChange?: never; + selectedValues?: never; +} + +type AccountsSelectorProps = + | AccountsSelectorBatchProps + | AccountsSelectorInstantProps; + +export function AccountsSelector({ + providers, + onBatchChange, + selectedValues, + selectedProviderTypes, + search = { + placeholder: "Search accounts...", + emptyMessage: "No accounts found.", + }, +}: AccountsSelectorProps) { const searchParams = useSearchParams(); const { navigateWithParams } = useUrlFilters(); const filterKey = "filter[provider_id__in]"; const current = searchParams.get(filterKey) || ""; - const selectedTypes = searchParams.get("filter[provider_type__in]") || ""; - const selectedTypesList = selectedTypes - ? selectedTypes.split(",").filter(Boolean) - : []; - const selectedIds = current ? current.split(",").filter(Boolean) : []; - const visibleProviders = providers - // .filter((p) => p.attributes.connection?.connected) - .filter((p) => - selectedTypesList.length > 0 - ? selectedTypesList.includes(p.attributes.provider) - : true, - ); + const urlSelectedIds = current ? current.split(",").filter(Boolean) : []; + + // In batch mode, use the parent-controlled pending values; otherwise, use URL state. + const selectedIds = onBatchChange ? selectedValues : urlSelectedIds; + const visibleProviders = providers; + // .filter((p) => p.attributes.connection?.connected) const handleMultiValueChange = (ids: string[]) => { + if (onBatchChange) { + onBatchChange("provider_id__in", ids); + return; + } navigateWithParams((params) => { params.delete(filterKey); if (ids.length > 0) { params.set(filterKey, ids.join(",")); } - - // Auto-deselect provider types that no longer have any selected accounts - if (selectedTypesList.length > 0) { - // Get provider types of currently selected accounts - const selectedProviders = providers.filter((p) => ids.includes(p.id)); - const selectedProviderTypes = new Set( - selectedProviders.map((p) => p.attributes.provider), - ); - - // Keep only provider types that still have selected accounts - const remainingProviderTypes = selectedTypesList.filter((type) => - selectedProviderTypes.has(type as ProviderType), - ); - - // Update provider_type__in filter - if (remainingProviderTypes.length > 0) { - params.set( - "filter[provider_type__in]", - remainingProviderTypes.join(","), - ); - } else { - params.delete("filter[provider_type__in]"); - } - } }); }; @@ -111,9 +140,12 @@ export function AccountsSelector({ providers }: AccountsSelectorProps) { ); }; + // Build a contextual description based on currently selected provider types. + // This is purely for user guidance (aria label + empty state) and does NOT + // narrow the list of available accounts — all providers remain selectable. const filterDescription = - selectedTypesList.length > 0 - ? `Showing accounts for ${selectedTypesList.join(", ")} providers` + selectedProviderTypes && selectedProviderTypes.length > 0 + ? `Accounts for ${selectedProviderTypes.map(getProviderDisplayName).join(", ")}` : "All connected cloud provider accounts"; return ( @@ -133,7 +165,7 @@ export function AccountsSelector({ providers }: AccountsSelectorProps) { > {selectedLabel() || } - + {visibleProviders.length > 0 ? ( <>
@@ -172,8 +212,8 @@ export function AccountsSelector({ providers }: AccountsSelectorProps) { ) : (
- {selectedTypesList.length > 0 - ? "No accounts available for selected providers" + {selectedProviderTypes && selectedProviderTypes.length > 0 + ? `No accounts available for ${selectedProviderTypes.map(getProviderDisplayName).join(", ")}` : "No connected accounts available"}
)} diff --git a/ui/app/(prowler)/_overview/_components/provider-type-selector.test.tsx b/ui/app/(prowler)/_overview/_components/provider-type-selector.test.tsx new file mode 100644 index 0000000000..a60766394f --- /dev/null +++ b/ui/app/(prowler)/_overview/_components/provider-type-selector.test.tsx @@ -0,0 +1,138 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import { ProviderTypeSelector } from "./provider-type-selector"; + +const multiSelectContentSpy = vi.fn(); + +vi.mock("next/navigation", () => ({ + useSearchParams: () => new URLSearchParams(), +})); + +vi.mock("@/hooks/use-url-filters", () => ({ + useUrlFilters: () => ({ + navigateWithParams: vi.fn(), + }), +})); + +vi.mock("@/components/icons/providers-badge", () => ({ + AWSProviderBadge: () => AWS, + AzureProviderBadge: () => Azure, + GCPProviderBadge: () => GCP, + KS8ProviderBadge: () => Kubernetes, + M365ProviderBadge: () => M365, + GitHubProviderBadge: () => GitHub, + GoogleWorkspaceProviderBadge: () => Google Workspace, + IacProviderBadge: () => IaC, + ImageProviderBadge: () => Image, + OracleCloudProviderBadge: () => Oracle Cloud, + MongoDBAtlasProviderBadge: () => MongoDB Atlas, + AlibabaCloudProviderBadge: () => Alibaba Cloud, + CloudflareProviderBadge: () => Cloudflare, + OpenStackProviderBadge: () => OpenStack, + VercelProviderBadge: () => Vercel, +})); + +vi.mock("@/components/shadcn/select/multiselect", () => ({ + MultiSelect: ({ children }: { children: React.ReactNode }) => ( +
{children}
+ ), + MultiSelectTrigger: ({ children }: { children: React.ReactNode }) => ( +
{children}
+ ), + MultiSelectValue: ({ placeholder }: { placeholder: string }) => ( + {placeholder} + ), + MultiSelectContent: ({ + children, + search, + }: { + children: React.ReactNode; + search?: unknown; + }) => { + multiSelectContentSpy(search); + return
{children}
; + }, + MultiSelectItem: ({ + children, + value, + keywords, + }: { + children: React.ReactNode; + value: string; + keywords?: string[]; + }) => ( +
+ {children} +
+ ), +})); + +const providers = [ + { + id: "provider-1", + type: "providers" as const, + attributes: { + provider: "aws" as const, + uid: "123456789012", + alias: "Production AWS", + status: "completed" as const, + resources: 0, + connection: { + connected: true, + last_checked_at: "2026-04-13T00:00:00Z", + }, + scanner_args: { + only_logs: false, + excluded_checks: [], + aws_retries_max_attempts: 3, + }, + inserted_at: "2026-04-13T00:00:00Z", + updated_at: "2026-04-13T00:00:00Z", + created_by: { + object: "user", + id: "user-1", + }, + }, + relationships: { + secret: { + data: null, + }, + provider_groups: { + meta: { + count: 0, + }, + data: [], + }, + }, + }, +]; + +describe("ProviderTypeSelector", () => { + it("passes searchable dropdown defaults to MultiSelectContent", () => { + render(); + + expect(multiSelectContentSpy).toHaveBeenCalledWith({ + placeholder: "Search providers...", + emptyMessage: "No providers found.", + }); + expect(screen.getByText("Amazon Web Services")).toBeInTheDocument(); + }); + + it("allows disabling search explicitly", () => { + render(); + + expect(multiSelectContentSpy).toHaveBeenLastCalledWith(false); + }); + + it("passes provider label as search keywords", () => { + render(); + + expect( + screen.getByText("Amazon Web Services").closest("[data-value]"), + ).toHaveAttribute( + "data-keywords", + expect.stringContaining("Amazon Web Services"), + ); + }); +}); diff --git a/ui/app/(prowler)/_overview/_components/provider-type-selector.tsx b/ui/app/(prowler)/_overview/_components/provider-type-selector.tsx index 881bc54969..819bc49000 100644 --- a/ui/app/(prowler)/_overview/_components/provider-type-selector.tsx +++ b/ui/app/(prowler)/_overview/_components/provider-type-selector.tsx @@ -1,12 +1,13 @@ "use client"; import { useSearchParams } from "next/navigation"; -import { lazy, Suspense } from "react"; +import { type ComponentType, lazy, Suspense } from "react"; import { MultiSelect, MultiSelectContent, MultiSelectItem, + type MultiSelectSearchProp, MultiSelectTrigger, MultiSelectValue, } from "@/components/shadcn/select/multiselect"; @@ -48,6 +49,11 @@ const IacProviderBadge = lazy(() => default: m.IacProviderBadge, })), ); +const ImageProviderBadge = lazy(() => + import("@/components/icons/providers-badge").then((m) => ({ + default: m.ImageProviderBadge, + })), +); const OracleCloudProviderBadge = lazy(() => import("@/components/icons/providers-badge").then((m) => ({ default: m.OracleCloudProviderBadge, @@ -73,6 +79,16 @@ const OpenStackProviderBadge = lazy(() => default: m.OpenStackProviderBadge, })), ); +const GoogleWorkspaceProviderBadge = lazy(() => + import("@/components/icons/providers-badge").then((m) => ({ + default: m.GoogleWorkspaceProviderBadge, + })), +); +const VercelProviderBadge = lazy(() => + import("@/components/icons/providers-badge").then((m) => ({ + default: m.VercelProviderBadge, + })), +); type IconProps = { width: number; height: number }; @@ -82,7 +98,7 @@ const IconPlaceholder = ({ width, height }: IconProps) => ( const PROVIDER_DATA: Record< ProviderType, - { label: string; icon: React.ComponentType } + { label: string; icon: ComponentType } > = { aws: { label: "Amazon Web Services", @@ -108,10 +124,18 @@ const PROVIDER_DATA: Record< label: "GitHub", icon: GitHubProviderBadge, }, + googleworkspace: { + label: "Google Workspace", + icon: GoogleWorkspaceProviderBadge, + }, iac: { label: "Infrastructure as Code", icon: IacProviderBadge, }, + image: { + label: "Container Registry", + icon: ImageProviderBadge, + }, oraclecloud: { label: "Oracle Cloud Infrastructure", icon: OracleCloudProviderBadge, @@ -132,24 +156,71 @@ const PROVIDER_DATA: Record< label: "OpenStack", icon: OpenStackProviderBadge, }, + vercel: { + label: "Vercel", + icon: VercelProviderBadge, + }, }; -type ProviderTypeSelectorProps = { +/** Common props shared by both batch and instant modes. */ +interface ProviderTypeSelectorBaseProps { providers: ProviderProps[]; -}; + search?: MultiSelectSearchProp; +} + +/** Batch mode: caller controls both pending state and notification callback (all-or-nothing). */ +interface ProviderTypeSelectorBatchProps extends ProviderTypeSelectorBaseProps { + /** + * Called instead of navigating immediately. + * Use this on pages that batch filter changes (e.g. Findings). + * + * @param filterKey - The raw filter key without "filter[]" wrapper, e.g. "provider_type__in" + * @param values - The selected values array + */ + onBatchChange: (filterKey: string, values: string[]) => void; + /** + * Pending selected values controlled by the parent. + * Reflects pending state before Apply is clicked. + */ + selectedValues: string[]; +} + +/** Instant mode: URL-driven — neither callback nor controlled value. */ +interface ProviderTypeSelectorInstantProps + extends ProviderTypeSelectorBaseProps { + onBatchChange?: never; + selectedValues?: never; +} + +type ProviderTypeSelectorProps = + | ProviderTypeSelectorBatchProps + | ProviderTypeSelectorInstantProps; export const ProviderTypeSelector = ({ providers, + onBatchChange, + selectedValues, + search = { + placeholder: "Search providers...", + emptyMessage: "No providers found.", + }, }: ProviderTypeSelectorProps) => { const searchParams = useSearchParams(); const { navigateWithParams } = useUrlFilters(); const currentProviders = searchParams.get("filter[provider_type__in]") || ""; - const selectedTypes = currentProviders + const urlSelectedTypes = currentProviders ? currentProviders.split(",").filter(Boolean) : []; + // In batch mode, use the parent-controlled pending values; otherwise, use URL state. + const selectedTypes = onBatchChange ? selectedValues : urlSelectedTypes; + const handleMultiValueChange = (values: string[]) => { + if (onBatchChange) { + onBatchChange("provider_type__in", values); + return; + } navigateWithParams((params) => { // Update provider_type__in if (values.length > 0) { @@ -157,10 +228,6 @@ export const ProviderTypeSelector = ({ } else { params.delete("filter[provider_type__in]"); } - - // Clear account selection when changing provider types - // User should manually select accounts if they want to filter by specific accounts - params.delete("filter[provider_id__in]"); }); }; @@ -170,7 +237,11 @@ export const ProviderTypeSelector = ({ // .filter((p) => p.attributes.connection?.connected) .map((p) => p.attributes.provider), ), - ).filter((type): type is ProviderType => type in PROVIDER_DATA); + ) + .filter((type): type is ProviderType => type in PROVIDER_DATA) + .sort((a, b) => + PROVIDER_DATA[a].label.localeCompare(PROVIDER_DATA[b].label), + ); const renderIcon = (providerType: ProviderType) => { const IconComponent = PROVIDER_DATA[providerType].icon; @@ -219,7 +290,7 @@ export const ProviderTypeSelector = ({ > {selectedLabel() || } - + {availableTypes.length > 0 ? ( <>
diff --git a/ui/app/(prowler)/_overview/attack-surface/_components/attack-surface-card-item.tsx b/ui/app/(prowler)/_overview/attack-surface/_components/attack-surface-card-item.tsx index 46f2b0dd8e..ef92b7b4b3 100644 --- a/ui/app/(prowler)/_overview/attack-surface/_components/attack-surface-card-item.tsx +++ b/ui/app/(prowler)/_overview/attack-surface/_components/attack-surface-card-item.tsx @@ -2,6 +2,7 @@ import Link from "next/link"; import { AttackSurfaceItem } from "@/actions/overview"; import { Card, CardContent } from "@/components/shadcn"; +import { applyFailNonMutedFilters } from "@/lib"; interface AttackSurfaceCardItemProps { item: AttackSurfaceItem; @@ -18,8 +19,7 @@ export function AttackSurfaceCardItem({ // Add attack surface category filter params.set("filter[category__in]", item.id); - params.set("filter[status__in]", "FAIL"); - params.set("filter[muted]", "false"); + applyFailNonMutedFilters(params); // Add current page filters (provider, account, etc.) Object.entries(filters).forEach(([key, value]) => { diff --git a/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.test.ts b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.test.ts new file mode 100644 index 0000000000..03b990c1d5 --- /dev/null +++ b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.test.ts @@ -0,0 +1,16 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +describe("findings view overview SSR", () => { + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const filePath = path.join(currentDir, "findings-view.ssr.tsx"); + const source = readFileSync(filePath, "utf8"); + + it("uses the non-legacy latest findings columns", () => { + expect(source).toContain("ColumnLatestFindings"); + expect(source).not.toContain("ColumnNewFindingsToDate"); + }); +}); diff --git a/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx index be2aec7770..0396795cb9 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/findings-view/findings-view.ssr.tsx @@ -3,8 +3,10 @@ import { getLatestFindings } from "@/actions/findings/findings"; import { LighthouseBanner } from "@/components/lighthouse/banner"; import { LinkToFindings } from "@/components/overview"; -import { ColumnNewFindingsToDate } from "@/components/overview/new-findings-table/table/column-new-findings-to-date"; +import { ColumnLatestFindings } from "@/components/overview/new-findings-table/table"; +import { CardTitle } from "@/components/shadcn"; import { DataTable } from "@/components/ui/table"; +import { FINDINGS_FILTERED_SORT } from "@/lib"; import { createDict } from "@/lib/helper"; import { FindingProps, SearchParamsProps } from "@/types"; @@ -16,7 +18,7 @@ interface FindingsViewSSRProps { export async function FindingsViewSSR({ searchParams }: FindingsViewSSRProps) { const page = 1; - const sort = "severity,-inserted_at"; + const sort = FINDINGS_FILTERED_SORT; const defaultFilters = { "filter[status]": "FAIL", @@ -57,24 +59,23 @@ export async function FindingsViewSSR({ searchParams }: FindingsViewSSRProps) { }; return ( -
+
-
-
-

- Latest new failing findings -

-

- Showing the latest 10 new failing findings by severity. -

- -
-
- +
+ Latest New Failed Findings +

+ Showing the latest 10 sorted by severity +

+
+ +
+ } />
); diff --git a/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx b/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx index c21491e37f..f9741dc75e 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/graphs-tabs-wrapper.tsx @@ -1,6 +1,7 @@ import { Skeleton } from "@heroui/skeleton"; import { Suspense } from "react"; +import { SkeletonTableNewFindings } from "@/components/overview/new-findings-table/table"; import { SearchParamsProps } from "@/types"; import { GraphsTabsClient } from "./_components/graphs-tabs-client"; @@ -18,6 +19,10 @@ const LoadingFallback = () => (
); +const TAB_FALLBACKS: Partial> = { + findings: , +}; + type GraphComponent = React.ComponentType<{ searchParams: SearchParamsProps }>; const GRAPH_COMPONENTS: Record = { @@ -38,9 +43,10 @@ export const GraphsTabsWrapper = async ({ const tabsContent = Object.fromEntries( GRAPH_TABS.map((tab) => { const Component = GRAPH_COMPONENTS[tab.id]; + const fallback = TAB_FALLBACKS[tab.id] ?? ; return [ tab.id, - }> + , ]; diff --git a/ui/app/(prowler)/_overview/graphs-tabs/risk-plot/risk-plot-client.tsx b/ui/app/(prowler)/_overview/graphs-tabs/risk-plot/risk-plot-client.tsx index 84405fe0f1..f1db021599 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/risk-plot/risk-plot-client.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/risk-plot/risk-plot-client.tsx @@ -8,6 +8,7 @@ import { HorizontalBarChart } from "@/components/graphs/horizontal-bar-chart"; import { ScatterPlot } from "@/components/graphs/scatter-plot"; import { AlertPill } from "@/components/graphs/shared/alert-pill"; import type { BarDataPoint } from "@/components/graphs/types"; +import { applyFailNonMutedFilters } from "@/lib"; import { SEVERITY_FILTER_MAP } from "@/types/severities"; // Score color thresholds (0-100 scale, higher = better) @@ -50,11 +51,7 @@ export function RiskPlotClient({ data }: RiskPlotClientProps) { // Add provider filter for the selected point params.set("filter[provider_id__in]", selectedPoint.providerId); - // Add exclude muted findings filter - params.set("filter[muted]", "false"); - - // Filter by FAIL findings - params.set("filter[status__in]", "FAIL"); + applyFailNonMutedFilters(params); // Navigate to findings page router.push(`/findings?${params.toString()}`); diff --git a/ui/app/(prowler)/_overview/graphs-tabs/risk-radar-view/risk-radar-view-client.tsx b/ui/app/(prowler)/_overview/graphs-tabs/risk-radar-view/risk-radar-view-client.tsx index 5cd079ed2f..41a09b4809 100644 --- a/ui/app/(prowler)/_overview/graphs-tabs/risk-radar-view/risk-radar-view-client.tsx +++ b/ui/app/(prowler)/_overview/graphs-tabs/risk-radar-view/risk-radar-view-client.tsx @@ -7,6 +7,7 @@ import { HorizontalBarChart } from "@/components/graphs/horizontal-bar-chart"; import { RadarChart } from "@/components/graphs/radar-chart"; import type { BarDataPoint, RadarDataPoint } from "@/components/graphs/types"; import { Card } from "@/components/shadcn/card/card"; +import { applyFailNonMutedFilters } from "@/lib"; import { SEVERITY_FILTER_MAP } from "@/types/severities"; import { CategorySelector } from "./category-selector"; @@ -50,11 +51,7 @@ export function RiskRadarViewClient({ data }: RiskRadarViewClientProps) { // Add category filter for the selected point params.set("filter[category__in]", selectedPoint.categoryId); - // Add exclude muted findings filter - params.set("filter[muted]", "false"); - - // Filter by FAIL findings - params.set("filter[status__in]", "FAIL"); + applyFailNonMutedFilters(params); // Navigate to findings page router.push(`/findings?${params.toString()}`); diff --git a/ui/app/(prowler)/_overview/resources-inventory/_components/resources-inventory-card-item.test.tsx b/ui/app/(prowler)/_overview/resources-inventory/_components/resources-inventory-card-item.test.tsx new file mode 100644 index 0000000000..9d6683cdf5 --- /dev/null +++ b/ui/app/(prowler)/_overview/resources-inventory/_components/resources-inventory-card-item.test.tsx @@ -0,0 +1,120 @@ +import { render, screen } from "@testing-library/react"; +import { Shield } from "lucide-react"; +import type { ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { ResourceInventoryItem } from "@/actions/overview"; + +import { ResourcesInventoryCardItem } from "./resources-inventory-card-item"; + +vi.mock("next/link", () => ({ + default: ({ children, href }: { children: ReactNode; href: string }) => ( + {children} + ), +})); + +const baseItem: ResourceInventoryItem = { + id: "security", + label: "Security", + icon: Shield, + totalResources: 616, + totalFindings: 319, + failedFindings: 319, + newFailedFindings: 64, + severity: { + critical: 12, + high: 44, + medium: 108, + low: 155, + informational: 0, + }, +}; + +describe("ResourcesInventoryCardItem", () => { + describe("when the group has resources and failed findings", () => { + it("builds a resources link that forwards current page filters", () => { + render( + , + ); + + const link = screen.getByRole("link"); + + expect(link).toHaveAttribute( + "href", + expect.stringContaining("/resources?"), + ); + expect(link).toHaveAttribute( + "href", + expect.stringContaining("filter%5Bgroups__in%5D=security"), + ); + expect(link).toHaveAttribute( + "href", + expect.stringContaining("filter%5Bprovider__in%5D=aws-provider"), + ); + expect(link).toHaveAttribute( + "href", + expect.stringContaining("filter%5Baccount_id__in%5D=account-1"), + ); + }); + + it("renders a fail accent bar so the card is theme-agnostic", () => { + render(); + + const card = screen.getByText("Security").closest("[data-slot='card']"); + const accent = card?.querySelector( + "[data-slot='resource-stats-card-accent']", + ); + + expect(card).not.toBeNull(); + expect(accent).not.toBeNull(); + }); + }); + + describe("when the group has resources but no failed findings", () => { + it("renders a pass accent bar and the ShieldCheck badge", () => { + render( + , + ); + + const card = screen.getByText("Security").closest("[data-slot='card']"); + const accent = card?.querySelector( + "[data-slot='resource-stats-card-accent']", + ); + + expect(accent).not.toBeNull(); + expect(screen.getByRole("link")).toBeInTheDocument(); + }); + }); + + describe("when the group has no resources", () => { + it("renders the empty state without a link", () => { + render( + , + ); + + expect(screen.queryByRole("link")).not.toBeInTheDocument(); + expect(screen.getByText("No Findings to display")).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/app/(prowler)/_overview/resources-inventory/_components/resources-inventory-card-item.tsx b/ui/app/(prowler)/_overview/resources-inventory/_components/resources-inventory-card-item.tsx index e820519308..af4267c58a 100644 --- a/ui/app/(prowler)/_overview/resources-inventory/_components/resources-inventory-card-item.tsx +++ b/ui/app/(prowler)/_overview/resources-inventory/_components/resources-inventory-card-item.tsx @@ -1,8 +1,9 @@ -import { Bell, TriangleAlert } from "lucide-react"; +import { Bell, ShieldCheck, TriangleAlert } from "lucide-react"; import Link from "next/link"; import { ResourceInventoryItem } from "@/actions/overview"; import { CardVariant, ResourceStatsCard, StatItem } from "@/components/shadcn"; +import { cn } from "@/lib/utils"; interface ResourcesInventoryCardItemProps { item: ResourceInventoryItem; @@ -15,6 +16,7 @@ export function ResourcesInventoryCardItem({ }: ResourcesInventoryCardItemProps) { const hasFailedFindings = item.failedFindings > 0; const hasResources = item.totalResources > 0; + const accent = hasFailedFindings ? CardVariant.fail : CardVariant.pass; // Build URL with current filters + resource group specific filters const buildResourcesUrl = () => { @@ -49,52 +51,49 @@ export function ResourcesInventoryCardItem({ }); } - // Empty state when no resources + const header = { + icon: item.icon, + title: item.label, + resourceCount: `${item.totalResources.toLocaleString()} Resources`, + }; + if (!hasResources) { - const cardContent = ( + return ( ); - - return cardContent; } - // Card with findings data const cardContent = ( ); if (resourcesUrl) { return ( - + {cardContent} ); diff --git a/ui/app/(prowler)/_overview/resources-inventory/resources-inventory-skeleton.tsx b/ui/app/(prowler)/_overview/resources-inventory/resources-inventory-skeleton.tsx index 461fe1ae5a..45a3b82115 100644 --- a/ui/app/(prowler)/_overview/resources-inventory/resources-inventory-skeleton.tsx +++ b/ui/app/(prowler)/_overview/resources-inventory/resources-inventory-skeleton.tsx @@ -3,7 +3,8 @@ import { Skeleton } from "@/components/shadcn/skeleton/skeleton"; function ResourceCardSkeleton() { return ( -
+
+ {/* Header */}
diff --git a/ui/app/(prowler)/_overview/severity-over-time/_components/finding-severity-over-time.tsx b/ui/app/(prowler)/_overview/severity-over-time/_components/finding-severity-over-time.tsx index fa127244bd..9e0802d4ff 100644 --- a/ui/app/(prowler)/_overview/severity-over-time/_components/finding-severity-over-time.tsx +++ b/ui/app/(prowler)/_overview/severity-over-time/_components/finding-severity-over-time.tsx @@ -6,6 +6,7 @@ import { useState } from "react"; import { getSeverityTrendsByTimeRange } from "@/actions/overview/severity-trends"; import { LineChart } from "@/components/graphs/line-chart"; import { LineConfig, LineDataPoint } from "@/components/graphs/types"; +import { applyFailNonMutedFilters } from "@/lib"; import { SEVERITY_LEVELS, SEVERITY_LINE_CONFIGS, @@ -57,11 +58,8 @@ export const FindingSeverityOverTime = ({ }) => { const params = new URLSearchParams(); - // Always filter by FAIL status since this chart shows failed findings - params.set("filter[status__in]", "FAIL"); - - // Exclude muted findings - params.set("filter[muted]", "false"); + // Show active failing findings only for this chart's drill-down. + applyFailNonMutedFilters(params); // Add scan_ids filter if ( diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/graph-loading.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/graph-loading.tsx index be4bd1b528..7b5645dfdf 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/graph-loading.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/graph/graph-loading.tsx @@ -1,6 +1,6 @@ "use client"; -import { TreeSpinner } from "@/components/shadcn/tree-view/tree-spinner"; +import { Spinner } from "@/components/shadcn/spinner/spinner"; /** * Loading skeleton for graph visualization @@ -12,7 +12,7 @@ export const GraphLoading = () => { data-testid="graph-loading" className="flex min-h-[320px] flex-col items-center justify-center gap-4 text-center" > - +

Loading Attack Paths graph...

diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/index.ts b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/index.ts index eac86fccc7..83161cc4ef 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/index.ts +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/index.ts @@ -1,6 +1,8 @@ export { ExecuteButton } from "./execute-button"; export * from "./graph"; export * from "./node-detail"; +export { QueryDescription } from "./query-description"; +export { QueryExecutionError } from "./query-execution-error"; export { QueryParametersForm } from "./query-parameters-form"; export { QuerySelector } from "./query-selector"; export { ScanListTable } from "./scan-list-table"; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.test.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.test.tsx new file mode 100644 index 0000000000..d7209c68c8 --- /dev/null +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.test.tsx @@ -0,0 +1,93 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import type { ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import type { GraphNode } from "@/types/attack-paths"; + +import { NodeDetailPanel } from "./node-detail-panel"; + +vi.mock("@/components/ui/sheet/sheet", () => ({ + Sheet: ({ children }: { children: ReactNode }) =>
{children}
, + SheetContent: ({ children }: { children: ReactNode }) => ( +
{children}
+ ), + SheetDescription: ({ children }: { children: ReactNode }) => ( +
{children}
+ ), + SheetHeader: ({ children }: { children: ReactNode }) =>
{children}
, + SheetTitle: ({ children }: { children: ReactNode }) =>
{children}
, +})); + +vi.mock("./node-overview", () => ({ + NodeOverview: () =>
Node overview
, +})); + +vi.mock("./node-findings", () => ({ + NodeFindings: () =>
Node findings
, +})); + +vi.mock("./node-resources", () => ({ + NodeResources: () =>
Node resources
, +})); + +const findingNode: GraphNode = { + id: "graph-node-id", + labels: ["ProwlerFinding"], + properties: { + id: "finding-123", + check_title: "Open S3 bucket", + name: "Open S3 bucket", + }, +}; + +const resourceNode: GraphNode = { + id: "resource-node-id", + labels: ["S3Bucket"], + properties: { + id: "bucket-123", + name: "bucket-123", + }, +}; + +describe("NodeDetailPanel", () => { + it("renders the view finding button only for finding nodes", () => { + const { rerender } = render(); + + expect( + screen.getByRole("button", { name: /view finding finding-123/i }), + ).toBeInTheDocument(); + + rerender(); + + expect( + screen.queryByRole("button", { name: /view finding/i }), + ).not.toBeInTheDocument(); + }); + + it("calls onViewFinding with the node finding id", async () => { + const user = userEvent.setup(); + const onViewFinding = vi.fn(); + + render( + , + ); + + await user.click( + screen.getByRole("button", { name: /view finding finding-123/i }), + ); + + expect(onViewFinding).toHaveBeenCalledWith("finding-123"); + }); + + it("disables the button and shows the spinner while loading", () => { + render(); + + const button = screen.getByRole("button", { + name: /view finding finding-123/i, + }); + + expect(button).toBeDisabled(); + expect(screen.getByLabelText("Loading")).toHaveClass("size-4"); + }); +}); diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.tsx index 4d8885e65c..97d8657bfd 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-detail-panel.tsx @@ -1,6 +1,7 @@ "use client"; import { Button, Card, CardContent } from "@/components/shadcn"; +import { Spinner } from "@/components/shadcn/spinner/spinner"; import { Sheet, SheetContent, @@ -18,6 +19,8 @@ interface NodeDetailPanelProps { node: GraphNode | null; allNodes?: GraphNode[]; onClose?: () => void; + onViewFinding?: (findingId: string) => void; + viewFindingLoading?: boolean; } /** @@ -26,9 +29,13 @@ interface NodeDetailPanelProps { export const NodeDetailContent = ({ node, allNodes = [], + onViewFinding, + viewFindingLoading = false, }: { node: GraphNode; allNodes?: GraphNode[]; + onViewFinding?: (findingId: string) => void; + viewFindingLoading?: boolean; }) => { const isProwlerFinding = node?.labels.some((label) => label.toLowerCase().includes("finding"), @@ -56,7 +63,12 @@ export const NodeDetailContent = ({
Findings connected to this node
- + )} @@ -88,12 +100,15 @@ export const NodeDetailPanel = ({ node, allNodes = [], onClose, + onViewFinding, + viewFindingLoading = false, }: NodeDetailPanelProps) => { const isOpen = node !== null; const isProwlerFinding = node?.labels.some((label) => label.toLowerCase().includes("finding"), ); + const findingId = node ? String(node.properties?.id || node.id) : ""; return ( !open && onClose?.()}> @@ -107,15 +122,19 @@ export const NodeDetailPanel = ({
{node && isProwlerFinding && ( - )}
@@ -123,7 +142,12 @@ export const NodeDetailPanel = ({ {node && (
- +
)} diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-findings.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-findings.tsx index bb424a818c..253401fa97 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-findings.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-findings.tsx @@ -1,5 +1,7 @@ "use client"; +import { Button } from "@/components/shadcn"; +import { Spinner } from "@/components/shadcn/spinner/spinner"; import { SeverityBadge } from "@/components/ui/table/severity-badge"; import type { GraphNode } from "@/types/attack-paths"; @@ -16,13 +18,20 @@ type Severity = (typeof SEVERITY_LEVELS)[keyof typeof SEVERITY_LEVELS]; interface NodeFindingsProps { node: GraphNode; allNodes?: GraphNode[]; + onViewFinding?: (findingId: string) => void; + viewFindingLoading?: boolean; } /** * Node findings section showing related findings for the selected node * Displays findings that are connected to the node via HAS_FINDING edges */ -export const NodeFindings = ({ node, allNodes = [] }: NodeFindingsProps) => { +export const NodeFindings = ({ + node, + allNodes = [], + onViewFinding, + viewFindingLoading = false, +}: NodeFindingsProps) => { // Get finding IDs from the node's findings array (populated by adapter) const findingIds = node.findings || []; @@ -79,15 +88,20 @@ export const NodeFindings = ({ node, allNodes = [] }: NodeFindingsProps) => { ID: {findingId}

- onViewFinding?.(findingId)} + disabled={viewFindingLoading} aria-label={`View full finding for ${findingName}`} className="text-text-info dark:text-text-info h-auto shrink-0 p-0 text-xs font-medium hover:underline" > - View Full Finding → - + {viewFindingLoading ? ( + + ) : ( + "View Full Finding →" + )} +
{finding.properties?.description && (
diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-overview.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-overview.tsx index e614ce521a..2e294cf069 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-overview.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-overview.tsx @@ -1,7 +1,7 @@ "use client"; +import { InfoField } from "@/components/shadcn/info-field/info-field"; import { CodeSnippet } from "@/components/ui/code-snippet/code-snippet"; -import { InfoField } from "@/components/ui/entities"; import { DateWithTime } from "@/components/ui/entities/date-with-time"; import type { GraphNode, GraphNodePropertyValue } from "@/types/attack-paths"; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-remediation.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-remediation.tsx index e3421e5a43..3ece3184f0 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-remediation.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/node-detail/node-remediation.tsx @@ -1,8 +1,8 @@ "use client"; -import Link from "next/link"; - import { Badge } from "@/components/shadcn/badge/badge"; +import { Button } from "@/components/shadcn/button/button"; +import { Spinner } from "@/components/shadcn/spinner/spinner"; interface Finding { id: string; @@ -13,12 +13,18 @@ interface Finding { interface NodeRemediationProps { findings: Finding[]; + onViewFinding?: (findingId: string) => void; + viewFindingLoading?: boolean; } /** * Node remediation section showing related Prowler findings */ -export const NodeRemediation = ({ findings }: NodeRemediationProps) => { +export const NodeRemediation = ({ + findings, + onViewFinding, + viewFindingLoading = false, +}: NodeRemediationProps) => { const getSeverityVariant = (severity: string) => { switch (severity) { case "critical": @@ -66,15 +72,20 @@ export const NodeRemediation = ({ findings }: NodeRemediationProps) => {
- onViewFinding?.(finding.id)} + disabled={viewFindingLoading} aria-label={`View full finding for ${finding.title}`} - className="text-text-info dark:text-text-info text-sm transition-all hover:opacity-80 dark:hover:opacity-80" + className="text-text-info dark:text-text-info h-auto p-0 text-sm transition-all hover:opacity-80 dark:hover:opacity-80" > - View Full Finding → - + {viewFindingLoading ? ( + + ) : ( + "View Full Finding →" + )} +
))} diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-code-editor.test.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-code-editor.test.tsx new file mode 100644 index 0000000000..fb2f94d0fe --- /dev/null +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-code-editor.test.tsx @@ -0,0 +1,82 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import { QueryCodeEditor } from "./query-code-editor"; + +describe("QueryCodeEditor", () => { + it("renders a code editor surface with the compact read-only badge", () => { + // Given + render( + {}} + />, + ); + + // When + const editor = screen.getByRole("textbox", { name: /opencypher/i }); + + // Then + expect(editor).toHaveAttribute("contenteditable", "true"); + expect(screen.getByText("openCypher")).toBeInTheDocument(); + expect(screen.getByText("Read-only*")).toBeInTheDocument(); + expect(screen.getByText("MATCH (n) RETURN n LIMIT 25")).toBeInTheDocument(); + expect(screen.getByText("1")).toBeInTheDocument(); + }); + + it("keeps the accessible editor name when the visible label is hidden", () => { + // Given + render( + {}} + />, + ); + + // When + const editor = screen.getByRole("textbox", { + name: /resource metadata/i, + }); + + // Then + expect(editor).toBeInTheDocument(); + expect(screen.queryByText("Resource metadata")).not.toBeInTheDocument(); + expect( + screen.getByRole("button", { name: /copy resource metadata/i }), + ).toBeInTheDocument(); + }); + + it("propagates content changes and exposes the invalid state in the container", async () => { + // Given + const user = userEvent.setup(); + const onChange = vi.fn(); + + render( + , + ); + + // When + await user.type(screen.getByRole("textbox", { name: /opencypher/i }), "A"); + + // Then + expect(onChange).toHaveBeenCalled(); + expect(screen.getByTestId("query-code-editor")).toHaveClass( + "border-border-error-primary", + ); + }); +}); diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-code-editor.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-code-editor.tsx new file mode 100644 index 0000000000..c0c2283cb0 --- /dev/null +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-code-editor.tsx @@ -0,0 +1 @@ +export { QueryCodeEditor } from "@/components/shared/query-code-editor"; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-description.test.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-description.test.tsx new file mode 100644 index 0000000000..88ec6f139d --- /dev/null +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-description.test.tsx @@ -0,0 +1,76 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import type { AttackPathQuery } from "@/types/attack-paths"; + +import { QueryDescription } from "./query-description"; + +const customQuery: AttackPathQuery = { + type: "attack-paths-scans", + id: "custom-query", + attributes: { + name: "Custom openCypher query", + short_description: "Write your own query", + description: + "Run a read-only openCypher query against the selected Attack Paths scan.", + provider: "aws", + attribution: null, + documentation_link: { + text: "Learn how to write custom openCypher queries", + link: "https://example.com/docs", + }, + parameters: [], + }, +}; + +describe("QueryDescription", () => { + it("renders the documentation link inside an info alert", () => { + // Given + render(); + + // When + const alert = screen.getByRole("alert"); + const link = screen.getByRole("link", { + name: /learn how to write custom opencypher queries/i, + }); + + // Then + expect(alert).toBeInTheDocument(); + expect(link).toHaveAttribute("href", "https://example.com/docs"); + }); + + it("does not render unsafe documentation or attribution URLs as clickable links", () => { + // Given + const queryWithUnsafeLinks: AttackPathQuery = { + ...customQuery, + attributes: { + ...customQuery.attributes, + documentation_link: { + text: "Learn how to write custom openCypher queries", + link: "javascript:alert('xss')", + }, + attribution: { + text: "Unsafe source", + link: "javascript:alert('xss')", + }, + }, + }; + + // When + render(); + + // Then + expect( + screen.queryByRole("link", { + name: /learn how to write custom opencypher queries/i, + }), + ).not.toBeInTheDocument(); + expect( + screen.queryByRole("link", { name: /unsafe source/i }), + ).not.toBeInTheDocument(); + expect( + screen.getByText(/learn how to write custom opencypher queries/i), + ).toBeInTheDocument(); + expect(screen.getByText(/unsafe source/i)).toBeInTheDocument(); + }); +}); diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-description.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-description.tsx new file mode 100644 index 0000000000..d1cb6e85fe --- /dev/null +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-description.tsx @@ -0,0 +1,70 @@ +import { Info } from "lucide-react"; + +import { Alert, AlertDescription } from "@/components/shadcn"; +import type { AttackPathQuery } from "@/types/attack-paths"; + +interface QueryDescriptionProps { + query: AttackPathQuery; +} + +const isSafeUrl = (url: string): boolean => { + try { + const parsedUrl = new URL(url); + return parsedUrl.protocol === "https:" || parsedUrl.protocol === "http:"; + } catch { + return false; + } +}; + +export const QueryDescription = ({ query }: QueryDescriptionProps) => { + const documentationLink = query.attributes.documentation_link; + const attribution = query.attributes.attribution; + + return ( + + + +

{query.attributes.description}

+ + {documentationLink && ( +

+ {isSafeUrl(documentationLink.link) ? ( + + {documentationLink.text} + + ) : ( + {documentationLink.text} + )} +

+ )} + + {attribution && ( +

+ {isSafeUrl(attribution.link) ? ( + <> + Source:{" "} + + {attribution.text} + + + ) : ( + <> + Source: {attribution.text} + + )} +

+ )} + + + ); +}; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-execution-error.test.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-execution-error.test.tsx new file mode 100644 index 0000000000..1c7fb29712 --- /dev/null +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-execution-error.test.tsx @@ -0,0 +1,46 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { QueryExecutionError } from "./query-execution-error"; + +describe("QueryExecutionError", () => { + it("renders the default title and the raw query error details without extra copy", () => { + // Given + const error = + "Invalid input 'WHERE': expected 'MATCH' or 'WITH' (line 1, column 1)"; + + // When + render(); + + // Then + expect(screen.getByRole("alert")).toBeInTheDocument(); + expect(screen.getByText(/query execution failed/i)).toBeInTheDocument(); + expect( + screen.queryByText(/the attack paths query could not be executed/i), + ).not.toBeInTheDocument(); + expect(screen.getByText(error)).toBeInTheDocument(); + }); + + it("renders custom title and description when provided", () => { + // Given + const error = "Failed to load available queries"; + + // When + render( + , + ); + + // Then + expect(screen.getByText(/failed to load queries/i)).toBeInTheDocument(); + expect( + screen.getByText( + /available attack paths queries could not be loaded for this scan/i, + ), + ).toBeInTheDocument(); + expect(screen.getByText(error)).toBeInTheDocument(); + }); +}); diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-execution-error.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-execution-error.tsx new file mode 100644 index 0000000000..084bcf6e16 --- /dev/null +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-execution-error.tsx @@ -0,0 +1,30 @@ +import { CircleAlert } from "lucide-react"; + +import { Alert, AlertDescription, AlertTitle } from "@/components/shadcn"; + +interface QueryExecutionErrorProps { + error: string; + title?: string; + description?: string; +} + +export const QueryExecutionError = ({ + error, + title = "Query execution failed", + description, +}: QueryExecutionErrorProps) => { + return ( + + + {title} + + {description ?

{description}

: null} +
+ 
+            {error}
+
+
+ + + ); +}; diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-parameters-form.test.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-parameters-form.test.tsx index ac2b81be7f..d8493f4426 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-parameters-form.test.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-parameters-form.test.tsx @@ -1,8 +1,12 @@ import { render, screen } from "@testing-library/react"; +import { useEffect } from "react"; import { FormProvider, useForm } from "react-hook-form"; import { describe, expect, it } from "vitest"; -import type { AttackPathQuery } from "@/types/attack-paths"; +import { + ATTACK_PATH_QUERY_IDS, + type AttackPathQuery, +} from "@/types/attack-paths"; import { QueryParametersForm } from "./query-parameters-form"; @@ -42,6 +46,66 @@ function TestForm() { ); } +function TestCustomQueryForm() { + const customQuery: AttackPathQuery = { + type: "attack-paths-scans", + id: ATTACK_PATH_QUERY_IDS.CUSTOM, + attributes: { + name: "Custom openCypher query", + short_description: "Write your own query", + description: "Run a custom query against the graph.", + provider: "aws", + attribution: null, + parameters: [ + { + name: "query", + label: "openCypher", + data_type: "string", + input_type: "code-editor", + placeholder: "MATCH (n) RETURN n LIMIT 25", + description: "", + required: true, + editor_language: "openCypher", + requirement_badge: "Read-only*", + }, + ], + }, + }; + + const form = useForm({ + defaultValues: { + query: "", + }, + }); + + return ( + + + + ); +} + +function TestFormWithError() { + const form = useForm({ + defaultValues: { + tag_key: "", + }, + }); + + useEffect(() => { + form.setError("tag_key", { + type: "manual", + message: "Tag key is required", + }); + }, [form]); + + return ( + + + + ); +} + describe("QueryParametersForm", () => { it("uses the field description as the placeholder instead of rendering helper text below", () => { // Given @@ -70,4 +134,55 @@ describe("QueryParametersForm", () => { screen.queryByText("Tag key to filter the S3 bucket."), ).not.toBeInTheDocument(); }); + + it("renders a code editor when the parameter input type is code-editor", () => { + // Given + render(); + + // When + const input = screen.getByRole("textbox", { name: /opencypher/i }); + const codeEditor = screen.getByTestId("query-code-editor"); + + // Then + expect(input.tagName).toBe("DIV"); + expect(input).toHaveAttribute("contenteditable", "true"); + expect(codeEditor).toHaveAttribute("data-language", "openCypher"); + expect(codeEditor).toHaveClass( + "rounded-xl", + "border", + "bg-bg-neutral-primary", + ); + expect(screen.getByText("Read-only*")).toBeInTheDocument(); + expect(screen.getByText("openCypher")).toBeInTheDocument(); + expect(screen.queryByText("openCypher*")).not.toBeInTheDocument(); + expect(screen.queryByText("Read-only")).not.toBeInTheDocument(); + }); + + it("uses the design-system error token for field validation messages", async () => { + // Given + render(); + + // When + const errorMessage = await screen.findByText("Tag key is required"); + + // Then + expect(errorMessage).toHaveClass("text-text-error-primary", "text-xs"); + }); + + it("connects field errors to the input for accessibility", async () => { + // Given + render(); + + // When + const input = screen.getByRole("textbox", { name: /tag key/i }); + const errorMessage = await screen.findByText("Tag key is required"); + + // Then + expect(input).toHaveAttribute("aria-invalid", "true"); + expect(errorMessage).toHaveAttribute("id"); + expect(input).toHaveAttribute( + "aria-describedby", + expect.stringContaining(errorMessage.getAttribute("id") ?? ""), + ); + }); }); diff --git a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-parameters-form.tsx b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-parameters-form.tsx index 24937c0b7b..94edbf2a68 100644 --- a/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-parameters-form.tsx +++ b/ui/app/(prowler)/attack-paths/(workflow)/query-builder/_components/query-parameters-form.tsx @@ -1,9 +1,22 @@ "use client"; -import { Controller, useFormContext } from "react-hook-form"; +import { useFormContext } from "react-hook-form"; -import { Input } from "@/components/shadcn"; -import type { AttackPathQuery } from "@/types/attack-paths"; +import { Input, Textarea } from "@/components/shadcn"; +import { + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { cn } from "@/lib/utils"; +import { + type AttackPathQuery, + QUERY_PARAMETER_INPUT_TYPES, +} from "@/types/attack-paths"; + +import { QueryCodeEditor } from "./query-code-editor"; interface QueryParametersFormProps { selectedQuery: AttackPathQuery | null | undefined; @@ -16,10 +29,7 @@ interface QueryParametersFormProps { export const QueryParametersForm = ({ selectedQuery, }: QueryParametersFormProps) => { - const { - control, - formState: { errors }, - } = useFormContext(); + const { control } = useFormContext(); if (!selectedQuery || !selectedQuery.attributes.parameters.length) { return null; @@ -36,23 +46,26 @@ export const QueryParametersForm = ({ className="grid grid-cols-1 gap-4 md:grid-cols-2" > {selectedQuery.attributes.parameters.map((param) => ( - { + render={({ field, fieldState }) => { if (param.data_type === "boolean") { return ( -
+ -
+ + ); } - const errorMessage = (() => { - const error = errors[param.name]; - if (error && typeof error.message === "string") { - return error.message; - } - return undefined; - })(); + const placeholder = + param.description || + param.placeholder || + `Enter ${param.label.toLowerCase()}`; + + const isTextarea = + param.input_type === QUERY_PARAMETER_INPUT_TYPES.TEXTAREA; + const isCodeEditor = + param.input_type === QUERY_PARAMETER_INPUT_TYPES.CODE_EDITOR; return ( -
- - - {errorMessage && ( - {errorMessage} + + > + {!isCodeEditor && ( + + {param.label} + {param.required && ( + * + )} + + )} + {isCodeEditor ? ( + + + + ) : ( + + {isTextarea ? ( +