Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b82d7318ce | |||
| fdff780624 |
@@ -145,7 +145,7 @@ SENTRY_RELEASE=local
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.25.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.16.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -13,15 +13,11 @@ inputs:
|
||||
poetry-version:
|
||||
description: 'Poetry version to install'
|
||||
required: false
|
||||
default: '2.3.4'
|
||||
default: '2.1.1'
|
||||
install-dependencies:
|
||||
description: 'Install Python dependencies with Poetry'
|
||||
required: false
|
||||
default: 'true'
|
||||
update-lock:
|
||||
description: 'Run `poetry lock` during setup. Only enable when a prior step mutates pyproject.toml (e.g. API `@master` VCS rewrite). Default: false.'
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
@@ -78,7 +74,7 @@ runs:
|
||||
grep -A2 -B2 "resolved_reference" poetry.lock
|
||||
|
||||
- name: Update poetry.lock (prowler repo only)
|
||||
if: github.repository == 'prowler-cloud/prowler' && inputs.update-lock == 'true'
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: poetry lock
|
||||
|
||||
@@ -66,18 +66,6 @@ updates:
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: "pre-commit"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "pre-commit"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/04/15
|
||||
# v4.6
|
||||
# - package-ecosystem: "pip"
|
||||
|
||||
@@ -13,8 +13,6 @@ env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -17,8 +17,6 @@ concurrency:
|
||||
env:
|
||||
API_WORKING_DIR: ./api
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
api-code-quality:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -69,7 +67,6 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
working-directory: ./api
|
||||
update-lock: 'true'
|
||||
|
||||
- name: Poetry check
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -24,8 +24,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
api-analyze:
|
||||
name: CodeQL Security Analysis
|
||||
|
||||
@@ -33,8 +33,6 @@ env:
|
||||
PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud
|
||||
PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-api
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -18,8 +18,6 @@ env:
|
||||
API_WORKING_DIR: ./api
|
||||
IMAGE_NAME: prowler-api
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
api-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -17,8 +17,6 @@ concurrency:
|
||||
env:
|
||||
API_WORKING_DIR: ./api
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
api-security-scans:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -72,7 +70,6 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
working-directory: ./api
|
||||
update-lock: 'true'
|
||||
|
||||
- name: Bandit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -80,10 +77,9 @@ jobs:
|
||||
|
||||
- name: Safety
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run safety check --ignore 79023,79027,86217,71600
|
||||
run: poetry run safety check --ignore 79023,79027,86217
|
||||
# TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0
|
||||
# TODO: 86217 because `alibabacloud-tea-openapi == 0.4.3` don't let us upgrade `cryptography >= 46.0.0`
|
||||
# TODO: 71600 CVE-2024-1135 false positive - fixed in gunicorn 22.0.0, project uses 23.0.0
|
||||
|
||||
- name: Vulture
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -30,8 +30,6 @@ env:
|
||||
VALKEY_DB: 0
|
||||
API_WORKING_DIR: ./api
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
api-tests:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -118,7 +116,6 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
working-directory: ./api
|
||||
update-lock: 'true'
|
||||
|
||||
- name: Run tests with pytest
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -17,8 +17,6 @@ env:
|
||||
BACKPORT_LABEL_PREFIX: backport-to-
|
||||
BACKPORT_LABEL_IGNORE: was-backported
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
backport:
|
||||
if: github.event.pull_request.merged == true && !(contains(github.event.pull_request.labels.*.name, 'backport')) && !(contains(github.event.pull_request.labels.*.name, 'was-backported'))
|
||||
|
||||
@@ -21,8 +21,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
zizmor:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -9,8 +9,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.issue.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
update-labels:
|
||||
if: contains(github.event.issue.labels.*.name, 'status/awaiting-response')
|
||||
|
||||
@@ -16,8 +16,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
conventional-commit-check:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -13,8 +13,6 @@ env:
|
||||
BACKPORT_LABEL_PREFIX: backport-to-
|
||||
BACKPORT_LABEL_COLOR: B60205
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
create-label:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -13,8 +13,6 @@ env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -14,8 +14,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
scan-secrets:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -21,8 +21,6 @@ concurrency:
|
||||
env:
|
||||
CHART_PATH: contrib/k8s/helm/prowler-app
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
helm-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -13,8 +13,6 @@ concurrency:
|
||||
env:
|
||||
CHART_PATH: contrib/k8s/helm/prowler-app
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
release-helm-chart:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -9,8 +9,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.issue.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
lock:
|
||||
if: |
|
||||
|
||||
@@ -15,8 +15,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
labeler:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -32,8 +32,6 @@ env:
|
||||
PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud
|
||||
PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-mcp
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -18,8 +18,6 @@ env:
|
||||
MCP_WORKING_DIR: ./mcp_server
|
||||
IMAGE_NAME: prowler-mcp
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
mcp-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
@@ -89,9 +87,6 @@ jobs:
|
||||
api.github.com:443
|
||||
mirror.gcr.io:443
|
||||
check.trivy.dev:443
|
||||
get.trivy.dev:443
|
||||
release-assets.githubusercontent.com:443
|
||||
objects.githubusercontent.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -14,8 +14,6 @@ env:
|
||||
PYTHON_VERSION: "3.12"
|
||||
WORKING_DIRECTORY: ./mcp_server
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
validate-release:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -16,8 +16,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-changelog:
|
||||
if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false
|
||||
|
||||
@@ -16,8 +16,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-compliance-mapping:
|
||||
if: contains(github.event.pull_request.labels.*.name, 'no-compliance-check') == false
|
||||
|
||||
@@ -15,8 +15,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-conflicts:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -12,8 +12,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
trigger-cloud-pull-request:
|
||||
if: |
|
||||
|
||||
@@ -17,8 +17,6 @@ concurrency:
|
||||
env:
|
||||
PROWLER_VERSION: ${{ inputs.prowler_version }}
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
prepare-release:
|
||||
if: github.event_name == 'workflow_dispatch' && github.repository == 'prowler-cloud/prowler'
|
||||
@@ -40,11 +38,15 @@ jobs:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
install-dependencies: 'false'
|
||||
|
||||
- name: Install Poetry
|
||||
run: |
|
||||
python3 -m pip install --user poetry==2.1.1
|
||||
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
|
||||
@@ -13,8 +13,6 @@ env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -10,8 +10,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-duplicate-test-names:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -14,8 +14,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
sdk-code-quality:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
@@ -71,11 +69,22 @@ jobs:
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Install Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
run: pipx install poetry==2.1.1
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'poetry'
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: |
|
||||
poetry install --no-root
|
||||
poetry run pip list
|
||||
|
||||
- name: Check Poetry lock file
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -30,8 +30,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
sdk-analyze:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -47,8 +47,6 @@ env:
|
||||
# AWS configuration (for ECR)
|
||||
AWS_REGION: us-east-1
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
@@ -76,14 +74,15 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-dependencies: 'false'
|
||||
|
||||
- name: Inject poetry-bumpversion plugin
|
||||
run: pipx inject poetry poetry-bumpversion
|
||||
- name: Install Poetry
|
||||
run: |
|
||||
pipx install poetry==2.1.1
|
||||
pipx inject poetry poetry-bumpversion
|
||||
|
||||
- name: Get Prowler version and set tags
|
||||
id: get-prowler-version
|
||||
|
||||
@@ -17,8 +17,6 @@ concurrency:
|
||||
env:
|
||||
IMAGE_NAME: prowler
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
sdk-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
@@ -87,7 +85,6 @@ jobs:
|
||||
check.trivy.dev:443
|
||||
debian.map.fastlydns.net:80
|
||||
release-assets.githubusercontent.com:443
|
||||
objects.githubusercontent.com:443
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
www.powershellgallery.com:443
|
||||
|
||||
@@ -13,8 +13,6 @@ env:
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
||||
PYTHON_VERSION: '3.12'
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
validate-release:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
@@ -75,11 +73,13 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Install Poetry
|
||||
run: pipx install poetry==2.1.1
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-dependencies: 'false'
|
||||
|
||||
- name: Build Prowler package
|
||||
run: poetry build
|
||||
@@ -111,11 +111,13 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
- name: Install Poetry
|
||||
run: pipx install poetry==2.1.1
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-dependencies: 'false'
|
||||
|
||||
- name: Install toml package
|
||||
run: pip install toml
|
||||
|
||||
@@ -13,8 +13,6 @@ env:
|
||||
PYTHON_VERSION: '3.12'
|
||||
AWS_REGION: 'us-east-1'
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
refresh-aws-regions:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -12,8 +12,6 @@ concurrency:
|
||||
env:
|
||||
PYTHON_VERSION: '3.12'
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
refresh-oci-regions:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -14,8 +14,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
sdk-security-scans:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
@@ -71,11 +69,20 @@ jobs:
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Install Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
run: pipx install poetry==2.1.1
|
||||
|
||||
- name: Set up Python 3.12
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
cache: 'poetry'
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry install --no-root
|
||||
|
||||
- name: Security scan with Bandit
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -14,8 +14,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
sdk-tests:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
@@ -92,11 +90,20 @@ jobs:
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
- name: Install Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-python-poetry
|
||||
run: pipx install poetry==2.1.1
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'poetry'
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry install --no-root
|
||||
|
||||
# AWS Provider
|
||||
- name: Check if AWS files changed
|
||||
|
||||
@@ -31,8 +31,6 @@ on:
|
||||
description: "Whether there are UI E2E tests to run"
|
||||
value: ${{ jobs.analyze.outputs.has-ui-e2e }}
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -13,8 +13,6 @@ env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -26,8 +26,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
ui-analyze:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -35,8 +35,6 @@ env:
|
||||
# Build args
|
||||
NEXT_PUBLIC_API_BASE_URL: http://prowler-api:8080/api/v1
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
|
||||
@@ -18,8 +18,6 @@ env:
|
||||
UI_WORKING_DIR: ./ui
|
||||
IMAGE_NAME: prowler-ui
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
ui-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
@@ -91,8 +89,6 @@ jobs:
|
||||
mirror.gcr.io:443
|
||||
check.trivy.dev:443
|
||||
get.trivy.dev:443
|
||||
release-assets.githubusercontent.com:443
|
||||
objects.githubusercontent.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -15,8 +15,6 @@ on:
|
||||
- 'ui/**'
|
||||
- 'api/**' # API changes can affect UI E2E
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
# First, analyze which tests need to run
|
||||
impact-analysis:
|
||||
|
||||
@@ -18,8 +18,6 @@ env:
|
||||
UI_WORKING_DIR: ./ui
|
||||
NODE_VERSION: '24.13.0'
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
ui-tests:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -84,7 +84,6 @@ continue.json
|
||||
.continuerc.json
|
||||
|
||||
# AI Coding Assistants - OpenCode
|
||||
.opencode/
|
||||
opencode.json
|
||||
|
||||
# AI Coding Assistants - GitHub Copilot
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
repos:
|
||||
## GENERAL
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
rev: v4.6.0
|
||||
hooks:
|
||||
- id: check-merge-conflict
|
||||
- id: check-yaml
|
||||
@@ -16,7 +16,7 @@ repos:
|
||||
|
||||
## TOML
|
||||
- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
|
||||
rev: v2.16.0
|
||||
rev: v2.13.0
|
||||
hooks:
|
||||
- id: pretty-format-toml
|
||||
args: [--autofix]
|
||||
@@ -24,21 +24,21 @@ repos:
|
||||
|
||||
## GITHUB ACTIONS
|
||||
- repo: https://github.com/zizmorcore/zizmor-pre-commit
|
||||
rev: v1.24.1
|
||||
rev: v1.6.0
|
||||
hooks:
|
||||
- id: zizmor
|
||||
files: ^\.github/
|
||||
|
||||
## BASH
|
||||
- repo: https://github.com/koalaman/shellcheck-precommit
|
||||
rev: v0.11.0
|
||||
rev: v0.10.0
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
exclude: contrib
|
||||
|
||||
## PYTHON
|
||||
- repo: https://github.com/myint/autoflake
|
||||
rev: v2.3.3
|
||||
rev: v2.3.1
|
||||
hooks:
|
||||
- id: autoflake
|
||||
exclude: ^skills/
|
||||
@@ -50,27 +50,27 @@ repos:
|
||||
]
|
||||
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 8.0.1
|
||||
rev: 5.13.2
|
||||
hooks:
|
||||
- id: isort
|
||||
exclude: ^skills/
|
||||
args: ["--profile", "black"]
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 26.3.1
|
||||
rev: 24.4.2
|
||||
hooks:
|
||||
- id: black
|
||||
exclude: ^skills/
|
||||
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
rev: 7.3.0
|
||||
rev: 7.0.0
|
||||
hooks:
|
||||
- id: flake8
|
||||
exclude: (contrib|^skills/)
|
||||
args: ["--ignore=E266,W503,E203,E501,W605"]
|
||||
|
||||
- repo: https://github.com/python-poetry/poetry
|
||||
rev: 2.3.4
|
||||
rev: 2.1.1
|
||||
hooks:
|
||||
- id: poetry-check
|
||||
name: API - poetry-check
|
||||
@@ -93,7 +93,7 @@ repos:
|
||||
pass_filenames: false
|
||||
|
||||
- repo: https://github.com/hadolint/hadolint
|
||||
rev: v2.14.0
|
||||
rev: v2.13.0-beta
|
||||
hooks:
|
||||
- id: hadolint
|
||||
args: ["--ignore=DL3013"]
|
||||
@@ -128,8 +128,7 @@ repos:
|
||||
# TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X
|
||||
# TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0
|
||||
# TODO: 86217 because `alibabacloud-tea-openapi == 0.4.3` don't let us upgrade `cryptography >= 46.0.0`
|
||||
# TODO: 71600 CVE-2024-1135 false positive - fixed in gunicorn 22.0.0, project uses 23.0.0
|
||||
entry: bash -c 'safety check --ignore 70612,66963,74429,76352,76353,77744,77745,79023,79027,86217,71600'
|
||||
entry: bash -c 'safety check --ignore 70612,66963,74429,76352,76353,77744,77745,79023,79027,86217'
|
||||
language: system
|
||||
|
||||
- id: vulture
|
||||
|
||||
@@ -13,7 +13,7 @@ build:
|
||||
post_create_environment:
|
||||
# Install poetry
|
||||
# https://python-poetry.org/docs/#installing-manually
|
||||
- python -m pip install poetry==2.3.4
|
||||
- python -m pip install poetry
|
||||
post_install:
|
||||
# Install dependencies with 'docs' dependency group
|
||||
# https://python-poetry.org/docs/managing-dependencies/#dependency-groups
|
||||
|
||||
@@ -140,7 +140,7 @@ Prowler is an open-source cloud security assessment tool supporting AWS, Azure,
|
||||
|
||||
| Component | Location | Tech Stack |
|
||||
|-----------|----------|------------|
|
||||
| SDK | `prowler/` | Python 3.10+, Poetry 2.3+ |
|
||||
| SDK | `prowler/` | Python 3.10+, Poetry |
|
||||
| API | `api/` | Django 5.1, DRF, Celery |
|
||||
| UI | `ui/` | Next.js 15, React 19, Tailwind 4 |
|
||||
| MCP Server | `mcp_server/` | FastMCP, Python 3.12+ |
|
||||
@@ -153,12 +153,12 @@ Prowler is an open-source cloud security assessment tool supporting AWS, Azure,
|
||||
```bash
|
||||
# Setup
|
||||
poetry install --with dev
|
||||
poetry run prek install
|
||||
poetry run pre-commit install
|
||||
|
||||
# Code quality
|
||||
poetry run make lint
|
||||
poetry run make format
|
||||
poetry run prek run --all-files
|
||||
poetry run pre-commit run --all-files
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -68,7 +68,7 @@ ENV HOME='/home/prowler'
|
||||
ENV PATH="${HOME}/.local/bin:${PATH}"
|
||||
#hadolint ignore=DL3013
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir poetry==2.3.4
|
||||
pip install --no-cache-dir poetry
|
||||
|
||||
RUN poetry install --compile && \
|
||||
rm -rf ~/.cache/pip
|
||||
|
||||
@@ -246,7 +246,14 @@ Some pre-commit hooks require tools installed on your system:
|
||||
|
||||
1. **Install [TruffleHog](https://github.com/trufflesecurity/trufflehog#install)** (secret scanning) — see the [official installation options](https://github.com/trufflesecurity/trufflehog#install).
|
||||
|
||||
2. **Install [Hadolint](https://github.com/hadolint/hadolint#install)** (Dockerfile linting) — see the [official installation options](https://github.com/hadolint/hadolint#install).
|
||||
2. **Install [Safety](https://github.com/pyupio/safety)** (dependency vulnerability checking):
|
||||
|
||||
```console
|
||||
# Requires a Python environment (e.g. via pyenv)
|
||||
pip install safety
|
||||
```
|
||||
|
||||
3. **Install [Hadolint](https://github.com/hadolint/hadolint#install)** (Dockerfile linting) — see the [official installation options](https://github.com/hadolint/hadolint#install).
|
||||
|
||||
## Prowler CLI
|
||||
### Pip package
|
||||
|
||||
@@ -2,55 +2,24 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.25.1] (Prowler v5.24.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: Missing `tenant_id` filter while getting related findings after scan completes [(#10722)](https://github.com/prowler-cloud/prowler/pull/10722)
|
||||
- Finding group counters `pass_count`, `fail_count` and `manual_count` now exclude muted findings [(#10753)](https://github.com/prowler-cloud/prowler/pull/10753)
|
||||
|
||||
---
|
||||
|
||||
## [1.25.0] (Prowler v5.24.0)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Bump Poetry to `2.3.4` in Dockerfile and pre-commit hooks. Regenerate `api/poetry.lock` [(#10681)](https://github.com/prowler-cloud/prowler/pull/10681)
|
||||
- Attack Paths: Remove dead `cleanup_findings` no-op and its supporting `prowler_finding_lastupdated` index [(#10684)](https://github.com/prowler-cloud/prowler/pull/10684)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Worker-beat race condition on cold start: replaced `sleep 15` with API service healthcheck dependency (Docker Compose) and init containers (Helm), aligned Gunicorn default port to `8080` [(#10603)](https://github.com/prowler-cloud/prowler/pull/10603)
|
||||
- API container startup crash on Linux due to root-owned bind-mount preventing JWT key generation [(#10646)](https://github.com/prowler-cloud/prowler/pull/10646)
|
||||
- Finding group resources endpoints now include findings without associated resources (orphan IaC findings) as simulated resource rows, and return one row per finding when multiple findings share a resource [(#10708)](https://github.com/prowler-cloud/prowler/pull/10708)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `pytest` from 8.2.2 to 9.0.3 to fix CVE-2025-71176 [(#10678)](https://github.com/prowler-cloud/prowler/pull/10678)
|
||||
|
||||
---
|
||||
|
||||
## [1.24.0] (Prowler v5.23.0)
|
||||
## [1.24.0] (Prowler UNRELEASED)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- RBAC role lookup filtered by `tenant_id` to prevent cross-tenant privilege leak [(#10491)](https://github.com/prowler-cloud/prowler/pull/10491)
|
||||
- Pin all unpinned dependencies to exact versions to prevent supply chain attacks and ensure reproducible builds [(#10469)](https://github.com/prowler-cloud/prowler/pull/10469)
|
||||
- Filter RBAC role lookup by `tenant_id` to prevent cross-tenant privilege leak [(#10491)](https://github.com/prowler-cloud/prowler/pull/10491)
|
||||
- `VALKEY_SCHEME`, `VALKEY_USERNAME`, and `VALKEY_PASSWORD` environment variables to configure Celery broker TLS/auth connection details for Valkey/ElastiCache [(#10420)](https://github.com/prowler-cloud/prowler/pull/10420)
|
||||
- `Vercel` provider support [(#10190)](https://github.com/prowler-cloud/prowler/pull/10190)
|
||||
- Finding groups list and latest endpoints support `sort=delta`, ordering by `new_count` then `changed_count` so groups with the most new findings rank highest [(#10606)](https://github.com/prowler-cloud/prowler/pull/10606)
|
||||
- Finding group resources endpoints (`/finding-groups/{check_id}/resources` and `/finding-groups/latest/{check_id}/resources`) now expose `finding_id` per row, pointing to the most recent matching Finding for each resource. UUIDv7 ordering guarantees `Max(finding__id)` resolves to the latest snapshot [(#10630)](https://github.com/prowler-cloud/prowler/pull/10630)
|
||||
- Handle CIS and CISA SCuBA compliance framework from google workspace [(#10629)](https://github.com/prowler-cloud/prowler/pull/10629)
|
||||
- Sort support for all finding group counter fields: `pass_muted_count`, `fail_muted_count`, `manual_muted_count`, and all `new_*`/`changed_*` status-mute breakdown counters [(#10655)](https://github.com/prowler-cloud/prowler/pull/10655)
|
||||
- Finding groups list and latest endpoints support `sort=status`, ordering by aggregated status with the FAIL > PASS > MUTED priority [(#10628)](https://github.com/prowler-cloud/prowler/pull/10628)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Finding groups list/latest/resources now expose `status` ∈ `{FAIL, PASS, MANUAL}` and `muted: bool` as orthogonal fields. The aggregated `status` reflects the underlying check outcome regardless of mute state, and `muted=true` signals that every finding in the group/resource is muted. New `manual_count` is exposed alongside `pass_count`/`fail_count`, plus `pass_muted_count`/`fail_muted_count`/`manual_muted_count` siblings so clients can isolate the muted half of each status. The `new_*`/`changed_*` deltas are now broken down by status and mute state via 12 new counters (`new_fail_count`, `new_fail_muted_count`, `new_pass_count`, `new_pass_muted_count`, `new_manual_count`, `new_manual_muted_count` and the matching `changed_*` set). New `filter[muted]=true|false` and `sort=status` (FAIL > PASS > MANUAL) / `sort=muted` are supported. `filter[status]=MUTED` is no longer accepted [(#10630)](https://github.com/prowler-cloud/prowler/pull/10630)
|
||||
- Attack Paths: Periodic cleanup of stale scans with dead-worker detection via Celery inspect, marking orphaned `EXECUTING` scans as `FAILED` and recovering `graph_data_ready` [(#10387)](https://github.com/prowler-cloud/prowler/pull/10387)
|
||||
- Attack Paths: Replace `_provider_id` property with `_Provider_{uuid}` label for provider isolation, add regex-based label injection for custom queries [(#10402)](https://github.com/prowler-cloud/prowler/pull/10402)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `reaggregate_all_finding_group_summaries_task` now refreshes finding group daily summaries for every `(provider, day)` combination instead of only the latest scan per provider, matching the unbounded scope of `mute_historical_findings_task`. Mute rule operations no longer leave older daily summaries drifting from the underlying muted findings [(#10630)](https://github.com/prowler-cloud/prowler/pull/10630)
|
||||
- Finding groups list/latest now apply computed status/severity filters and finding-level prefilters (delta, region, service, category, resource group, scan, resource type), plus `check_title` support for sort/filter consistency [(#10428)](https://github.com/prowler-cloud/prowler/pull/10428)
|
||||
- Populate compliance data inside `check_metadata` for findings, which was always returned as `null` [(#10449)](https://github.com/prowler-cloud/prowler/pull/10449)
|
||||
- 403 error for admin users listing tenants due to roles query not using the admin database connection [(#10460)](https://github.com/prowler-cloud/prowler/pull/10460)
|
||||
|
||||
@@ -71,7 +71,7 @@ RUN mkdir -p /tmp/prowler_api_output
|
||||
COPY pyproject.toml ./
|
||||
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir poetry==2.3.4
|
||||
pip install --no-cache-dir poetry
|
||||
|
||||
ENV PATH="/home/prowler/.local/bin:$PATH"
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ start_worker() {
|
||||
|
||||
start_worker_beat() {
|
||||
echo "Starting the worker-beat..."
|
||||
sleep 15
|
||||
poetry run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand.
|
||||
# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand.
|
||||
|
||||
[[package]]
|
||||
name = "about-time"
|
||||
@@ -2961,7 +2961,7 @@ files = [
|
||||
[package.dependencies]
|
||||
autopep8 = "*"
|
||||
Django = ">=4.2"
|
||||
gprof2dot = ">=2017.9.19"
|
||||
gprof2dot = ">=2017.09.19"
|
||||
sqlparse = "*"
|
||||
|
||||
[[package]]
|
||||
@@ -4569,7 +4569,7 @@ files = [
|
||||
|
||||
[package.dependencies]
|
||||
attrs = ">=22.2.0"
|
||||
jsonschema-specifications = ">=2023.3.6"
|
||||
jsonschema-specifications = ">=2023.03.6"
|
||||
referencing = ">=0.28.4"
|
||||
rpds-py = ">=0.7.1"
|
||||
|
||||
@@ -4777,7 +4777,7 @@ librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""]
|
||||
mongodb = ["pymongo (==4.15.3)"]
|
||||
msgpack = ["msgpack (==1.1.2)"]
|
||||
pyro = ["pyro4 (==4.82)"]
|
||||
qpid = ["qpid-python (==1.36.0.post1)", "qpid-tools (==1.36.0.post1)"]
|
||||
qpid = ["qpid-python (==1.36.0-1)", "qpid-tools (==1.36.0-1)"]
|
||||
redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<6.5)"]
|
||||
slmq = ["softlayer_messaging (>=1.0.3)"]
|
||||
sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"]
|
||||
@@ -4798,7 +4798,7 @@ files = [
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
certifi = ">=14.5.14"
|
||||
certifi = ">=14.05.14"
|
||||
durationpy = ">=0.7"
|
||||
google-auth = ">=1.0.1"
|
||||
oauthlib = ">=3.2.2"
|
||||
@@ -6445,33 +6445,6 @@ docs = ["sphinx (>=1.7.1)"]
|
||||
redis = ["redis"]
|
||||
tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)", "types-redis"]
|
||||
|
||||
[[package]]
|
||||
name = "prek"
|
||||
version = "0.3.9"
|
||||
description = "A Git hook manager written in Rust, designed as a drop-in alternative to pre-commit."
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["dev"]
|
||||
files = [
|
||||
{file = "prek-0.3.9-py3-none-linux_armv6l.whl", hash = "sha256:3ed793d51bfaa27bddb64d525d7acb77a7c8644f549412d82252e3eb0b88aad8"},
|
||||
{file = "prek-0.3.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:399c58400c0bd0b82a93a3c09dc1bfd88d8d0cfb242d414d2ed247187b06ead1"},
|
||||
{file = "prek-0.3.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ea1ffb124e92f081b8e2ca5b5a623a733efb3be0c5b1f4b7ffe2ee17d1f20c"},
|
||||
{file = "prek-0.3.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:aaf639f95b7301639298311d8d44aad0d0b4864e9736083ad3c71ce9765d37ab"},
|
||||
{file = "prek-0.3.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff104863b187fa443ea8451ca55d51e2c6e94f99f00d88784b5c3c4c623f1ebe"},
|
||||
{file = "prek-0.3.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:039ecaf87c63a3e67cca645ebd5bc5eb6aafa6c9d929e9a27b2921e7849d7ef9"},
|
||||
{file = "prek-0.3.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3bde2a3d045705095983c7f78ba04f72a7565fe1c2b4e85f5628502a254754ff"},
|
||||
{file = "prek-0.3.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28a0960a21543563e2c8e19aaad176cc8423a87aac3c914d0f313030d7a9244a"},
|
||||
{file = "prek-0.3.9-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:0dfb5d5171d7523271909246ee306b4dc3d5b63752e7dd7c7e8a8908fc9490d1"},
|
||||
{file = "prek-0.3.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82b791bd36c1430c84d3ae7220a85152babc7eaf00f70adcb961bd594e756ba3"},
|
||||
{file = "prek-0.3.9-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:6eac6d2f736b041118f053a1487abed468a70dd85a8688eaf87bb42d3dcecf20"},
|
||||
{file = "prek-0.3.9-py3-none-musllinux_1_1_i686.whl", hash = "sha256:5517e46e761367a3759b3168eabc120840ffbca9dfbc53187167298a98f87dc4"},
|
||||
{file = "prek-0.3.9-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:92024778cf78683ca32687bb249ab6a7d5c33887b5ee1d1a9f6d0c14228f4cf3"},
|
||||
{file = "prek-0.3.9-py3-none-win32.whl", hash = "sha256:7f89c55e5f480f5d073769e319924ad69d4bf9f98c5cb46a83082e26e634c958"},
|
||||
{file = "prek-0.3.9-py3-none-win_amd64.whl", hash = "sha256:7722f3372eaa83b147e70a43cb7b9fe2128c13d0c78d8a1cdbf2a8ec2ee071eb"},
|
||||
{file = "prek-0.3.9-py3-none-win_arm64.whl", hash = "sha256:0bced6278d6cc8a4b46048979e36bc9da034611dc8facd77ab123177b833a929"},
|
||||
{file = "prek-0.3.9.tar.gz", hash = "sha256:f82b92d81f42f1f90a47f5fbbf492373e25ef1f790080215b2722dd6da66510e"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prompt-toolkit"
|
||||
version = "3.0.52"
|
||||
@@ -7156,14 +7129,14 @@ windows-terminal = ["colorama (>=0.4.6)"]
|
||||
|
||||
[[package]]
|
||||
name = "pyjwt"
|
||||
version = "2.12.1"
|
||||
version = "2.11.0"
|
||||
description = "JSON Web Token implementation in Python"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c"},
|
||||
{file = "pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b"},
|
||||
{file = "pyjwt-2.11.0-py3-none-any.whl", hash = "sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469"},
|
||||
{file = "pyjwt-2.11.0.tar.gz", hash = "sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@@ -7188,7 +7161,7 @@ files = [
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
astroid = ">=3.2.2,<=3.3.0.dev0"
|
||||
astroid = ">=3.2.2,<=3.3.0-dev0"
|
||||
colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""}
|
||||
dill = [
|
||||
{version = ">=0.3.7", markers = "python_version >= \"3.12\""},
|
||||
@@ -7351,25 +7324,24 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "9.0.3"
|
||||
version = "8.2.2"
|
||||
description = "pytest: simple powerful testing with Python"
|
||||
optional = false
|
||||
python-versions = ">=3.10"
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main", "dev"]
|
||||
files = [
|
||||
{file = "pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9"},
|
||||
{file = "pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c"},
|
||||
{file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"},
|
||||
{file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""}
|
||||
iniconfig = ">=1.0.1"
|
||||
packaging = ">=22"
|
||||
pluggy = ">=1.5,<2"
|
||||
pygments = ">=2.7.2"
|
||||
colorama = {version = "*", markers = "sys_platform == \"win32\""}
|
||||
iniconfig = "*"
|
||||
packaging = "*"
|
||||
pluggy = ">=1.5,<2.0"
|
||||
|
||||
[package.extras]
|
||||
dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"]
|
||||
dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
|
||||
|
||||
[[package]]
|
||||
name = "pytest-celery"
|
||||
@@ -8202,10 +8174,10 @@ files = [
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
botocore = ">=1.37.4,<2.0a0"
|
||||
botocore = ">=1.37.4,<2.0a.0"
|
||||
|
||||
[package.extras]
|
||||
crt = ["botocore[crt] (>=1.37.4,<2.0a0)"]
|
||||
crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "safety"
|
||||
@@ -9400,4 +9372,4 @@ files = [
|
||||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.11,<3.13"
|
||||
content-hash = "077e89853cfe3a6d934841488cfa5a98ff6c92b71f74b817b71387d11559f143"
|
||||
content-hash = "167d4549788b8bc8bb7772b9a81ade1eab73d8f354251a8d6af4901223cc7f67"
|
||||
|
||||
@@ -50,7 +50,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.26.0"
|
||||
version = "1.24.0"
|
||||
|
||||
[project.scripts]
|
||||
celery = "src.backend.config.settings.celery"
|
||||
@@ -65,7 +65,7 @@ freezegun = "1.5.1"
|
||||
marshmallow = "==3.26.2"
|
||||
mypy = "1.10.1"
|
||||
pylint = "3.2.5"
|
||||
pytest = "9.0.3"
|
||||
pytest = "8.2.2"
|
||||
pytest-cov = "5.0.0"
|
||||
pytest-django = "4.8.0"
|
||||
pytest-env = "1.1.3"
|
||||
@@ -75,4 +75,3 @@ ruff = "0.5.0"
|
||||
safety = "3.7.0"
|
||||
tqdm = "4.67.1"
|
||||
vulture = "2.14"
|
||||
prek = "0.3.9"
|
||||
|
||||
@@ -1115,14 +1115,13 @@ class FindingGroupAggregatedComputedFilter(FilterSet):
|
||||
STATUS_CHOICES = (
|
||||
("FAIL", "Fail"),
|
||||
("PASS", "Pass"),
|
||||
("MANUAL", "Manual"),
|
||||
("MUTED", "Muted"),
|
||||
)
|
||||
|
||||
status = ChoiceFilter(method="filter_status", choices=STATUS_CHOICES)
|
||||
status__in = CharInFilter(method="filter_status_in", lookup_expr="in")
|
||||
severity = ChoiceFilter(method="filter_severity", choices=SeverityChoices)
|
||||
severity__in = CharInFilter(method="filter_severity_in", lookup_expr="in")
|
||||
muted = BooleanFilter(field_name="muted")
|
||||
include_muted = BooleanFilter(method="filter_include_muted")
|
||||
|
||||
def filter_status(self, queryset, name, value):
|
||||
@@ -1199,7 +1198,7 @@ class FindingGroupAggregatedComputedFilter(FilterSet):
|
||||
if value is True:
|
||||
return queryset
|
||||
# include_muted=false: exclude fully-muted groups
|
||||
return queryset.exclude(muted=True)
|
||||
return queryset.exclude(fail_count=0, pass_count=0, muted_count__gt=0)
|
||||
|
||||
|
||||
class ProviderSecretFilter(FilterSet):
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0087_vercel_provider"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="manual_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="pass_muted_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="fail_muted_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="manual_muted_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="muted",
|
||||
field=models.BooleanField(default=False),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="new_fail_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="new_fail_muted_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="new_pass_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="new_pass_muted_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="new_manual_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="new_manual_muted_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="changed_fail_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="changed_fail_muted_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="changed_pass_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="changed_pass_muted_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="changed_manual_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="findinggroupdailysummary",
|
||||
name="changed_manual_muted_count",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
]
|
||||
@@ -1,31 +0,0 @@
|
||||
from django.db import migrations
|
||||
from tasks.tasks import backfill_finding_group_summaries_task
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.rls import Tenant
|
||||
|
||||
|
||||
def trigger_backfill_task(apps, schema_editor):
|
||||
"""
|
||||
Re-dispatch the finding-group backfill task for every tenant so the new
|
||||
`manual_count` and `muted` columns added in 0088 get populated from the
|
||||
last 10 days of completed scans.
|
||||
|
||||
The aggregator (`aggregate_finding_group_summaries`) recomputes every
|
||||
column on each call, so it back-populates the new fields without touching
|
||||
the existing ones beyond a normal upsert.
|
||||
"""
|
||||
tenant_ids = Tenant.objects.using(MainRouter.admin_db).values_list("id", flat=True)
|
||||
|
||||
for tenant_id in tenant_ids:
|
||||
backfill_finding_group_summaries_task.delay(tenant_id=str(tenant_id), days=10)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0088_finding_group_status_muted_fields"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(trigger_backfill_task, migrations.RunPython.noop),
|
||||
]
|
||||
@@ -1748,45 +1748,15 @@ class FindingGroupDailySummary(RowLevelSecurityProtectedModel):
|
||||
# Severity stored as integer for MAX aggregation (5=critical, 4=high, etc.)
|
||||
severity_order = models.SmallIntegerField(default=1)
|
||||
|
||||
# Finding counts (inclusive of muted findings; use the `muted` flag to
|
||||
# tell whether the group has any actionable findings).
|
||||
# Finding counts
|
||||
pass_count = models.IntegerField(default=0)
|
||||
fail_count = models.IntegerField(default=0)
|
||||
manual_count = models.IntegerField(default=0)
|
||||
muted_count = models.IntegerField(default=0)
|
||||
|
||||
# Status counts restricted to muted findings, so clients can isolate the
|
||||
# muted half of each status (e.g. `pass_count - pass_muted_count` gives the
|
||||
# actionable PASS findings).
|
||||
pass_muted_count = models.IntegerField(default=0)
|
||||
fail_muted_count = models.IntegerField(default=0)
|
||||
manual_muted_count = models.IntegerField(default=0)
|
||||
|
||||
# Whether every finding for this (provider, check, day) is muted.
|
||||
muted = models.BooleanField(default=False)
|
||||
|
||||
# Delta counts (non-muted, kept for convenience and as a "total" view).
|
||||
# Delta counts
|
||||
new_count = models.IntegerField(default=0)
|
||||
changed_count = models.IntegerField(default=0)
|
||||
|
||||
# Delta breakdown by (status, muted) so clients can answer questions like
|
||||
# "how many new failing findings appeared in this scan?" without scanning
|
||||
# the underlying findings table. Mirrors the existing pass/fail/manual
|
||||
# naming, with `_muted_count` siblings tracking the muted half of each
|
||||
# bucket explicitly.
|
||||
new_fail_count = models.IntegerField(default=0)
|
||||
new_fail_muted_count = models.IntegerField(default=0)
|
||||
new_pass_count = models.IntegerField(default=0)
|
||||
new_pass_muted_count = models.IntegerField(default=0)
|
||||
new_manual_count = models.IntegerField(default=0)
|
||||
new_manual_muted_count = models.IntegerField(default=0)
|
||||
changed_fail_count = models.IntegerField(default=0)
|
||||
changed_fail_muted_count = models.IntegerField(default=0)
|
||||
changed_pass_count = models.IntegerField(default=0)
|
||||
changed_pass_muted_count = models.IntegerField(default=0)
|
||||
changed_manual_count = models.IntegerField(default=0)
|
||||
changed_manual_muted_count = models.IntegerField(default=0)
|
||||
|
||||
# Resource counts
|
||||
resources_fail = models.IntegerField(default=0)
|
||||
resources_total = models.IntegerField(default=0)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.26.0
|
||||
version: 1.24.0
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
|
||||
@@ -57,7 +57,6 @@ from api.models import (
|
||||
ProviderGroupMembership,
|
||||
ProviderSecret,
|
||||
Resource,
|
||||
ResourceFindingMapping,
|
||||
Role,
|
||||
RoleProviderGroupRelationship,
|
||||
SAMLConfiguration,
|
||||
@@ -15446,16 +15445,10 @@ class TestFindingGroupViewSet:
|
||||
# iam_password_policy has only PASS findings
|
||||
assert data[0]["attributes"]["status"] == "PASS"
|
||||
|
||||
def test_finding_groups_fully_muted_group_reflects_underlying_status(
|
||||
def test_finding_groups_status_muted_all(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
"""A fully-muted group still surfaces its underlying status (no MUTED).
|
||||
|
||||
rds_encryption has 2 muted FAIL findings, so the group must report
|
||||
status=FAIL (the orthogonal `muted` boolean signals it isn't actionable).
|
||||
The status×muted breakdown lets clients answer 'how many failing
|
||||
findings are muted in this group'.
|
||||
"""
|
||||
"""Test that MUTED status returned when all findings are muted."""
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-group-list"),
|
||||
{"filter[inserted_at]": TODAY, "filter[check_id]": "rds_encryption"},
|
||||
@@ -15463,21 +15456,8 @@ class TestFindingGroupViewSet:
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert len(data) == 1
|
||||
attrs = data[0]["attributes"]
|
||||
assert attrs["status"] == "FAIL"
|
||||
assert attrs["muted"] is True
|
||||
assert attrs["fail_count"] == 0
|
||||
assert attrs["fail_muted_count"] == 2
|
||||
assert attrs["pass_muted_count"] == 0
|
||||
assert attrs["manual_muted_count"] == 0
|
||||
assert attrs["muted_count"] == 2
|
||||
# Sanity: the per-status muted counts must add up to muted_count.
|
||||
assert (
|
||||
attrs["pass_muted_count"]
|
||||
+ attrs["fail_muted_count"]
|
||||
+ attrs["manual_muted_count"]
|
||||
== attrs["muted_count"]
|
||||
)
|
||||
# rds_encryption has all muted findings
|
||||
assert data[0]["attributes"]["status"] == "MUTED"
|
||||
|
||||
def test_finding_groups_status_filter(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
@@ -15969,7 +15949,7 @@ class TestFindingGroupViewSet:
|
||||
"extra_filters",
|
||||
[
|
||||
{},
|
||||
{"filter[delta]": "new"},
|
||||
{"filter[muted]": "include"},
|
||||
],
|
||||
ids=["summary_path", "finding_level_path"],
|
||||
)
|
||||
@@ -15987,8 +15967,7 @@ class TestFindingGroupViewSet:
|
||||
|
||||
Parametrized to cover both aggregation paths:
|
||||
- summary_path: default, uses _CheckTitleToCheckIdMixin on summaries
|
||||
- finding_level_path: filter[delta]=new forces _aggregate_findings via
|
||||
CommonFindingFilters (delta is finding-level, not summary-level)
|
||||
- finding_level_path: filter[muted]=include forces CommonFindingFilters
|
||||
"""
|
||||
params = {
|
||||
"filter[inserted_at]": TODAY,
|
||||
@@ -16031,36 +16010,6 @@ class TestFindingGroupViewSet:
|
||||
# s3_bucket_public_access has 2 findings with 2 different resources
|
||||
assert len(data) == 2
|
||||
|
||||
def test_resources_id_matches_resource_id_for_mapped_findings(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
"""Findings with a resource expose the resource id as row id (hot path contract)."""
|
||||
response = authenticated_client.get(
|
||||
reverse(
|
||||
"finding-group-resources", kwargs={"pk": "s3_bucket_public_access"}
|
||||
),
|
||||
{"filter[inserted_at]": TODAY},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert data, "expected resources in response"
|
||||
|
||||
resource_ids = set(
|
||||
ResourceFindingMapping.objects.filter(
|
||||
finding__check_id="s3_bucket_public_access",
|
||||
).values_list("resource_id", flat=True)
|
||||
)
|
||||
finding_ids = set(
|
||||
Finding.objects.filter(
|
||||
check_id="s3_bucket_public_access",
|
||||
).values_list("id", flat=True)
|
||||
)
|
||||
|
||||
returned_ids = {item["id"] for item in data}
|
||||
assert returned_ids <= {str(rid) for rid in resource_ids}
|
||||
assert returned_ids.isdisjoint({str(fid) for fid in finding_ids})
|
||||
|
||||
def test_resources_fields(self, authenticated_client, finding_groups_fixture):
|
||||
"""Test resource fields (uid, name, service, region, type) have valid values."""
|
||||
response = authenticated_client.get(
|
||||
@@ -16923,6 +16872,68 @@ class TestFindingGroupViewSet:
|
||||
asc_keys = [delta_key(item) for item in response.json()["data"]]
|
||||
assert asc_keys == sorted(asc_keys)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
)
|
||||
def test_finding_groups_sort_by_status(
|
||||
self,
|
||||
authenticated_client,
|
||||
finding_groups_fixture,
|
||||
endpoint_name,
|
||||
):
|
||||
"""Sort by status orders by aggregated status (FAIL > PASS > MUTED)."""
|
||||
status_order = {"FAIL": 3, "PASS": 2, "MUTED": 1}
|
||||
|
||||
# Descending: FAIL groups first, then PASS
|
||||
params = {"sort": "-status"}
|
||||
if endpoint_name == "finding-group-list":
|
||||
params["filter[inserted_at]"] = TODAY
|
||||
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert len(data) > 0
|
||||
|
||||
desc_statuses = [item["attributes"]["status"] for item in data]
|
||||
desc_keys = [status_order[s] for s in desc_statuses]
|
||||
assert desc_keys == sorted(desc_keys, reverse=True)
|
||||
|
||||
# Ascending: PASS groups first, then FAIL
|
||||
params["sort"] = "status"
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
asc_statuses = [
|
||||
item["attributes"]["status"] for item in response.json()["data"]
|
||||
]
|
||||
asc_keys = [status_order[s] for s in asc_statuses]
|
||||
assert asc_keys == sorted(asc_keys)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
)
|
||||
def test_finding_groups_sort_by_status_includes_muted(
|
||||
self,
|
||||
authenticated_client,
|
||||
finding_groups_fixture,
|
||||
endpoint_name,
|
||||
):
|
||||
"""When include_muted is set, MUTED groups participate in status sort."""
|
||||
status_order = {"FAIL": 3, "PASS": 2, "MUTED": 1}
|
||||
|
||||
params = {"sort": "status", "filter[include_muted]": "true"}
|
||||
if endpoint_name == "finding-group-list":
|
||||
params["filter[inserted_at]"] = TODAY
|
||||
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
|
||||
statuses = [item["attributes"]["status"] for item in data]
|
||||
assert "MUTED" in statuses
|
||||
assert statuses[0] == "MUTED"
|
||||
keys = [status_order[s] for s in statuses]
|
||||
assert keys == sorted(keys)
|
||||
|
||||
def test_finding_groups_latest_ignores_date_filters(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
@@ -16936,338 +16947,3 @@ class TestFindingGroupViewSet:
|
||||
data = response.json()["data"]
|
||||
# Should still return data, not filtered by the old date
|
||||
assert len(data) == 5
|
||||
|
||||
def test_finding_groups_status_choices_no_muted(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
"""Every returned group must have status ∈ {FAIL, PASS, MANUAL}."""
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-group-list"),
|
||||
{"filter[inserted_at]": TODAY},
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
statuses = {item["attributes"]["status"] for item in response.json()["data"]}
|
||||
assert statuses, "fixture should produce at least one group"
|
||||
assert statuses <= {"FAIL", "PASS", "MANUAL"}
|
||||
assert "MUTED" not in statuses
|
||||
|
||||
def test_finding_groups_serializer_exposes_muted_and_manual_count(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
"""The /finding-groups payload must expose `muted`, `manual_count` and
|
||||
the per-status muted siblings (`pass_muted_count`/`fail_muted_count`/
|
||||
`manual_muted_count`)."""
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-group-list"),
|
||||
{"filter[inserted_at]": TODAY, "filter[check_id]": "iam_password_policy"},
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
attrs = response.json()["data"][0]["attributes"]
|
||||
assert "muted" in attrs and isinstance(attrs["muted"], bool)
|
||||
assert "manual_count" in attrs and isinstance(attrs["manual_count"], int)
|
||||
assert attrs["muted"] is False # iam_password_policy has only non-muted PASS
|
||||
assert attrs["manual_count"] == 0
|
||||
assert attrs["pass_muted_count"] == 0
|
||||
assert attrs["fail_muted_count"] == 0
|
||||
assert attrs["manual_muted_count"] == 0
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
)
|
||||
def test_finding_groups_filter_status_muted_is_rejected(
|
||||
self, authenticated_client, finding_groups_fixture, endpoint_name
|
||||
):
|
||||
"""`filter[status]=MUTED` is no longer a valid status value."""
|
||||
params = {"filter[status]": "MUTED"}
|
||||
if endpoint_name == "finding-group-list":
|
||||
params["filter[inserted_at]"] = TODAY
|
||||
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
)
|
||||
def test_finding_groups_filter_muted_true(
|
||||
self, authenticated_client, finding_groups_fixture, endpoint_name
|
||||
):
|
||||
"""`filter[muted]=true` returns only fully-muted groups."""
|
||||
params = {"filter[muted]": "true"}
|
||||
if endpoint_name == "finding-group-list":
|
||||
params["filter[inserted_at]"] = TODAY
|
||||
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
check_ids = {item["id"] for item in data}
|
||||
# Only rds_encryption is fully muted in the fixture
|
||||
assert check_ids == {"rds_encryption"}
|
||||
assert all(item["attributes"]["muted"] is True for item in data)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
)
|
||||
def test_finding_groups_filter_muted_false(
|
||||
self, authenticated_client, finding_groups_fixture, endpoint_name
|
||||
):
|
||||
"""`filter[muted]=false` returns only groups with actionable findings."""
|
||||
params = {"filter[muted]": "false"}
|
||||
if endpoint_name == "finding-group-list":
|
||||
params["filter[inserted_at]"] = TODAY
|
||||
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
check_ids = {item["id"] for item in data}
|
||||
assert "rds_encryption" not in check_ids
|
||||
assert check_ids == {
|
||||
"s3_bucket_public_access",
|
||||
"ec2_instance_public_ip",
|
||||
"iam_password_policy",
|
||||
"cloudtrail_enabled",
|
||||
}
|
||||
assert all(item["attributes"]["muted"] is False for item in data)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
)
|
||||
def test_finding_groups_sort_by_status(
|
||||
self, authenticated_client, finding_groups_fixture, endpoint_name
|
||||
):
|
||||
"""sort=status orders by aggregated status (FAIL > PASS > MANUAL)."""
|
||||
priority = {"FAIL": 3, "PASS": 2, "MANUAL": 1}
|
||||
params = {"sort": "-status"}
|
||||
if endpoint_name == "finding-group-list":
|
||||
params["filter[inserted_at]"] = TODAY
|
||||
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert data, "fixture should produce groups"
|
||||
|
||||
desc_keys = [priority[item["attributes"]["status"]] for item in data]
|
||||
assert desc_keys == sorted(desc_keys, reverse=True)
|
||||
|
||||
params["sort"] = "status"
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
asc_keys = [
|
||||
priority[item["attributes"]["status"]] for item in response.json()["data"]
|
||||
]
|
||||
assert asc_keys == sorted(asc_keys)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
)
|
||||
def test_finding_groups_sort_by_muted(
|
||||
self, authenticated_client, finding_groups_fixture, endpoint_name
|
||||
):
|
||||
"""sort=muted orders by the boolean muted attribute."""
|
||||
# Need include_muted=true so the fully-muted group is part of the result
|
||||
params = {"sort": "-muted", "filter[include_muted]": "true"}
|
||||
if endpoint_name == "finding-group-list":
|
||||
params["filter[inserted_at]"] = TODAY
|
||||
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert data, "fixture should produce groups"
|
||||
|
||||
muted_values = [item["attributes"]["muted"] for item in data]
|
||||
# Descending boolean: True (1) before False (0)
|
||||
assert muted_values == sorted(muted_values, reverse=True)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"sort_field",
|
||||
[
|
||||
"pass_muted_count",
|
||||
"fail_muted_count",
|
||||
"manual_muted_count",
|
||||
"new_fail_count",
|
||||
"new_fail_muted_count",
|
||||
"new_pass_count",
|
||||
"new_pass_muted_count",
|
||||
"new_manual_count",
|
||||
"new_manual_muted_count",
|
||||
"changed_fail_count",
|
||||
"changed_fail_muted_count",
|
||||
"changed_pass_count",
|
||||
"changed_pass_muted_count",
|
||||
"changed_manual_count",
|
||||
"changed_manual_muted_count",
|
||||
],
|
||||
)
|
||||
def test_finding_groups_sort_by_counter_fields(
|
||||
self,
|
||||
authenticated_client,
|
||||
finding_groups_fixture,
|
||||
endpoint_name,
|
||||
sort_field,
|
||||
):
|
||||
"""All counter fields are accepted as sort parameters (asc and desc)."""
|
||||
params = {"sort": f"-{sort_field}"}
|
||||
if endpoint_name == "finding-group-list":
|
||||
params["filter[inserted_at]"] = TODAY
|
||||
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert len(data) > 0
|
||||
|
||||
desc_values = [item["attributes"][sort_field] for item in data]
|
||||
assert desc_values == sorted(desc_values, reverse=True)
|
||||
|
||||
params["sort"] = sort_field
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
asc_values = [
|
||||
item["attributes"][sort_field] for item in response.json()["data"]
|
||||
]
|
||||
assert asc_values == sorted(asc_values)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
)
|
||||
def test_finding_groups_delta_status_breakdown(
|
||||
self, authenticated_client, finding_groups_fixture, endpoint_name
|
||||
):
|
||||
"""`new_*` and `changed_*` counters split by status and mute state.
|
||||
|
||||
s3_bucket_public_access has 1 new FAIL and 1 changed FAIL (both
|
||||
non-muted) so the breakdown must reflect exactly that and the totals
|
||||
must equal the sum of the buckets.
|
||||
"""
|
||||
params = {"filter[check_id]": "s3_bucket_public_access"}
|
||||
if endpoint_name == "finding-group-list":
|
||||
params["filter[inserted_at]"] = TODAY
|
||||
|
||||
response = authenticated_client.get(reverse(endpoint_name), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert len(data) == 1
|
||||
attrs = data[0]["attributes"]
|
||||
|
||||
assert attrs["new_fail_count"] == 1
|
||||
assert attrs["new_fail_muted_count"] == 0
|
||||
assert attrs["new_pass_count"] == 0
|
||||
assert attrs["new_pass_muted_count"] == 0
|
||||
assert attrs["new_manual_count"] == 0
|
||||
assert attrs["new_manual_muted_count"] == 0
|
||||
assert attrs["changed_fail_count"] == 1
|
||||
assert attrs["changed_fail_muted_count"] == 0
|
||||
assert attrs["changed_pass_count"] == 0
|
||||
assert attrs["changed_pass_muted_count"] == 0
|
||||
assert attrs["changed_manual_count"] == 0
|
||||
assert attrs["changed_manual_muted_count"] == 0
|
||||
|
||||
new_total = (
|
||||
attrs["new_fail_count"]
|
||||
+ attrs["new_fail_muted_count"]
|
||||
+ attrs["new_pass_count"]
|
||||
+ attrs["new_pass_muted_count"]
|
||||
+ attrs["new_manual_count"]
|
||||
+ attrs["new_manual_muted_count"]
|
||||
)
|
||||
changed_total = (
|
||||
attrs["changed_fail_count"]
|
||||
+ attrs["changed_fail_muted_count"]
|
||||
+ attrs["changed_pass_count"]
|
||||
+ attrs["changed_pass_muted_count"]
|
||||
+ attrs["changed_manual_count"]
|
||||
+ attrs["changed_manual_muted_count"]
|
||||
)
|
||||
# The non-muted variants of the breakdown must sum to the legacy
|
||||
# totals (new_count/changed_count are stored as non-muted).
|
||||
assert (
|
||||
attrs["new_fail_count"]
|
||||
+ attrs["new_pass_count"]
|
||||
+ attrs["new_manual_count"]
|
||||
== attrs["new_count"]
|
||||
)
|
||||
assert (
|
||||
attrs["changed_fail_count"]
|
||||
+ attrs["changed_pass_count"]
|
||||
+ attrs["changed_manual_count"]
|
||||
== attrs["changed_count"]
|
||||
)
|
||||
# And the *full* breakdown (including the muted halves) is exposed
|
||||
# so clients can also count muted-only deltas without losing data.
|
||||
assert new_total >= attrs["new_count"]
|
||||
assert changed_total >= attrs["changed_count"]
|
||||
|
||||
def test_finding_groups_resources_serializer_exposes_muted(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
"""The /finding-groups/<id>/resources payload must expose `muted`."""
|
||||
response = authenticated_client.get(
|
||||
reverse(
|
||||
"finding-group-resources",
|
||||
kwargs={"pk": "rds_encryption"},
|
||||
),
|
||||
{"filter[inserted_at]": TODAY},
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert data, "rds_encryption should expose its resources"
|
||||
for item in data:
|
||||
attrs = item["attributes"]
|
||||
assert "muted" in attrs and isinstance(attrs["muted"], bool)
|
||||
# rds_encryption has all muted findings
|
||||
assert attrs["muted"] is True
|
||||
# Status reflects the underlying check outcome (FAIL), not MUTED
|
||||
assert attrs["status"] == "FAIL"
|
||||
|
||||
def test_finding_groups_resources_exposes_finding_id(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
"""The /resources payload exposes the most recent matching finding_id.
|
||||
|
||||
rds_encryption has 2 findings, one per resource. Each resource row must
|
||||
report the UUID of its corresponding Finding (UUIDv7 ordering means
|
||||
Max(finding__id) resolves to the latest snapshot in time).
|
||||
"""
|
||||
response = authenticated_client.get(
|
||||
reverse(
|
||||
"finding-group-resources",
|
||||
kwargs={"pk": "rds_encryption"},
|
||||
),
|
||||
{"filter[inserted_at]": TODAY},
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert data, "rds_encryption should expose its resources"
|
||||
|
||||
rds_finding_ids = {
|
||||
str(f.id) for f in finding_groups_fixture if f.check_id == "rds_encryption"
|
||||
}
|
||||
assert rds_finding_ids, "fixture sanity"
|
||||
|
||||
for item in data:
|
||||
attrs = item["attributes"]
|
||||
assert "finding_id" in attrs
|
||||
assert attrs["finding_id"] in rds_finding_ids
|
||||
|
||||
def test_finding_groups_latest_resources_exposes_finding_id(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
"""The /latest/.../resources payload also exposes finding_id."""
|
||||
response = authenticated_client.get(
|
||||
reverse(
|
||||
"finding-group-latest_resources",
|
||||
kwargs={"check_id": "rds_encryption"},
|
||||
),
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert data, "rds_encryption should expose its resources via /latest"
|
||||
|
||||
rds_finding_ids = {
|
||||
str(f.id) for f in finding_groups_fixture if f.check_id == "rds_encryption"
|
||||
}
|
||||
for item in data:
|
||||
attrs = item["attributes"]
|
||||
assert "finding_id" in attrs
|
||||
assert attrs["finding_id"] in rds_finding_ids
|
||||
|
||||
@@ -4185,7 +4185,6 @@ class FindingGroupSerializer(BaseSerializerV1):
|
||||
check_description = serializers.CharField(required=False, allow_null=True)
|
||||
severity = serializers.CharField()
|
||||
status = serializers.CharField()
|
||||
muted = serializers.BooleanField()
|
||||
impacted_providers = serializers.ListField(
|
||||
child=serializers.CharField(), required=False
|
||||
)
|
||||
@@ -4193,25 +4192,9 @@ class FindingGroupSerializer(BaseSerializerV1):
|
||||
resources_total = serializers.IntegerField()
|
||||
pass_count = serializers.IntegerField()
|
||||
fail_count = serializers.IntegerField()
|
||||
manual_count = serializers.IntegerField()
|
||||
pass_muted_count = serializers.IntegerField()
|
||||
fail_muted_count = serializers.IntegerField()
|
||||
manual_muted_count = serializers.IntegerField()
|
||||
muted_count = serializers.IntegerField()
|
||||
new_count = serializers.IntegerField()
|
||||
changed_count = serializers.IntegerField()
|
||||
new_fail_count = serializers.IntegerField()
|
||||
new_fail_muted_count = serializers.IntegerField()
|
||||
new_pass_count = serializers.IntegerField()
|
||||
new_pass_muted_count = serializers.IntegerField()
|
||||
new_manual_count = serializers.IntegerField()
|
||||
new_manual_muted_count = serializers.IntegerField()
|
||||
changed_fail_count = serializers.IntegerField()
|
||||
changed_fail_muted_count = serializers.IntegerField()
|
||||
changed_pass_count = serializers.IntegerField()
|
||||
changed_pass_muted_count = serializers.IntegerField()
|
||||
changed_manual_count = serializers.IntegerField()
|
||||
changed_manual_muted_count = serializers.IntegerField()
|
||||
first_seen_at = serializers.DateTimeField(required=False, allow_null=True)
|
||||
last_seen_at = serializers.DateTimeField(required=False, allow_null=True)
|
||||
failing_since = serializers.DateTimeField(required=False, allow_null=True)
|
||||
@@ -4225,17 +4208,14 @@ class FindingGroupResourceSerializer(BaseSerializerV1):
|
||||
Serializer for Finding Group Resources - resources within a finding group.
|
||||
|
||||
Returns individual resources with their current status, severity,
|
||||
and timing information. Orphan findings (without any resource) expose the
|
||||
finding id as `id` so the row stays identifiable in the UI.
|
||||
and timing information.
|
||||
"""
|
||||
|
||||
id = serializers.UUIDField(source="row_id")
|
||||
id = serializers.UUIDField(source="resource_id")
|
||||
resource = serializers.SerializerMethodField()
|
||||
provider = serializers.SerializerMethodField()
|
||||
finding_id = serializers.UUIDField()
|
||||
status = serializers.CharField()
|
||||
severity = serializers.CharField()
|
||||
muted = serializers.BooleanField()
|
||||
delta = serializers.CharField(required=False, allow_null=True)
|
||||
first_seen_at = serializers.DateTimeField(required=False, allow_null=True)
|
||||
last_seen_at = serializers.DateTimeField(required=False, allow_null=True)
|
||||
|
||||
@@ -26,22 +26,19 @@ from config.settings.social_login import (
|
||||
)
|
||||
from dj_rest_auth.registration.views import SocialLoginView
|
||||
from django.conf import settings as django_settings
|
||||
from django.contrib.postgres.aggregates import ArrayAgg, BoolAnd, StringAgg
|
||||
from django.contrib.postgres.aggregates import ArrayAgg, StringAgg
|
||||
from django.contrib.postgres.search import SearchQuery
|
||||
from django.db import transaction
|
||||
from django.db.models import (
|
||||
BooleanField,
|
||||
Case,
|
||||
CharField,
|
||||
Count,
|
||||
DecimalField,
|
||||
Exists,
|
||||
ExpressionWrapper,
|
||||
F,
|
||||
IntegerField,
|
||||
Max,
|
||||
Min,
|
||||
OuterRef,
|
||||
Prefetch,
|
||||
Q,
|
||||
QuerySet,
|
||||
@@ -417,7 +414,7 @@ class SchemaView(SpectacularAPIView):
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
spectacular_settings.TITLE = "Prowler API"
|
||||
spectacular_settings.VERSION = "1.26.0"
|
||||
spectacular_settings.VERSION = "1.24.0"
|
||||
spectacular_settings.DESCRIPTION = (
|
||||
"Prowler API specification.\n\nThis file is auto-generated."
|
||||
)
|
||||
@@ -7079,29 +7076,9 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
severity_order=Max("severity_order"),
|
||||
pass_count=Sum("pass_count"),
|
||||
fail_count=Sum("fail_count"),
|
||||
manual_count=Sum("manual_count"),
|
||||
pass_muted_count=Sum("pass_muted_count"),
|
||||
fail_muted_count=Sum("fail_muted_count"),
|
||||
manual_muted_count=Sum("manual_muted_count"),
|
||||
muted_count=Sum("muted_count"),
|
||||
# The group is muted only if every contributing daily summary is
|
||||
# itself fully muted. BoolAnd returns False as soon as one row has
|
||||
# at least one actionable finding.
|
||||
muted=BoolAnd("muted"),
|
||||
new_count=Sum("new_count"),
|
||||
changed_count=Sum("changed_count"),
|
||||
new_fail_count=Sum("new_fail_count"),
|
||||
new_fail_muted_count=Sum("new_fail_muted_count"),
|
||||
new_pass_count=Sum("new_pass_count"),
|
||||
new_pass_muted_count=Sum("new_pass_muted_count"),
|
||||
new_manual_count=Sum("new_manual_count"),
|
||||
new_manual_muted_count=Sum("new_manual_muted_count"),
|
||||
changed_fail_count=Sum("changed_fail_count"),
|
||||
changed_fail_muted_count=Sum("changed_fail_muted_count"),
|
||||
changed_pass_count=Sum("changed_pass_count"),
|
||||
changed_pass_muted_count=Sum("changed_pass_muted_count"),
|
||||
changed_manual_count=Sum("changed_manual_count"),
|
||||
changed_manual_muted_count=Sum("changed_manual_muted_count"),
|
||||
resources_total=Sum("resources_total"),
|
||||
resources_fail=Sum("resources_fail"),
|
||||
impacted_providers_str=StringAgg(
|
||||
@@ -7127,94 +7104,39 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
|
||||
# `pass_count`, `fail_count` and `manual_count` only count non-muted
|
||||
# findings. Muted findings are tracked separately via the
|
||||
# `*_muted_count` fields.
|
||||
return (
|
||||
queryset.values("check_id")
|
||||
.annotate(
|
||||
severity_order=Max(severity_case),
|
||||
pass_count=Count("id", filter=Q(status="PASS", muted=False)),
|
||||
fail_count=Count("id", filter=Q(status="FAIL", muted=False)),
|
||||
manual_count=Count("id", filter=Q(status="MANUAL", muted=False)),
|
||||
pass_muted_count=Count("id", filter=Q(status="PASS", muted=True)),
|
||||
fail_muted_count=Count("id", filter=Q(status="FAIL", muted=True)),
|
||||
manual_muted_count=Count("id", filter=Q(status="MANUAL", muted=True)),
|
||||
muted_count=Count("id", filter=Q(muted=True)),
|
||||
nonmuted_count=Count("id", filter=Q(muted=False)),
|
||||
new_count=Count("id", filter=Q(delta="new", muted=False)),
|
||||
changed_count=Count("id", filter=Q(delta="changed", muted=False)),
|
||||
new_fail_count=Count(
|
||||
"id", filter=Q(delta="new", status="FAIL", muted=False)
|
||||
),
|
||||
new_fail_muted_count=Count(
|
||||
"id", filter=Q(delta="new", status="FAIL", muted=True)
|
||||
),
|
||||
new_pass_count=Count(
|
||||
"id", filter=Q(delta="new", status="PASS", muted=False)
|
||||
),
|
||||
new_pass_muted_count=Count(
|
||||
"id", filter=Q(delta="new", status="PASS", muted=True)
|
||||
),
|
||||
new_manual_count=Count(
|
||||
"id", filter=Q(delta="new", status="MANUAL", muted=False)
|
||||
),
|
||||
new_manual_muted_count=Count(
|
||||
"id", filter=Q(delta="new", status="MANUAL", muted=True)
|
||||
),
|
||||
changed_fail_count=Count(
|
||||
"id", filter=Q(delta="changed", status="FAIL", muted=False)
|
||||
),
|
||||
changed_fail_muted_count=Count(
|
||||
"id", filter=Q(delta="changed", status="FAIL", muted=True)
|
||||
),
|
||||
changed_pass_count=Count(
|
||||
"id", filter=Q(delta="changed", status="PASS", muted=False)
|
||||
),
|
||||
changed_pass_muted_count=Count(
|
||||
"id", filter=Q(delta="changed", status="PASS", muted=True)
|
||||
),
|
||||
changed_manual_count=Count(
|
||||
"id", filter=Q(delta="changed", status="MANUAL", muted=False)
|
||||
),
|
||||
changed_manual_muted_count=Count(
|
||||
"id", filter=Q(delta="changed", status="MANUAL", muted=True)
|
||||
),
|
||||
resources_total=Count("resources__id", distinct=True),
|
||||
resources_fail=Count(
|
||||
"resources__id",
|
||||
distinct=True,
|
||||
filter=Q(status="FAIL", muted=False),
|
||||
),
|
||||
impacted_providers_str=StringAgg(
|
||||
Cast("scan__provider__provider", CharField()),
|
||||
delimiter=",",
|
||||
distinct=True,
|
||||
default="",
|
||||
),
|
||||
agg_first_seen_at=Min("first_seen_at"),
|
||||
agg_last_seen_at=Max("inserted_at"),
|
||||
agg_failing_since=Min(
|
||||
"first_seen_at", filter=Q(status="FAIL", muted=False)
|
||||
),
|
||||
check_title=Coalesce(
|
||||
Max(KeyTextTransform("checktitle", "check_metadata")),
|
||||
Max(KeyTextTransform("CheckTitle", "check_metadata")),
|
||||
Max(KeyTextTransform("Checktitle", "check_metadata")),
|
||||
),
|
||||
check_description=Coalesce(
|
||||
Max(KeyTextTransform("description", "check_metadata")),
|
||||
Max(KeyTextTransform("Description", "check_metadata")),
|
||||
),
|
||||
)
|
||||
.annotate(
|
||||
# Group is muted only if it has zero non-muted findings.
|
||||
muted=Case(
|
||||
When(nonmuted_count=0, then=Value(True)),
|
||||
default=Value(False),
|
||||
output_field=BooleanField(),
|
||||
),
|
||||
)
|
||||
return queryset.values("check_id").annotate(
|
||||
severity_order=Max(severity_case),
|
||||
pass_count=Count("id", filter=Q(status="PASS", muted=False)),
|
||||
fail_count=Count("id", filter=Q(status="FAIL", muted=False)),
|
||||
muted_count=Count("id", filter=Q(muted=True)),
|
||||
new_count=Count("id", filter=Q(delta="new", muted=False)),
|
||||
changed_count=Count("id", filter=Q(delta="changed", muted=False)),
|
||||
resources_total=Count("resources__id", distinct=True),
|
||||
resources_fail=Count(
|
||||
"resources__id",
|
||||
distinct=True,
|
||||
filter=Q(status="FAIL", muted=False),
|
||||
),
|
||||
impacted_providers_str=StringAgg(
|
||||
Cast("scan__provider__provider", CharField()),
|
||||
delimiter=",",
|
||||
distinct=True,
|
||||
default="",
|
||||
),
|
||||
agg_first_seen_at=Min("first_seen_at"),
|
||||
agg_last_seen_at=Max("inserted_at"),
|
||||
agg_failing_since=Min(
|
||||
"first_seen_at", filter=Q(status="FAIL", muted=False)
|
||||
),
|
||||
check_title=Coalesce(
|
||||
Max(KeyTextTransform("checktitle", "check_metadata")),
|
||||
Max(KeyTextTransform("CheckTitle", "check_metadata")),
|
||||
Max(KeyTextTransform("Checktitle", "check_metadata")),
|
||||
),
|
||||
check_description=Coalesce(
|
||||
Max(KeyTextTransform("description", "check_metadata")),
|
||||
Max(KeyTextTransform("Description", "check_metadata")),
|
||||
),
|
||||
)
|
||||
|
||||
def _split_computed_aggregate_filters(
|
||||
@@ -7226,7 +7148,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
"status__in",
|
||||
"severity",
|
||||
"severity__in",
|
||||
"muted",
|
||||
"include_muted",
|
||||
}
|
||||
finding_params = QueryDict(mutable=True)
|
||||
@@ -7258,8 +7179,7 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
Post-process aggregation results to add computed fields.
|
||||
|
||||
- Converts severity integer back to string
|
||||
- Computes aggregated status (FAIL > PASS > MANUAL); the orthogonal
|
||||
``muted`` boolean is already on the row from the SQL aggregation
|
||||
- Computes aggregated status (FAIL > PASS > MUTED)
|
||||
- Converts provider string to list
|
||||
"""
|
||||
results = []
|
||||
@@ -7277,21 +7197,13 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
if "agg_failing_since" in row:
|
||||
row["failing_since"] = row.pop("agg_failing_since")
|
||||
|
||||
# Drop the helper count we use to derive `muted` in the
|
||||
# finding-level aggregation path.
|
||||
row.pop("nonmuted_count", None)
|
||||
|
||||
# Compute aggregated status from non-muted counts first, then
|
||||
# fall back to muted counts so fully-muted groups still reflect
|
||||
# the underlying check outcome.
|
||||
total_fail = row.get("fail_count", 0) + row.get("fail_muted_count", 0)
|
||||
total_pass = row.get("pass_count", 0) + row.get("pass_muted_count", 0)
|
||||
if total_fail > 0:
|
||||
# Compute aggregated status
|
||||
if row.get("fail_count", 0) > 0:
|
||||
row["status"] = "FAIL"
|
||||
elif total_pass > 0:
|
||||
elif row.get("pass_count", 0) > 0:
|
||||
row["status"] = "PASS"
|
||||
else:
|
||||
row["status"] = "MANUAL"
|
||||
row["status"] = "MUTED"
|
||||
|
||||
# Convert provider string to list
|
||||
providers_str = row.pop("impacted_providers_str", "") or ""
|
||||
@@ -7308,29 +7220,12 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
"check_title": "check_title",
|
||||
"severity": "severity_order",
|
||||
"status": "status_order",
|
||||
"muted": "muted",
|
||||
"delta": "delta_order",
|
||||
"fail_count": "fail_count",
|
||||
"pass_count": "pass_count",
|
||||
"manual_count": "manual_count",
|
||||
"muted_count": "muted_count",
|
||||
"pass_muted_count": "pass_muted_count",
|
||||
"fail_muted_count": "fail_muted_count",
|
||||
"manual_muted_count": "manual_muted_count",
|
||||
"new_count": "new_count",
|
||||
"new_fail_count": "new_fail_count",
|
||||
"new_fail_muted_count": "new_fail_muted_count",
|
||||
"new_pass_count": "new_pass_count",
|
||||
"new_pass_muted_count": "new_pass_muted_count",
|
||||
"new_manual_count": "new_manual_count",
|
||||
"new_manual_muted_count": "new_manual_muted_count",
|
||||
"changed_count": "changed_count",
|
||||
"changed_fail_count": "changed_fail_count",
|
||||
"changed_fail_muted_count": "changed_fail_muted_count",
|
||||
"changed_pass_count": "changed_pass_count",
|
||||
"changed_pass_muted_count": "changed_pass_muted_count",
|
||||
"changed_manual_count": "changed_manual_count",
|
||||
"changed_manual_muted_count": "changed_manual_muted_count",
|
||||
"resources_total": "resources_total",
|
||||
"resources_fail": "resources_fail",
|
||||
"first_seen_at": "agg_first_seen_at",
|
||||
@@ -7382,28 +7277,23 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
return ordering
|
||||
|
||||
def _apply_aggregated_computed_filters(self, queryset, computed_params: QueryDict):
|
||||
"""Apply computed filters (status/severity/muted) on aggregated finding-group rows."""
|
||||
"""Apply computed filters (status/severity) on aggregated finding-group rows."""
|
||||
if not computed_params:
|
||||
return queryset
|
||||
|
||||
if computed_params.get("status") or computed_params.getlist("status__in"):
|
||||
queryset = queryset.annotate(
|
||||
total_fail=F("fail_count") + F("fail_muted_count"),
|
||||
total_pass=F("pass_count") + F("pass_muted_count"),
|
||||
).annotate(
|
||||
aggregated_status=Case(
|
||||
When(total_fail__gt=0, then=Value("FAIL")),
|
||||
When(total_pass__gt=0, then=Value("PASS")),
|
||||
default=Value("MANUAL"),
|
||||
When(fail_count__gt=0, then=Value("FAIL")),
|
||||
When(pass_count__gt=0, then=Value("PASS")),
|
||||
default=Value("MUTED"),
|
||||
output_field=CharField(),
|
||||
)
|
||||
)
|
||||
|
||||
# Exclude fully-muted groups by default unless the caller has opted in
|
||||
# via either `include_muted` or an explicit `muted` filter (the latter
|
||||
# gives the caller direct control over the column).
|
||||
if "include_muted" not in computed_params and "muted" not in computed_params:
|
||||
queryset = queryset.exclude(muted=True)
|
||||
# Exclude fully-muted groups by default unless include_muted is set
|
||||
if "include_muted" not in computed_params:
|
||||
queryset = queryset.exclude(fail_count=0, pass_count=0, muted_count__gt=0)
|
||||
|
||||
filterset = FindingGroupAggregatedComputedFilter(
|
||||
computed_params, queryset=queryset
|
||||
@@ -7458,14 +7348,18 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
provider_type=Max("resource__provider__provider"),
|
||||
provider_uid=Max("resource__provider__uid"),
|
||||
provider_alias=Max("resource__provider__alias"),
|
||||
# status_order considers ALL findings (muted or not) so it
|
||||
# surfaces FAIL/PASS/MANUAL based on the underlying check
|
||||
# outcome. Whether the resource is actionable is signalled by
|
||||
# the orthogonal `muted` flag below.
|
||||
status_order=Max(
|
||||
Case(
|
||||
When(finding__status="FAIL", then=Value(3)),
|
||||
When(finding__status="PASS", then=Value(2)),
|
||||
When(
|
||||
finding__status="FAIL",
|
||||
finding__muted=False,
|
||||
then=Value(3),
|
||||
),
|
||||
When(
|
||||
finding__status="PASS",
|
||||
finding__muted=False,
|
||||
then=Value(2),
|
||||
),
|
||||
default=Value(1),
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
@@ -7497,8 +7391,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
),
|
||||
first_seen_at=Min("finding__first_seen_at"),
|
||||
last_seen_at=Max("finding__inserted_at"),
|
||||
# True only if every finding for this resource+check is muted.
|
||||
muted=BoolAnd("finding__muted"),
|
||||
# Max() on muted_reason / check_metadata is safe because
|
||||
# all findings for the same resource+check share identical
|
||||
# values (mute rules and metadata are applied per-check).
|
||||
@@ -7506,12 +7398,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
resource_group=Max(
|
||||
KeyTextTransform("resourcegroup", "finding__check_metadata")
|
||||
),
|
||||
# Most recent matching Finding for this (resource, check):
|
||||
# Finding.id is a UUIDv7 (time-ordered in its high 48 bits).
|
||||
# Cast to text first because PostgreSQL has no built-in
|
||||
# `max(uuid)` aggregate; on the canonical lowercase form a
|
||||
# lexicographic Max() still resolves to the latest snapshot.
|
||||
finding_id=Max(Cast("finding__id", output_field=CharField())),
|
||||
)
|
||||
.filter(resource_id__isnull=False)
|
||||
)
|
||||
@@ -7520,8 +7406,8 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
_RESOURCE_SORT_ANNOTATIONS = {
|
||||
"status_order": lambda: Max(
|
||||
Case(
|
||||
When(finding__status="FAIL", then=Value(3)),
|
||||
When(finding__status="PASS", then=Value(2)),
|
||||
When(finding__status="FAIL", finding__muted=False, then=Value(3)),
|
||||
When(finding__status="PASS", finding__muted=False, then=Value(2)),
|
||||
default=Value(1),
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
@@ -7584,53 +7470,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
.order_by(*ordering)
|
||||
)
|
||||
|
||||
def _orphan_findings_queryset(self, filtered_queryset, finding_ids=None):
|
||||
"""Findings in the filtered set with no ResourceFindingMapping entries."""
|
||||
orphan_qs = filtered_queryset.filter(
|
||||
~Exists(ResourceFindingMapping.objects.filter(finding_id=OuterRef("pk")))
|
||||
)
|
||||
if finding_ids is not None:
|
||||
orphan_qs = orphan_qs.filter(id__in=finding_ids)
|
||||
return orphan_qs
|
||||
|
||||
def _has_orphan_findings(self, filtered_queryset) -> bool:
|
||||
"""Return True if any finding in the filtered set has no resource mapping."""
|
||||
return self._orphan_findings_queryset(filtered_queryset).exists()
|
||||
|
||||
def _orphan_aggregation_values(self, orphan_queryset):
|
||||
"""Raw rows for orphan findings; resource payload synthesized from metadata.
|
||||
|
||||
check_metadata is stored with lowercase keys (see
|
||||
`prowler.lib.outputs.finding.Finding.get_metadata`) and
|
||||
`Finding.resource_groups` is already denormalized at ingest time.
|
||||
"""
|
||||
return orphan_queryset.annotate(
|
||||
_provider_type=F("scan__provider__provider"),
|
||||
_provider_uid=F("scan__provider__uid"),
|
||||
_provider_alias=F("scan__provider__alias"),
|
||||
_svc=KeyTextTransform("servicename", "check_metadata"),
|
||||
_region=KeyTextTransform("region", "check_metadata"),
|
||||
_rtype=KeyTextTransform("resourcetype", "check_metadata"),
|
||||
_rgroup=F("resource_groups"),
|
||||
).values(
|
||||
"id",
|
||||
"uid",
|
||||
"status",
|
||||
"severity",
|
||||
"delta",
|
||||
"muted",
|
||||
"muted_reason",
|
||||
"first_seen_at",
|
||||
"inserted_at",
|
||||
"_provider_type",
|
||||
"_provider_uid",
|
||||
"_provider_alias",
|
||||
"_svc",
|
||||
"_region",
|
||||
"_rtype",
|
||||
"_rgroup",
|
||||
)
|
||||
|
||||
def _post_process_resources(self, resource_data):
|
||||
"""Convert resource aggregation rows to API output."""
|
||||
results = []
|
||||
@@ -7642,7 +7481,7 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
elif status_order == 2:
|
||||
status = "PASS"
|
||||
else:
|
||||
status = "MANUAL"
|
||||
status = "MUTED"
|
||||
|
||||
delta_order = row.get("delta_order", 0)
|
||||
if delta_order == 2:
|
||||
@@ -7652,13 +7491,9 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
else:
|
||||
delta = None
|
||||
|
||||
resource_id = row["resource_id"]
|
||||
finding_id = str(row["finding_id"]) if row.get("finding_id") else None
|
||||
|
||||
results.append(
|
||||
{
|
||||
"row_id": resource_id,
|
||||
"resource_id": resource_id,
|
||||
"resource_id": row["resource_id"],
|
||||
"resource_uid": row["resource_uid"],
|
||||
"resource_name": row["resource_name"],
|
||||
"resource_service": row["resource_service"],
|
||||
@@ -7674,49 +7509,8 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
"delta": delta,
|
||||
"first_seen_at": row["first_seen_at"],
|
||||
"last_seen_at": row["last_seen_at"],
|
||||
"muted": bool(row.get("muted", False)),
|
||||
"muted_reason": row.get("muted_reason"),
|
||||
"resource_group": row.get("resource_group", ""),
|
||||
"finding_id": finding_id,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _post_process_orphans(self, orphan_rows):
|
||||
"""Convert orphan finding rows into the same API shape as mapping rows."""
|
||||
results = []
|
||||
for row in orphan_rows:
|
||||
status_val = row["status"]
|
||||
status = status_val if status_val in ("FAIL", "PASS") else "MANUAL"
|
||||
|
||||
muted = bool(row["muted"])
|
||||
delta_val = row.get("delta")
|
||||
delta = delta_val if delta_val in ("new", "changed") and not muted else None
|
||||
|
||||
finding_id = str(row["id"])
|
||||
|
||||
results.append(
|
||||
{
|
||||
"row_id": finding_id,
|
||||
"resource_id": None,
|
||||
"resource_uid": row["uid"],
|
||||
"resource_name": row["uid"],
|
||||
"resource_service": row["_svc"] or "",
|
||||
"resource_region": row["_region"] or "",
|
||||
"resource_type": row["_rtype"] or "",
|
||||
"provider_type": row["_provider_type"],
|
||||
"provider_uid": row["_provider_uid"],
|
||||
"provider_alias": row["_provider_alias"],
|
||||
"status": status,
|
||||
"severity": row["severity"],
|
||||
"delta": delta,
|
||||
"first_seen_at": row["first_seen_at"],
|
||||
"last_seen_at": row["inserted_at"],
|
||||
"muted": muted,
|
||||
"muted_reason": row.get("muted_reason"),
|
||||
"resource_group": row["_rgroup"] or "",
|
||||
"finding_id": finding_id,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -7777,19 +7571,24 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
sort_param, self._FINDING_GROUP_SORT_MAP
|
||||
)
|
||||
if ordering:
|
||||
# status_order is annotated on demand so groups can be sorted
|
||||
# by their aggregated status (FAIL > PASS > MUTED), mirroring
|
||||
# the priority used in _post_process_aggregation.
|
||||
if any(field.lstrip("-") == "status_order" for field in ordering):
|
||||
aggregated_queryset = aggregated_queryset.annotate(
|
||||
total_fail_for_sort=F("fail_count") + F("fail_muted_count"),
|
||||
total_pass_for_sort=F("pass_count") + F("pass_muted_count"),
|
||||
).annotate(
|
||||
status_order=Case(
|
||||
When(total_fail_for_sort__gt=0, then=Value(3)),
|
||||
When(total_pass_for_sort__gt=0, then=Value(2)),
|
||||
When(fail_count__gt=0, then=Value(3)),
|
||||
When(pass_count__gt=0, then=Value(2)),
|
||||
default=Value(1),
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
)
|
||||
|
||||
# delta_order is a virtual sort field: expand it to a
|
||||
# lexicographic ordering by (new_count, changed_count) so groups
|
||||
# with more new findings rank higher, with changed_count as the
|
||||
# tie-breaker (preserves the "new > changed" priority used by
|
||||
# the resources endpoint, but driven by the actual counters).
|
||||
expanded_ordering = []
|
||||
for field in ordering:
|
||||
if field.lstrip("-") == "delta_order":
|
||||
@@ -7823,64 +7622,41 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
def _paginated_resource_response(
|
||||
self, request, filtered_queryset, resource_ids, tenant_id
|
||||
):
|
||||
"""Paginate and return resources, appending orphan findings when present.
|
||||
"""Paginate and return resources.
|
||||
|
||||
Hot path (no orphans, or resource filter applied): resources come from
|
||||
ResourceFindingMapping aggregation. Untouched pre-existing behaviour.
|
||||
|
||||
Orphan fallback: findings without a mapping (e.g. IaC) are appended
|
||||
after mapping rows as synthesised resource-like rows so they remain
|
||||
visible in the UI without paying the aggregation cost on the hot path.
|
||||
Without sort: paginate lightweight resource IDs first, aggregate only the page.
|
||||
With sort: build a lightweight ordering subquery (resource_id + sort keys),
|
||||
paginate that, then aggregate full details only for the page.
|
||||
"""
|
||||
sort_param = request.query_params.get("sort")
|
||||
ordering = None
|
||||
|
||||
if sort_param:
|
||||
validated = self._validate_sort_fields(sort_param, self._RESOURCE_SORT_MAP)
|
||||
ordering = validated if validated else None
|
||||
ordering = self._validate_sort_fields(sort_param, self._RESOURCE_SORT_MAP)
|
||||
if ordering:
|
||||
if "resource_id" not in {field.lstrip("-") for field in ordering}:
|
||||
ordering.append("resource_id")
|
||||
|
||||
# Resource filters can only match findings with resources; skip orphan
|
||||
# detection entirely when they are present.
|
||||
if resource_ids is not None:
|
||||
return self._mapping_paginated_response(
|
||||
request, filtered_queryset, resource_ids, tenant_id, ordering
|
||||
)
|
||||
# Phase 1: lightweight aggregation with only sort keys, paginate
|
||||
ordering_qs = self._build_resource_ordering_queryset(
|
||||
filtered_queryset,
|
||||
resource_ids=resource_ids,
|
||||
tenant_id=tenant_id,
|
||||
ordering=ordering,
|
||||
)
|
||||
page = self.paginate_queryset(ordering_qs)
|
||||
if page is not None:
|
||||
page_ids = [row["resource_id"] for row in page]
|
||||
resource_data = self._build_resource_aggregation(
|
||||
filtered_queryset, resource_ids=page_ids, tenant_id=tenant_id
|
||||
)
|
||||
# Re-sort to match the page ordering
|
||||
id_order = {rid: idx for idx, rid in enumerate(page_ids)}
|
||||
results = self._post_process_resources(resource_data)
|
||||
results.sort(key=lambda r: id_order.get(r["resource_id"], 0))
|
||||
serializer = FindingGroupResourceSerializer(results, many=True)
|
||||
return self.get_paginated_response(serializer.data)
|
||||
|
||||
has_mappings = self._build_resource_mapping_queryset(
|
||||
filtered_queryset, resource_ids=None, tenant_id=tenant_id
|
||||
).exists()
|
||||
|
||||
if has_mappings:
|
||||
# Normal or mixed group: serve only resource-mapped rows.
|
||||
# TODO: Orphan findings in mixed groups are intentionally excluded
|
||||
# until the ephemeral resources strategy is decided. When resolved,
|
||||
# route mixed groups to _combined_paginated_response instead.
|
||||
return self._mapping_paginated_response(
|
||||
request, filtered_queryset, resource_ids, tenant_id, ordering
|
||||
)
|
||||
|
||||
# Pure orphan group (e.g. IaC): synthesize resource-like rows.
|
||||
return self._combined_paginated_response(
|
||||
request, filtered_queryset, tenant_id, ordering
|
||||
)
|
||||
|
||||
def _mapping_paginated_response(
|
||||
self, request, filtered_queryset, resource_ids, tenant_id, ordering
|
||||
):
|
||||
"""Mapping-only paginated response (original fast path)."""
|
||||
if ordering:
|
||||
if "resource_id" not in {field.lstrip("-") for field in ordering}:
|
||||
ordering.append("resource_id")
|
||||
|
||||
# Phase 1: lightweight aggregation with only sort keys, paginate
|
||||
ordering_qs = self._build_resource_ordering_queryset(
|
||||
filtered_queryset,
|
||||
resource_ids=resource_ids,
|
||||
tenant_id=tenant_id,
|
||||
ordering=ordering,
|
||||
)
|
||||
page = self.paginate_queryset(ordering_qs)
|
||||
if page is not None:
|
||||
page_ids = [row["resource_id"] for row in page]
|
||||
page_ids = [row["resource_id"] for row in ordering_qs]
|
||||
resource_data = self._build_resource_aggregation(
|
||||
filtered_queryset, resource_ids=page_ids, tenant_id=tenant_id
|
||||
)
|
||||
@@ -7888,18 +7664,10 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
results = self._post_process_resources(resource_data)
|
||||
results.sort(key=lambda r: id_order.get(r["resource_id"], 0))
|
||||
serializer = FindingGroupResourceSerializer(results, many=True)
|
||||
return self.get_paginated_response(serializer.data)
|
||||
|
||||
page_ids = [row["resource_id"] for row in ordering_qs]
|
||||
resource_data = self._build_resource_aggregation(
|
||||
filtered_queryset, resource_ids=page_ids, tenant_id=tenant_id
|
||||
)
|
||||
id_order = {rid: idx for idx, rid in enumerate(page_ids)}
|
||||
results = self._post_process_resources(resource_data)
|
||||
results.sort(key=lambda r: id_order.get(r["resource_id"], 0))
|
||||
serializer = FindingGroupResourceSerializer(results, many=True)
|
||||
return Response(serializer.data)
|
||||
return Response(serializer.data)
|
||||
|
||||
# No sort (or only empty sort fragments): paginate lightweight resource IDs
|
||||
# first, aggregate only the page.
|
||||
mapping_qs = self._build_resource_mapping_queryset(
|
||||
filtered_queryset, resource_ids=resource_ids, tenant_id=tenant_id
|
||||
)
|
||||
@@ -7927,95 +7695,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
serializer = FindingGroupResourceSerializer(results, many=True)
|
||||
return Response(serializer.data)
|
||||
|
||||
def _combined_paginated_response(
|
||||
self, request, filtered_queryset, tenant_id, ordering
|
||||
):
|
||||
"""Mapping rows + orphan findings appended at end.
|
||||
|
||||
Orphans sit after mapping rows regardless of sort. This keeps the
|
||||
mapping-only code path intact for checks that have no orphans (the
|
||||
common case) and avoids paying UNION/coalesce costs there.
|
||||
"""
|
||||
mapping_qs = self._build_resource_mapping_queryset(
|
||||
filtered_queryset, resource_ids=None, tenant_id=tenant_id
|
||||
)
|
||||
mapping_count = mapping_qs.values("resource_id").distinct().count()
|
||||
|
||||
orphan_ids = list(
|
||||
self._orphan_findings_queryset(filtered_queryset)
|
||||
.order_by("id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
orphan_count = len(orphan_ids)
|
||||
total = mapping_count + orphan_count
|
||||
|
||||
# Paginate a simple [0..total) index sequence so DRF produces proper
|
||||
# links/meta; then slice mapping / orphan sources accordingly.
|
||||
page = self.paginate_queryset(range(total))
|
||||
page_indices = list(page) if page is not None else list(range(total))
|
||||
|
||||
mapping_indices = [i for i in page_indices if i < mapping_count]
|
||||
orphan_positions = [
|
||||
i - mapping_count for i in page_indices if i >= mapping_count
|
||||
]
|
||||
|
||||
mapping_results = []
|
||||
if mapping_indices:
|
||||
start = mapping_indices[0]
|
||||
stop = mapping_indices[-1] + 1
|
||||
if ordering:
|
||||
ordering_fields = list(ordering)
|
||||
if "resource_id" not in {
|
||||
field.lstrip("-") for field in ordering_fields
|
||||
}:
|
||||
ordering_fields.append("resource_id")
|
||||
ordered_qs = self._build_resource_ordering_queryset(
|
||||
filtered_queryset,
|
||||
resource_ids=None,
|
||||
tenant_id=tenant_id,
|
||||
ordering=ordering_fields,
|
||||
)
|
||||
slice_rids = [row["resource_id"] for row in ordered_qs[start:stop]]
|
||||
else:
|
||||
slice_rids = list(
|
||||
mapping_qs.values_list("resource_id", flat=True)
|
||||
.distinct()
|
||||
.order_by("resource_id")[start:stop]
|
||||
)
|
||||
if slice_rids:
|
||||
resource_data = self._build_resource_aggregation(
|
||||
filtered_queryset,
|
||||
resource_ids=slice_rids,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
rows_by_rid = {row["resource_id"]: row for row in resource_data}
|
||||
ordered_rows = [
|
||||
rows_by_rid[rid] for rid in slice_rids if rid in rows_by_rid
|
||||
]
|
||||
mapping_results = self._post_process_resources(ordered_rows)
|
||||
|
||||
orphan_results = []
|
||||
if orphan_positions:
|
||||
slice_fids = [orphan_ids[pos] for pos in orphan_positions]
|
||||
raw_rows = list(
|
||||
self._orphan_aggregation_values(
|
||||
self._orphan_findings_queryset(
|
||||
filtered_queryset, finding_ids=slice_fids
|
||||
)
|
||||
)
|
||||
)
|
||||
rows_by_fid = {row["id"]: row for row in raw_rows}
|
||||
ordered_rows = [
|
||||
rows_by_fid[fid] for fid in slice_fids if fid in rows_by_fid
|
||||
]
|
||||
orphan_results = self._post_process_orphans(ordered_rows)
|
||||
|
||||
results = mapping_results + orphan_results
|
||||
serializer = FindingGroupResourceSerializer(results, many=True)
|
||||
if page is not None:
|
||||
return self.get_paginated_response(serializer.data)
|
||||
return Response(serializer.data)
|
||||
|
||||
def list(self, request, *args, **kwargs):
|
||||
"""
|
||||
List finding groups with aggregation and filtering.
|
||||
|
||||
@@ -15,7 +15,7 @@ from config.django.production import LOGGING as DJANGO_LOGGERS, DEBUG # noqa: E
|
||||
from config.custom_logging import BackendLogger # noqa: E402
|
||||
|
||||
BIND_ADDRESS = env("DJANGO_BIND_ADDRESS", default="127.0.0.1")
|
||||
PORT = env("DJANGO_PORT", default=8080)
|
||||
PORT = env("DJANGO_PORT", default=8000)
|
||||
|
||||
# Server settings
|
||||
bind = f"{BIND_ADDRESS}:{PORT}"
|
||||
|
||||
@@ -5,6 +5,7 @@ This module handles:
|
||||
- Adding resource labels to Cartography nodes for efficient lookups
|
||||
- Loading Prowler findings into the graph
|
||||
- Linking findings to resources
|
||||
- Cleaning up stale findings
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
@@ -23,6 +24,7 @@ from tasks.jobs.attack_paths.config import (
|
||||
)
|
||||
from tasks.jobs.attack_paths.queries import (
|
||||
ADD_RESOURCE_LABEL_TEMPLATE,
|
||||
CLEANUP_FINDINGS_TEMPLATE,
|
||||
INSERT_FINDING_TEMPLATE,
|
||||
render_cypher_template,
|
||||
)
|
||||
@@ -90,13 +92,14 @@ def analysis(
|
||||
"""
|
||||
Main entry point for Prowler findings analysis.
|
||||
|
||||
Adds resource labels and loads findings.
|
||||
Adds resource labels, loads findings, and cleans up stale data.
|
||||
"""
|
||||
add_resource_label(
|
||||
neo4j_session, prowler_api_provider.provider, str(prowler_api_provider.uid)
|
||||
)
|
||||
findings_data = stream_findings_with_resources(prowler_api_provider, scan_id)
|
||||
load_findings(neo4j_session, findings_data, prowler_api_provider, config)
|
||||
cleanup_findings(neo4j_session, prowler_api_provider, config)
|
||||
|
||||
|
||||
def add_resource_label(
|
||||
@@ -180,6 +183,28 @@ def load_findings(
|
||||
logger.info(f"Finished loading {total_records} records in {batch_num} batches")
|
||||
|
||||
|
||||
def cleanup_findings(
|
||||
neo4j_session: neo4j.Session,
|
||||
prowler_api_provider: Provider,
|
||||
config: CartographyConfig,
|
||||
) -> None:
|
||||
"""Remove stale findings (classic Cartography behaviour)."""
|
||||
parameters = {
|
||||
"last_updated": config.update_tag,
|
||||
"batch_size": BATCH_SIZE,
|
||||
}
|
||||
|
||||
batch = 1
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
logger.info(f"Cleaning findings batch {batch}")
|
||||
|
||||
result = neo4j_session.run(CLEANUP_FINDINGS_TEMPLATE, parameters)
|
||||
|
||||
deleted_count = result.single().get("deleted_findings_count", 0)
|
||||
batch += 1
|
||||
|
||||
|
||||
# Findings Streaming (Generator-based)
|
||||
# -------------------------------------
|
||||
|
||||
@@ -248,9 +273,7 @@ def _fetch_findings_batch(
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
# Use `all_objects` to get `Findings` even on soft-deleted `Providers`
|
||||
# But even the provider is already validated as active in this context
|
||||
qs = FindingModel.all_objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).order_by("id")
|
||||
qs = FindingModel.all_objects.filter(scan_id=scan_id).order_by("id")
|
||||
|
||||
if after_id is not None:
|
||||
qs = qs.filter(id__gt=after_id)
|
||||
|
||||
@@ -13,13 +13,14 @@ from tasks.jobs.attack_paths.config import (
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
# Indexes for Prowler Findings and resource lookups
|
||||
# Indexes for Prowler findings and resource lookups
|
||||
FINDINGS_INDEX_STATEMENTS = [
|
||||
# Resource indexes for Prowler Finding lookups
|
||||
"CREATE INDEX aws_resource_arn IF NOT EXISTS FOR (n:_AWSResource) ON (n.arn);",
|
||||
"CREATE INDEX aws_resource_id IF NOT EXISTS FOR (n:_AWSResource) ON (n.id);",
|
||||
# Prowler Finding indexes
|
||||
f"CREATE INDEX prowler_finding_id IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.id);",
|
||||
f"CREATE INDEX prowler_finding_lastupdated IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.lastupdated);",
|
||||
f"CREATE INDEX prowler_finding_status IF NOT EXISTS FOR (n:{PROWLER_FINDING_LABEL}) ON (n.status);",
|
||||
# Internet node index for MERGE lookups
|
||||
f"CREATE INDEX internet_id IF NOT EXISTS FOR (n:{INTERNET_NODE_LABEL}) ON (n.id);",
|
||||
|
||||
@@ -80,6 +80,17 @@ INSERT_FINDING_TEMPLATE = f"""
|
||||
rel.lastupdated = $last_updated
|
||||
"""
|
||||
|
||||
CLEANUP_FINDINGS_TEMPLATE = f"""
|
||||
MATCH (finding:{PROWLER_FINDING_LABEL})
|
||||
WHERE finding.lastupdated < $last_updated
|
||||
|
||||
WITH finding LIMIT $batch_size
|
||||
|
||||
DETACH DELETE finding
|
||||
|
||||
RETURN COUNT(finding) AS deleted_findings_count
|
||||
"""
|
||||
|
||||
# Internet queries (used by internet.py)
|
||||
# ---------------------------------------
|
||||
|
||||
|
||||
@@ -32,13 +32,9 @@ from prowler.lib.outputs.compliance.cis.cis_aws import AWSCIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_azure import AzureCIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_github import GithubCIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_googleworkspace import GoogleWorkspaceCIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_kubernetes import KubernetesCIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_m365 import M365CIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_oraclecloud import OracleCloudCIS
|
||||
from prowler.lib.outputs.compliance.cisa_scuba.cisa_scuba_googleworkspace import (
|
||||
GoogleWorkspaceCISASCuBA,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.csa.csa_alibabacloud import AlibabaCloudCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_aws import AWSCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_azure import AzureCSA
|
||||
@@ -97,7 +93,7 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name.startswith("iso27001_"), AWSISO27001),
|
||||
(lambda name: name.startswith("kisa"), AWSKISAISMSP),
|
||||
(lambda name: name == "prowler_threatscore_aws", ProwlerThreatScoreAWS),
|
||||
(lambda name: name.startswith("ccc_"), CCC_AWS),
|
||||
(lambda name: name == "ccc_aws", CCC_AWS),
|
||||
(lambda name: name.startswith("c5_"), AWSC5),
|
||||
(lambda name: name.startswith("csa_"), AWSCSA),
|
||||
],
|
||||
@@ -106,7 +102,7 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name == "mitre_attack_azure", AzureMitreAttack),
|
||||
(lambda name: name.startswith("ens_"), AzureENS),
|
||||
(lambda name: name.startswith("iso27001_"), AzureISO27001),
|
||||
(lambda name: name.startswith("ccc_"), CCC_Azure),
|
||||
(lambda name: name == "ccc_azure", CCC_Azure),
|
||||
(lambda name: name == "prowler_threatscore_azure", ProwlerThreatScoreAzure),
|
||||
(lambda name: name == "c5_azure", AzureC5),
|
||||
(lambda name: name.startswith("csa_"), AzureCSA),
|
||||
@@ -117,7 +113,7 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name.startswith("ens_"), GCPENS),
|
||||
(lambda name: name.startswith("iso27001_"), GCPISO27001),
|
||||
(lambda name: name == "prowler_threatscore_gcp", ProwlerThreatScoreGCP),
|
||||
(lambda name: name.startswith("ccc_"), CCC_GCP),
|
||||
(lambda name: name == "ccc_gcp", CCC_GCP),
|
||||
(lambda name: name == "c5_gcp", GCPC5),
|
||||
(lambda name: name.startswith("csa_"), GCPCSA),
|
||||
],
|
||||
@@ -137,10 +133,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
"github": [
|
||||
(lambda name: name.startswith("cis_"), GithubCIS),
|
||||
],
|
||||
"googleworkspace": [
|
||||
(lambda name: name.startswith("cis_"), GoogleWorkspaceCIS),
|
||||
(lambda name: name.startswith("cisa_scuba_"), GoogleWorkspaceCISASCuBA),
|
||||
],
|
||||
"iac": [
|
||||
# IaC provider doesn't have specific compliance frameworks yet
|
||||
# Trivy handles its own compliance checks
|
||||
|
||||
@@ -1803,10 +1803,7 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str):
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
|
||||
# Aggregate findings by check_id for this scan.
|
||||
# `pass_count`, `fail_count` and `manual_count` only count non-muted
|
||||
# findings. Muted findings are tracked separately via the
|
||||
# `*_muted_count` fields.
|
||||
# Aggregate findings by check_id for this scan
|
||||
aggregated = (
|
||||
Finding.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
@@ -1817,50 +1814,9 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str):
|
||||
severity_order=Max(severity_case),
|
||||
pass_count=Count("id", filter=Q(status="PASS", muted=False)),
|
||||
fail_count=Count("id", filter=Q(status="FAIL", muted=False)),
|
||||
manual_count=Count("id", filter=Q(status="MANUAL", muted=False)),
|
||||
pass_muted_count=Count("id", filter=Q(status="PASS", muted=True)),
|
||||
fail_muted_count=Count("id", filter=Q(status="FAIL", muted=True)),
|
||||
manual_muted_count=Count("id", filter=Q(status="MANUAL", muted=True)),
|
||||
muted_count=Count("id", filter=Q(muted=True)),
|
||||
nonmuted_count=Count("id", filter=Q(muted=False)),
|
||||
new_count=Count("id", filter=Q(delta="new", muted=False)),
|
||||
changed_count=Count("id", filter=Q(delta="changed", muted=False)),
|
||||
new_fail_count=Count(
|
||||
"id", filter=Q(delta="new", status="FAIL", muted=False)
|
||||
),
|
||||
new_fail_muted_count=Count(
|
||||
"id", filter=Q(delta="new", status="FAIL", muted=True)
|
||||
),
|
||||
new_pass_count=Count(
|
||||
"id", filter=Q(delta="new", status="PASS", muted=False)
|
||||
),
|
||||
new_pass_muted_count=Count(
|
||||
"id", filter=Q(delta="new", status="PASS", muted=True)
|
||||
),
|
||||
new_manual_count=Count(
|
||||
"id", filter=Q(delta="new", status="MANUAL", muted=False)
|
||||
),
|
||||
new_manual_muted_count=Count(
|
||||
"id", filter=Q(delta="new", status="MANUAL", muted=True)
|
||||
),
|
||||
changed_fail_count=Count(
|
||||
"id", filter=Q(delta="changed", status="FAIL", muted=False)
|
||||
),
|
||||
changed_fail_muted_count=Count(
|
||||
"id", filter=Q(delta="changed", status="FAIL", muted=True)
|
||||
),
|
||||
changed_pass_count=Count(
|
||||
"id", filter=Q(delta="changed", status="PASS", muted=False)
|
||||
),
|
||||
changed_pass_muted_count=Count(
|
||||
"id", filter=Q(delta="changed", status="PASS", muted=True)
|
||||
),
|
||||
changed_manual_count=Count(
|
||||
"id", filter=Q(delta="changed", status="MANUAL", muted=False)
|
||||
),
|
||||
changed_manual_muted_count=Count(
|
||||
"id", filter=Q(delta="changed", status="MANUAL", muted=True)
|
||||
),
|
||||
resources_total=Count("resources__id", distinct=True),
|
||||
resources_fail=Count(
|
||||
"resources__id",
|
||||
@@ -1939,26 +1895,9 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str):
|
||||
severity_order=row["severity_order"] or 1,
|
||||
pass_count=row["pass_count"],
|
||||
fail_count=row["fail_count"],
|
||||
manual_count=row["manual_count"],
|
||||
pass_muted_count=row["pass_muted_count"],
|
||||
fail_muted_count=row["fail_muted_count"],
|
||||
manual_muted_count=row["manual_muted_count"],
|
||||
muted_count=row["muted_count"],
|
||||
muted=row["nonmuted_count"] == 0,
|
||||
new_count=row["new_count"],
|
||||
changed_count=row["changed_count"],
|
||||
new_fail_count=row["new_fail_count"],
|
||||
new_fail_muted_count=row["new_fail_muted_count"],
|
||||
new_pass_count=row["new_pass_count"],
|
||||
new_pass_muted_count=row["new_pass_muted_count"],
|
||||
new_manual_count=row["new_manual_count"],
|
||||
new_manual_muted_count=row["new_manual_muted_count"],
|
||||
changed_fail_count=row["changed_fail_count"],
|
||||
changed_fail_muted_count=row["changed_fail_muted_count"],
|
||||
changed_pass_count=row["changed_pass_count"],
|
||||
changed_pass_muted_count=row["changed_pass_muted_count"],
|
||||
changed_manual_count=row["changed_manual_count"],
|
||||
changed_manual_muted_count=row["changed_manual_muted_count"],
|
||||
resources_total=row["resources_total"],
|
||||
resources_fail=row["resources_fail"],
|
||||
first_seen_at=row["agg_first_seen_at"],
|
||||
@@ -1978,26 +1917,9 @@ def aggregate_finding_group_summaries(tenant_id: str, scan_id: str):
|
||||
"severity_order",
|
||||
"pass_count",
|
||||
"fail_count",
|
||||
"manual_count",
|
||||
"pass_muted_count",
|
||||
"fail_muted_count",
|
||||
"manual_muted_count",
|
||||
"muted_count",
|
||||
"muted",
|
||||
"new_count",
|
||||
"changed_count",
|
||||
"new_fail_count",
|
||||
"new_fail_muted_count",
|
||||
"new_pass_count",
|
||||
"new_pass_muted_count",
|
||||
"new_manual_count",
|
||||
"new_manual_muted_count",
|
||||
"changed_fail_count",
|
||||
"changed_fail_muted_count",
|
||||
"changed_pass_count",
|
||||
"changed_pass_muted_count",
|
||||
"changed_manual_count",
|
||||
"changed_manual_muted_count",
|
||||
"resources_total",
|
||||
"resources_fail",
|
||||
"first_seen_at",
|
||||
|
||||
@@ -771,49 +771,26 @@ def aggregate_finding_group_summaries_task(tenant_id: str, scan_id: str):
|
||||
)
|
||||
@set_tenant(keep_tenant=True)
|
||||
def reaggregate_all_finding_group_summaries_task(tenant_id: str):
|
||||
"""Reaggregate finding group summaries for every (provider, day) combination.
|
||||
|
||||
Mirrors the unbounded scope of `mute_historical_findings_task`: that task
|
||||
rewrites every Finding row whose UID matches a mute rule, with no time
|
||||
limit. To keep the daily summaries consistent with that update, this task
|
||||
re-runs the aggregator on the latest completed scan of every (provider,
|
||||
day) pair that exists in the database. Tasks are dispatched in parallel
|
||||
via a Celery group so the wallclock scales with the worker pool, not with
|
||||
the number of pairs.
|
||||
"""
|
||||
completed_scans = list(
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
completed_at__isnull=False,
|
||||
)
|
||||
.order_by("-completed_at")
|
||||
.values("id", "completed_at", "provider_id")
|
||||
"""Reaggregate finding group summaries for all providers' latest completed scans."""
|
||||
latest_scan_ids = list(
|
||||
Scan.objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED)
|
||||
.order_by("provider_id", "-completed_at", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
|
||||
# Keep the latest scan per (provider, day) pair so the daily summary row
|
||||
# the aggregator writes is the most recent snapshot of that day for that
|
||||
# provider. Iterating from most recent to oldest means the first scan we
|
||||
# see for a given key wins.
|
||||
latest_scans: dict[tuple, str] = {}
|
||||
for scan in completed_scans:
|
||||
key = (scan["provider_id"], scan["completed_at"].date())
|
||||
if key not in latest_scans:
|
||||
latest_scans[key] = str(scan["id"])
|
||||
|
||||
scan_ids = list(latest_scans.values())
|
||||
if scan_ids:
|
||||
if latest_scan_ids:
|
||||
logger.info(
|
||||
"Reaggregating finding group summaries for %d scans (provider x day)",
|
||||
len(scan_ids),
|
||||
"Reaggregating finding group summaries for %d scans: %s",
|
||||
len(latest_scan_ids),
|
||||
latest_scan_ids,
|
||||
)
|
||||
group(
|
||||
aggregate_finding_group_summaries_task.si(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
tenant_id=tenant_id, scan_id=str(scan_id)
|
||||
)
|
||||
for scan_id in scan_ids
|
||||
for scan_id in latest_scan_ids
|
||||
).apply_async()
|
||||
return {"scans_reaggregated": len(scan_ids)}
|
||||
return {"scans_reaggregated": len(latest_scan_ids)}
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="lighthouse-connection-check")
|
||||
|
||||
@@ -1298,6 +1298,23 @@ class TestAttackPathsFindingsHelpers:
|
||||
assert params["last_updated"] == config.update_tag
|
||||
assert "findings_data" in params
|
||||
|
||||
def test_cleanup_findings_runs_batches(self, providers_fixture):
|
||||
provider = providers_fixture[0]
|
||||
config = SimpleNamespace(update_tag=1024)
|
||||
mock_session = MagicMock()
|
||||
|
||||
first_batch = MagicMock()
|
||||
first_batch.single.return_value = {"deleted_findings_count": 3}
|
||||
second_batch = MagicMock()
|
||||
second_batch.single.return_value = {"deleted_findings_count": 0}
|
||||
mock_session.run.side_effect = [first_batch, second_batch]
|
||||
|
||||
findings_module.cleanup_findings(mock_session, provider, config)
|
||||
|
||||
assert mock_session.run.call_count == 2
|
||||
params = mock_session.run.call_args.args[1]
|
||||
assert params["last_updated"] == config.update_tag
|
||||
|
||||
def test_stream_findings_with_resources_returns_latest_scan_data(
|
||||
self,
|
||||
tenants_fixture,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import openai
|
||||
@@ -2362,96 +2362,35 @@ class TestReaggregateAllFindingGroupSummaries:
|
||||
@patch("tasks.tasks.group")
|
||||
@patch("tasks.tasks.aggregate_finding_group_summaries_task")
|
||||
@patch("tasks.tasks.Scan.objects.filter")
|
||||
def test_dispatches_subtasks_for_each_provider_per_day(
|
||||
def test_dispatches_subtasks_for_each_provider(
|
||||
self, mock_scan_filter, mock_agg_task, mock_group
|
||||
):
|
||||
provider_id_1 = uuid.uuid4()
|
||||
provider_id_2 = uuid.uuid4()
|
||||
scan_id_today_p1 = uuid.uuid4()
|
||||
scan_id_yesterday_p1 = uuid.uuid4()
|
||||
scan_id_today_p2 = uuid.uuid4()
|
||||
today = datetime.now(tz=timezone.utc)
|
||||
yesterday = today - timedelta(days=1)
|
||||
|
||||
scan_id_1 = uuid.uuid4()
|
||||
scan_id_2 = uuid.uuid4()
|
||||
mock_group_result = MagicMock()
|
||||
mock_group.side_effect = lambda gen: (list(gen), mock_group_result)[1]
|
||||
|
||||
mock_scan_filter.return_value.order_by.return_value.values.return_value = [
|
||||
{
|
||||
"id": scan_id_today_p1,
|
||||
"completed_at": today,
|
||||
"provider_id": provider_id_1,
|
||||
},
|
||||
{
|
||||
"id": scan_id_today_p2,
|
||||
"completed_at": today,
|
||||
"provider_id": provider_id_2,
|
||||
},
|
||||
{
|
||||
"id": scan_id_yesterday_p1,
|
||||
"completed_at": yesterday,
|
||||
"provider_id": provider_id_1,
|
||||
},
|
||||
mock_scan_filter.return_value.order_by.return_value.distinct.return_value.values_list.return_value = [
|
||||
scan_id_1,
|
||||
scan_id_2,
|
||||
]
|
||||
|
||||
result = reaggregate_all_finding_group_summaries_task(tenant_id=self.tenant_id)
|
||||
|
||||
assert result == {"scans_reaggregated": 3}
|
||||
assert mock_agg_task.si.call_count == 3
|
||||
assert result == {"scans_reaggregated": 2}
|
||||
assert mock_agg_task.si.call_count == 2
|
||||
mock_agg_task.si.assert_any_call(
|
||||
tenant_id=self.tenant_id, scan_id=str(scan_id_today_p1)
|
||||
tenant_id=self.tenant_id, scan_id=str(scan_id_1)
|
||||
)
|
||||
mock_agg_task.si.assert_any_call(
|
||||
tenant_id=self.tenant_id, scan_id=str(scan_id_today_p2)
|
||||
)
|
||||
mock_agg_task.si.assert_any_call(
|
||||
tenant_id=self.tenant_id, scan_id=str(scan_id_yesterday_p1)
|
||||
)
|
||||
mock_group_result.apply_async.assert_called_once()
|
||||
|
||||
@patch("tasks.tasks.group")
|
||||
@patch("tasks.tasks.aggregate_finding_group_summaries_task")
|
||||
@patch("tasks.tasks.Scan.objects.filter")
|
||||
def test_dedupes_scans_to_latest_per_provider_per_day(
|
||||
self, mock_scan_filter, mock_agg_task, mock_group
|
||||
):
|
||||
"""When several scans run on the same day for the same provider, only
|
||||
the latest one is dispatched (matching the daily summary unique key)."""
|
||||
provider_id = uuid.uuid4()
|
||||
latest_scan_today = uuid.uuid4()
|
||||
earlier_scan_today = uuid.uuid4()
|
||||
today_late = datetime.now(tz=timezone.utc)
|
||||
today_early = today_late - timedelta(hours=4)
|
||||
|
||||
mock_group_result = MagicMock()
|
||||
mock_group.side_effect = lambda gen: (list(gen), mock_group_result)[1]
|
||||
|
||||
# Returned ordered by `-completed_at`, so the most recent comes first.
|
||||
mock_scan_filter.return_value.order_by.return_value.values.return_value = [
|
||||
{
|
||||
"id": latest_scan_today,
|
||||
"completed_at": today_late,
|
||||
"provider_id": provider_id,
|
||||
},
|
||||
{
|
||||
"id": earlier_scan_today,
|
||||
"completed_at": today_early,
|
||||
"provider_id": provider_id,
|
||||
},
|
||||
]
|
||||
|
||||
result = reaggregate_all_finding_group_summaries_task(tenant_id=self.tenant_id)
|
||||
|
||||
assert result == {"scans_reaggregated": 1}
|
||||
mock_agg_task.si.assert_called_once_with(
|
||||
tenant_id=self.tenant_id, scan_id=str(latest_scan_today)
|
||||
tenant_id=self.tenant_id, scan_id=str(scan_id_2)
|
||||
)
|
||||
mock_group_result.apply_async.assert_called_once()
|
||||
|
||||
@patch("tasks.tasks.group")
|
||||
@patch("tasks.tasks.Scan.objects.filter")
|
||||
def test_no_completed_scans_skips_dispatch(self, mock_scan_filter, mock_group):
|
||||
mock_scan_filter.return_value.order_by.return_value.values.return_value = []
|
||||
mock_scan_filter.return_value.order_by.return_value.distinct.return_value.values_list.return_value = []
|
||||
|
||||
result = reaggregate_all_finding_group_summaries_task(tenant_id=self.tenant_id)
|
||||
|
||||
|
||||
@@ -34,10 +34,6 @@ spec:
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.worker.initContainers }}
|
||||
initContainers:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: worker
|
||||
{{- with .Values.worker.securityContext }}
|
||||
|
||||
@@ -32,10 +32,6 @@ spec:
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.worker_beat.initContainers }}
|
||||
initContainers:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: worker-beat
|
||||
{{- with .Values.worker_beat.securityContext }}
|
||||
|
||||
@@ -220,7 +220,7 @@ def _send_prowler_results(prowler_results, _prowler_version, options):
|
||||
try:
|
||||
_debug("RESULT MSG --- {0}".format(_check_result), 2)
|
||||
_check_result = json.loads(TEMPLATE_CHECK.format(_check_result))
|
||||
except Exception:
|
||||
except:
|
||||
_debug(
|
||||
"INVALID JSON --- {0}".format(TEMPLATE_CHECK.format(_check_result)), 1
|
||||
)
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
services:
|
||||
api-dev-init:
|
||||
image: busybox:1.37.0
|
||||
volumes:
|
||||
- ./_data/api:/data
|
||||
command: ["sh", "-c", "chown -R 1000:1000 /data"]
|
||||
restart: "no"
|
||||
|
||||
api-dev:
|
||||
hostname: "prowler-api"
|
||||
image: prowler-api-dev
|
||||
@@ -28,20 +21,12 @@ services:
|
||||
- ./_data/api:/home/prowler/.config/prowler-api
|
||||
- outputs:/tmp/prowler_api_output
|
||||
depends_on:
|
||||
api-dev-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
neo4j:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${DJANGO_PORT:-8080}/api/v1/ || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 60s
|
||||
entrypoint:
|
||||
- "/home/prowler/docker-entrypoint.sh"
|
||||
- "dev"
|
||||
@@ -154,7 +139,11 @@ services:
|
||||
- ./api/docker-entrypoint.sh:/home/prowler/docker-entrypoint.sh
|
||||
- outputs:/tmp/prowler_api_output
|
||||
depends_on:
|
||||
api-dev:
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
neo4j:
|
||||
condition: service_healthy
|
||||
ulimits:
|
||||
nofile:
|
||||
@@ -176,7 +165,11 @@ services:
|
||||
- path: ./.env
|
||||
required: false
|
||||
depends_on:
|
||||
api-dev:
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
neo4j:
|
||||
condition: service_healthy
|
||||
ulimits:
|
||||
nofile:
|
||||
|
||||
@@ -5,13 +5,6 @@
|
||||
# docker compose -f docker-compose-dev.yml up
|
||||
#
|
||||
services:
|
||||
api-init:
|
||||
image: busybox:1.37.0
|
||||
volumes:
|
||||
- ./_data/api:/data
|
||||
command: ["sh", "-c", "chown -R 1000:1000 /data"]
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
hostname: "prowler-api"
|
||||
image: prowlercloud/prowler-api:${PROWLER_API_VERSION:-stable}
|
||||
@@ -24,20 +17,12 @@ services:
|
||||
- ./_data/api:/home/prowler/.config/prowler-api
|
||||
- output:/tmp/prowler_api_output
|
||||
depends_on:
|
||||
api-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
neo4j:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${DJANGO_PORT:-8080}/api/v1/ || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 60s
|
||||
entrypoint:
|
||||
- "/home/prowler/docker-entrypoint.sh"
|
||||
- "prod"
|
||||
@@ -129,7 +114,9 @@ services:
|
||||
volumes:
|
||||
- "output:/tmp/prowler_api_output"
|
||||
depends_on:
|
||||
api:
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
ulimits:
|
||||
nofile:
|
||||
@@ -145,7 +132,9 @@ services:
|
||||
- path: ./.env
|
||||
required: false
|
||||
depends_on:
|
||||
api:
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
ulimits:
|
||||
nofile:
|
||||
|
||||
@@ -118,22 +118,18 @@ In case you have any doubts, consult the [Poetry environment activation guide](h
|
||||
|
||||
### Pre-Commit Hooks
|
||||
|
||||
This repository uses Git pre-commit hooks managed by the [prek](https://prek.j178.dev/) tool, it is installed with `poetry install --with dev`. Next, run the following command in the root of this repository:
|
||||
This repository uses Git pre-commit hooks managed by the [pre-commit](https://pre-commit.com/) tool, it is installed with `poetry install --with dev`. Next, run the following command in the root of this repository:
|
||||
|
||||
```shell
|
||||
prek install
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
Successful installation should produce the following output:
|
||||
|
||||
```shell
|
||||
prek installed at `.git/hooks/pre-commit`
|
||||
pre-commit installed at .git/hooks/pre-commit
|
||||
```
|
||||
|
||||
<Warning>
|
||||
If pre-commit hooks were previously installed, run `prek install --overwrite` to replace the existing hook. Otherwise, both tools will run on each commit.
|
||||
</Warning>
|
||||
|
||||
### Code Quality and Security Checks
|
||||
|
||||
Before merging pull requests, several automated checks and utilities ensure code security and updated dependencies:
|
||||
@@ -167,8 +163,6 @@ These resources help ensure that AI-assisted contributions maintain consistency
|
||||
|
||||
All dependencies are listed in the `pyproject.toml` file.
|
||||
|
||||
The SDK keeps direct dependencies pinned to exact versions, while `poetry.lock` records the full resolved dependency tree and the artifact hashes for every package. Use `poetry install` from the lock file instead of ad-hoc `pip` installs when you need a reproducible environment.
|
||||
|
||||
For proper code documentation, refer to the following and follow the code documentation practices presented there: [Google Python Style Guide - Comments and Docstrings](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings).
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -15,7 +15,8 @@ This document describes the internal architecture of Prowler Lighthouse AI, enab
|
||||
|
||||
Lighthouse AI operates as a Langchain-based agent that connects Large Language Models (LLMs) with Prowler security data through the Model Context Protocol (MCP).
|
||||
|
||||

|
||||
<img className="block dark:hidden" src="/images/lighthouse-architecture-light.png" alt="Prowler Lighthouse Architecture" />
|
||||
<img className="hidden dark:block" src="/images/lighthouse-architecture-dark.png" alt="Prowler Lighthouse Architecture" />
|
||||
|
||||
### Three-Tier Architecture
|
||||
|
||||
|
||||
@@ -12,24 +12,6 @@
|
||||
"dark": "/images/prowler-logo-white.png",
|
||||
"light": "/images/prowler-logo-black.png"
|
||||
},
|
||||
"contextual": {
|
||||
"options": [
|
||||
"copy",
|
||||
"view",
|
||||
{
|
||||
"title": "Request a feature",
|
||||
"description": "Open a feature request on GitHub",
|
||||
"icon": "plus",
|
||||
"href": "https://github.com/prowler-cloud/prowler/issues/new?template=feature-request.yml"
|
||||
},
|
||||
{
|
||||
"title": "Report an issue",
|
||||
"description": "Open a bug report on GitHub",
|
||||
"icon": "bug",
|
||||
"href": "https://github.com/prowler-cloud/prowler/issues/new?template=bug_report.yml"
|
||||
}
|
||||
]
|
||||
},
|
||||
"navigation": {
|
||||
"tabs": [
|
||||
{
|
||||
@@ -116,7 +98,6 @@
|
||||
]
|
||||
},
|
||||
"user-guide/tutorials/prowler-app-rbac",
|
||||
"user-guide/tutorials/prowler-app-multi-tenant",
|
||||
"user-guide/tutorials/prowler-app-api-keys",
|
||||
"user-guide/tutorials/prowler-app-import-findings",
|
||||
{
|
||||
@@ -151,7 +132,6 @@
|
||||
]
|
||||
},
|
||||
"user-guide/tutorials/prowler-app-attack-paths",
|
||||
"user-guide/tutorials/prowler-app-finding-groups",
|
||||
"user-guide/tutorials/prowler-cloud-public-ips",
|
||||
{
|
||||
"group": "Tutorials",
|
||||
|
||||
@@ -121,8 +121,8 @@ To update the environment file:
|
||||
Edit the `.env` file and change version values:
|
||||
|
||||
```env
|
||||
PROWLER_UI_VERSION="5.24.0"
|
||||
PROWLER_API_VERSION="5.24.0"
|
||||
PROWLER_UI_VERSION="5.22.0"
|
||||
PROWLER_API_VERSION="5.22.0"
|
||||
```
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -59,10 +59,6 @@ Prowler Lighthouse AI is powerful, but there are limitations:
|
||||
- **NextJS session dependence**: If your Prowler application session expires or logs out, Lighthouse AI will error out. Refresh and log back in to continue.
|
||||
- **Response quality**: The response quality depends on the selected LLM provider and model. Choose models with strong tool-calling capabilities for best results. We recommend `gpt-5` model from OpenAI.
|
||||
|
||||
## Architecture
|
||||
|
||||

|
||||
|
||||
## Extending Lighthouse AI
|
||||
|
||||
Lighthouse AI retrieves data through Prowler MCP. To add new capabilities, extend the Prowler MCP Server with additional tools and Lighthouse AI discovers them automatically.
|
||||
|
||||
@@ -46,7 +46,8 @@ Search and retrieve official Prowler documentation:
|
||||
|
||||
The following diagram illustrates the Prowler MCP Server architecture and its integration points:
|
||||
|
||||

|
||||
<img className="block dark:hidden" src="/images/prowler_mcp_schema_light.png" alt="Prowler MCP Server Schema" />
|
||||
<img className="hidden dark:block" src="/images/prowler_mcp_schema_dark.png" alt="Prowler MCP Server Schema" />
|
||||
|
||||
The architecture shows how AI assistants connect through the MCP protocol to access Prowler's three main components:
|
||||
- Prowler Cloud/App for security operations
|
||||
|
||||
|
Before Width: | Height: | Size: 755 KiB |
|
Before Width: | Height: | Size: 146 KiB |
|
Before Width: | Height: | Size: 340 KiB |
|
Before Width: | Height: | Size: 410 KiB |
|
After Width: | Height: | Size: 267 KiB |
|
After Width: | Height: | Size: 265 KiB |
@@ -1,37 +0,0 @@
|
||||
flowchart TB
|
||||
browser([Browser])
|
||||
|
||||
subgraph NEXTJS["Next.js Server"]
|
||||
route["API Route<br/>(auth + context assembly)"]
|
||||
agent["LangChain Agent"]
|
||||
|
||||
subgraph TOOLS["Agent Tools"]
|
||||
metatools["Meta-tools<br/>describe_tool / execute_tool / load_skill"]
|
||||
end
|
||||
|
||||
mcpclient["MCP Client<br/>(HTTP transport)"]
|
||||
end
|
||||
|
||||
llm["LLM Provider<br/>(OpenAI / Bedrock / OpenAI-compatible)"]
|
||||
|
||||
subgraph MCP["Prowler MCP Server"]
|
||||
app_tools["prowler_app_* tools<br/>(auth required)"]
|
||||
hub_tools["prowler_hub_* tools<br/>(no auth)"]
|
||||
docs_tools["prowler_docs_* tools<br/>(no auth)"]
|
||||
end
|
||||
|
||||
api["Prowler API"]
|
||||
hub["hub.prowler.com"]
|
||||
docs["docs.prowler.com<br/>(Mintlify)"]
|
||||
|
||||
browser <-->|SSE stream| route
|
||||
route --> agent
|
||||
agent <-->|LLM API| llm
|
||||
agent --> metatools
|
||||
metatools --> mcpclient
|
||||
mcpclient -->|MCP HTTP · Bearer token<br/>for prowler_app_* only| app_tools
|
||||
mcpclient -->|MCP HTTP| hub_tools
|
||||
mcpclient -->|MCP HTTP| docs_tools
|
||||
app_tools -->|REST| api
|
||||
hub_tools -->|REST| hub
|
||||
docs_tools -->|REST| docs
|
||||
|
Before Width: | Height: | Size: 286 KiB |
@@ -23,8 +23,6 @@ flowchart TB
|
||||
user --> ui
|
||||
user --> cli
|
||||
ui -->|REST| api
|
||||
ui -->|MCP HTTP| mcp
|
||||
mcp -->|REST| api
|
||||
api --> pg
|
||||
api --> valkey
|
||||
beat -->|enqueue jobs| valkey
|
||||
@@ -33,5 +31,7 @@ flowchart TB
|
||||
worker -->|Attack Paths| neo4j
|
||||
worker -->|invokes| sdk
|
||||
cli --> sdk
|
||||
api -. AI tools .-> mcp
|
||||
mcp -. context .-> api
|
||||
|
||||
sdk --> providers
|
||||
|
||||
|
Before Width: | Height: | Size: 348 KiB After Width: | Height: | Size: 268 KiB |
|
Before Width: | Height: | Size: 148 KiB |
|
Before Width: | Height: | Size: 81 KiB |
|
Before Width: | Height: | Size: 78 KiB |
|
Before Width: | Height: | Size: 90 KiB |