Compare commits

..

1 Commits

Author SHA1 Message Date
prowler-bot 5eee9ac2db feat(aws): update regions for AWS services 2025-12-15 09:04:48 +00:00
1907 changed files with 14592 additions and 74201 deletions
+2 -28
View File
@@ -15,13 +15,6 @@ AUTH_SECRET="N/c6mnaS5+SWq81+819OrzQZlmx1Vxtp/orjttJSmw8="
# Google Tag Manager ID
NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID=""
#### MCP Server ####
PROWLER_MCP_VERSION=stable
# For UI and MCP running on docker:
PROWLER_MCP_SERVER_URL=http://mcp-server:8000/mcp
# For UI running on host, MCP in docker:
# PROWLER_MCP_SERVER_URL=http://localhost:8000/mcp
#### Code Review Configuration ####
# Enable Claude Code standards validation on pre-push hook
# Set to 'true' to validate changes against AGENTS.md standards via Claude Code
@@ -48,26 +41,6 @@ POSTGRES_DB=prowler_db
# POSTGRES_REPLICA_MAX_ATTEMPTS=3
# POSTGRES_REPLICA_RETRY_BASE_DELAY=0.5
# Neo4j auth
NEO4J_HOST=neo4j
NEO4J_PORT=7687
NEO4J_USER=neo4j
NEO4J_PASSWORD=neo4j_password
# Neo4j settings
NEO4J_DBMS_MAX__DATABASES=1000000
NEO4J_SERVER_MEMORY_PAGECACHE_SIZE=1G
NEO4J_SERVER_MEMORY_HEAP_INITIAL__SIZE=1G
NEO4J_SERVER_MEMORY_HEAP_MAX__SIZE=1G
NEO4J_POC_EXPORT_FILE_ENABLED=true
NEO4J_APOC_IMPORT_FILE_ENABLED=true
NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG=true
NEO4J_PLUGINS=["apoc"]
NEO4J_DBMS_SECURITY_PROCEDURES_ALLOWLIST=apoc.*
NEO4J_DBMS_SECURITY_PROCEDURES_UNRESTRICTED=apoc.*
NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS=0.0.0.0:7687
# Neo4j Prowler settings
NEO4J_INSERT_BATCH_SIZE=500
# Celery-Prowler task settings
TASK_RETRY_DELAY_SECONDS=0.1
TASK_RETRY_ATTEMPTS=5
@@ -137,8 +110,9 @@ SENTRY_ENVIRONMENT=local
SENTRY_RELEASE=local
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
#### Prowler release version ####
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.16.0
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.12.2
# Social login credentials
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
-18
View File
@@ -47,16 +47,6 @@ provider/oci:
- any-glob-to-any-file: "prowler/providers/oraclecloud/**"
- any-glob-to-any-file: "tests/providers/oraclecloud/**"
provider/alibabacloud:
- changed-files:
- any-glob-to-any-file: "prowler/providers/alibabacloud/**"
- any-glob-to-any-file: "tests/providers/alibabacloud/**"
provider/cloudflare:
- changed-files:
- any-glob-to-any-file: "prowler/providers/cloudflare/**"
- any-glob-to-any-file: "tests/providers/cloudflare/**"
github_actions:
- changed-files:
- any-glob-to-any-file: ".github/workflows/*"
@@ -72,21 +62,13 @@ mutelist:
- any-glob-to-any-file: "prowler/providers/azure/lib/mutelist/**"
- any-glob-to-any-file: "prowler/providers/gcp/lib/mutelist/**"
- any-glob-to-any-file: "prowler/providers/kubernetes/lib/mutelist/**"
- any-glob-to-any-file: "prowler/providers/m365/lib/mutelist/**"
- any-glob-to-any-file: "prowler/providers/mongodbatlas/lib/mutelist/**"
- any-glob-to-any-file: "prowler/providers/oraclecloud/lib/mutelist/**"
- any-glob-to-any-file: "prowler/providers/alibabacloud/lib/mutelist/**"
- any-glob-to-any-file: "prowler/providers/cloudflare/lib/mutelist/**"
- any-glob-to-any-file: "tests/lib/mutelist/**"
- any-glob-to-any-file: "tests/providers/aws/lib/mutelist/**"
- any-glob-to-any-file: "tests/providers/azure/lib/mutelist/**"
- any-glob-to-any-file: "tests/providers/gcp/lib/mutelist/**"
- any-glob-to-any-file: "tests/providers/kubernetes/lib/mutelist/**"
- any-glob-to-any-file: "tests/providers/m365/lib/mutelist/**"
- any-glob-to-any-file: "tests/providers/mongodbatlas/lib/mutelist/**"
- any-glob-to-any-file: "tests/providers/oraclecloud/lib/mutelist/**"
- any-glob-to-any-file: "tests/providers/alibabacloud/lib/mutelist/**"
- any-glob-to-any-file: "tests/providers/cloudflare/lib/mutelist/**"
integration/s3:
- changed-files:
+2 -19
View File
@@ -14,26 +14,14 @@ Please add a detailed description of how to review this PR.
### Checklist
<details>
<summary><b>Community Checklist</b></summary>
- [ ] This feature/issue is listed in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
- [ ] Is it assigned to me, if not, request it via the issue/feature in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](goto.prowler.com/slack)
</details>
- Are there new checks included in this PR? Yes / No
- If so, do we need to update permissions for the provider? Please review this carefully.
- [ ] Review if the code is being covered by tests.
- [ ] Review if code is being documented following this specification https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings
- [ ] Review if backport is needed.
- [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable.
#### SDK/CLI
- Are there new checks included in this PR? Yes / No
- If so, do we need to update permissions for the provider? Please review this carefully.
#### UI
- [ ] All issue/task requirements work as expected on the UI
- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px)
@@ -42,11 +30,6 @@ Please add a detailed description of how to review this PR.
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable.
#### API
- [ ] All issue/task requirements work as expected on the API
- [ ] Endpoint response output (if applicable)
- [ ] EXPLAIN ANALYZE output for new/modified queries or indexes (if applicable)
- [ ] Performance test results (if applicable)
- [ ] Any other relevant evidence of the implementation (if applicable)
- [ ] Verify if API specs need to be regenerated.
- [ ] Check if version updates are required (e.g., specs, Poetry, etc.).
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable.
-254
View File
@@ -1,254 +0,0 @@
name: 'API: Bump Version'
on:
release:
types:
- 'published'
concurrency:
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
cancel-in-progress: false
env:
PROWLER_VERSION: ${{ github.event.release.tag_name }}
BASE_BRANCH: master
jobs:
detect-release-type:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
is_minor: ${{ steps.detect.outputs.is_minor }}
is_patch: ${{ steps.detect.outputs.is_patch }}
major_version: ${{ steps.detect.outputs.major_version }}
minor_version: ${{ steps.detect.outputs.minor_version }}
patch_version: ${{ steps.detect.outputs.patch_version }}
current_api_version: ${{ steps.get_api_version.outputs.current_api_version }}
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Get current API version
id: get_api_version
run: |
CURRENT_API_VERSION=$(grep -oP '^version = "\K[^"]+' api/pyproject.toml)
echo "current_api_version=${CURRENT_API_VERSION}" >> "${GITHUB_OUTPUT}"
echo "Current API version: $CURRENT_API_VERSION"
- name: Detect release type and parse version
id: detect
run: |
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
MAJOR_VERSION=${BASH_REMATCH[1]}
MINOR_VERSION=${BASH_REMATCH[2]}
PATCH_VERSION=${BASH_REMATCH[3]}
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
if (( MAJOR_VERSION != 5 )); then
echo "::error::Releasing another Prowler major version, aborting..."
exit 1
fi
if (( PATCH_VERSION == 0 )); then
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
echo "✓ Minor release detected: $PROWLER_VERSION"
else
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
echo "✓ Patch release detected: $PROWLER_VERSION"
fi
else
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
exit 1
fi
bump-minor-version:
needs: detect-release-type
if: needs.detect-release-type.outputs.is_minor == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Calculate next API minor version
run: |
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}"
# API version follows Prowler minor + 1
# For Prowler 5.17.0 -> API 1.18.0
# For next master (Prowler 5.18.0) -> API 1.19.0
NEXT_API_VERSION=1.$((MINOR_VERSION + 2)).0
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
echo "NEXT_API_VERSION=${NEXT_API_VERSION}" >> "${GITHUB_ENV}"
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0"
echo "Current API version: $CURRENT_API_VERSION"
echo "Next API minor version (for master): $NEXT_API_VERSION"
- name: Bump API versions in files for master
run: |
set -e
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_VERSION}\"|" api/pyproject.toml
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_VERSION}\"|" api/src/backend/api/v1/views.py
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_VERSION}|" api/src/backend/api/specs/v1.yaml
echo "Files modified:"
git --no-pager diff
- name: Create PR for next API minor version to master
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
base: master
commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}'
branch: api-version-bump-to-v${{ env.NEXT_API_VERSION }}
title: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}'
labels: no-changelog,skip-sync
body: |
### Description
Bump Prowler API version to v${{ env.NEXT_API_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
### License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
- name: Checkout version branch
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
- name: Calculate first API patch version
run: |
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}"
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
# API version follows Prowler minor + 1
# For Prowler 5.17.0 release -> version branch v5.17 should have API 1.18.1
FIRST_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).1
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
echo "FIRST_API_PATCH_VERSION=${FIRST_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0"
echo "First API patch version (for ${VERSION_BRANCH}): $FIRST_API_PATCH_VERSION"
echo "Version branch: $VERSION_BRANCH"
- name: Bump API versions in files for version branch
run: |
set -e
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${FIRST_API_PATCH_VERSION}\"|" api/pyproject.toml
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${FIRST_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${FIRST_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
echo "Files modified:"
git --no-pager diff
- name: Create PR for first API patch version to version branch
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
base: ${{ env.VERSION_BRANCH }}
commit-message: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}'
branch: api-version-bump-to-v${{ env.FIRST_API_PATCH_VERSION }}
title: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}'
labels: no-changelog,skip-sync
body: |
### Description
Bump Prowler API version to v${{ env.FIRST_API_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
### License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
bump-patch-version:
needs: detect-release-type
if: needs.detect-release-type.outputs.is_patch == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Calculate next API patch version
run: |
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }}
CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}"
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
# Extract current API patch to increment it
if [[ $CURRENT_API_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
API_PATCH=${BASH_REMATCH[3]}
# API version follows Prowler minor + 1
# Keep same API minor (based on Prowler minor), increment patch
NEXT_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).$((API_PATCH + 1))
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
echo "NEXT_API_PATCH_VERSION=${NEXT_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}"
echo "Current API version: $CURRENT_API_VERSION"
echo "Next API patch version: $NEXT_API_PATCH_VERSION"
echo "Target branch: $VERSION_BRANCH"
else
echo "::error::Invalid API version format: $CURRENT_API_VERSION"
exit 1
fi
- name: Bump API versions in files for version branch
run: |
set -e
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_PATCH_VERSION}\"|" api/pyproject.toml
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
echo "Files modified:"
git --no-pager diff
- name: Create PR for next API patch version to version branch
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
base: ${{ env.VERSION_BRANCH }}
commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}'
branch: api-version-bump-to-v${{ env.NEXT_API_PATCH_VERSION }}
title: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}'
labels: no-changelog,skip-sync
body: |
### Description
Bump Prowler API version to v${{ env.NEXT_API_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
### License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
+2 -3
View File
@@ -33,11 +33,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for API changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
api/**
@@ -46,7 +46,6 @@ jobs:
api/docs/**
api/README.md
api/CHANGELOG.md
api/AGENTS.md
- name: Setup Python with Poetry
if: steps.check-changes.outputs.any_changed == 'true'
+3 -3
View File
@@ -42,15 +42,15 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Initialize CodeQL
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
with:
languages: ${{ matrix.language }}
config-file: ./.github/codeql/api-codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
with:
category: '/language:${{ matrix.language }}'
+11 -11
View File
@@ -3,7 +3,7 @@ name: 'API: Container Build and Push'
on:
push:
branches:
- 'attack-paths-demo'
- 'master'
paths:
- 'api/**'
- 'prowler/**'
@@ -27,7 +27,7 @@ concurrency:
env:
# Tags
LATEST_TAG: attack-paths-demo
LATEST_TAG: latest
RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }}
STABLE_TAG: stable
WORKING_DIRECTORY: ./api
@@ -57,7 +57,7 @@ jobs:
message-ts: ${{ steps.slack-notification.outputs.ts }}
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Notify container push started
id: slack-notification
@@ -93,7 +93,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Login to DockerHub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
@@ -102,7 +102,7 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build and push API container for ${{ matrix.arch }}
id: container-push
@@ -120,18 +120,18 @@ jobs:
# Create and push multi-architecture manifest
create-manifest:
needs: [setup, container-build-push]
if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Create and push manifests for push event
if: github.event_name == 'push'
@@ -170,7 +170,7 @@ jobs:
timeout-minutes: 5
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Determine overall outcome
id: outcome
@@ -198,8 +198,8 @@ jobs:
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
trigger-deployment:
if: github.event_name == 'push'
needs: [setup, container-build-push]
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
@@ -207,7 +207,7 @@ jobs:
steps:
- name: Trigger API deployment
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
repository: ${{ secrets.CLOUD_DISPATCH }}
+6 -9
View File
@@ -20,7 +20,6 @@ env:
jobs:
api-dockerfile-lint:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
@@ -28,11 +27,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check if Dockerfile changed
id: dockerfile-changed
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: api/Dockerfile
@@ -44,7 +43,6 @@ jobs:
ignore: DL3013
api-container-build-and-scan:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ${{ matrix.runner }}
strategy:
matrix:
@@ -63,22 +61,21 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for API changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: api/**
files_ignore: |
api/docs/**
api/README.md
api/CHANGELOG.md
api/AGENTS.md
- name: Set up Docker Buildx
if: steps.check-changes.outputs.any_changed == 'true'
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build container for ${{ matrix.arch }}
if: steps.check-changes.outputs.any_changed == 'true'
@@ -93,7 +90,7 @@ jobs:
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
- name: Scan container with Trivy for ${{ matrix.arch }}
if: steps.check-changes.outputs.any_changed == 'true'
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
uses: ./.github/actions/trivy-scan
with:
image-name: ${{ env.IMAGE_NAME }}
+5 -4
View File
@@ -33,11 +33,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for API changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
api/**
@@ -46,7 +46,6 @@ jobs:
api/docs/**
api/README.md
api/CHANGELOG.md
api/AGENTS.md
- name: Setup Python with Poetry
if: steps.check-changes.outputs.any_changed == 'true'
@@ -61,7 +60,9 @@ jobs:
- name: Safety
if: steps.check-changes.outputs.any_changed == 'true'
run: poetry run safety check
# 76352, 76353, 77323 come from SDK, but they cannot upgrade it yet. It does not affect API
# TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X
run: poetry run safety check --ignore 70612,66963,74429,76352,76353,77323,77744,77745
- name: Vulture
if: steps.check-changes.outputs.any_changed == 'true'
+3 -4
View File
@@ -73,11 +73,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for API changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
api/**
@@ -86,7 +86,6 @@ jobs:
api/docs/**
api/README.md
api/CHANGELOG.md
api/AGENTS.md
- name: Setup Python with Poetry
if: steps.check-changes.outputs.any_changed == 'true'
@@ -101,7 +100,7 @@ jobs:
- name: Upload coverage reports to Codecov
if: steps.check-changes.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
-247
View File
@@ -1,247 +0,0 @@
name: 'Docs: Bump Version'
on:
release:
types:
- 'published'
concurrency:
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
cancel-in-progress: false
env:
PROWLER_VERSION: ${{ github.event.release.tag_name }}
BASE_BRANCH: master
jobs:
detect-release-type:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
is_minor: ${{ steps.detect.outputs.is_minor }}
is_patch: ${{ steps.detect.outputs.is_patch }}
major_version: ${{ steps.detect.outputs.major_version }}
minor_version: ${{ steps.detect.outputs.minor_version }}
patch_version: ${{ steps.detect.outputs.patch_version }}
current_docs_version: ${{ steps.get_docs_version.outputs.current_docs_version }}
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Get current documentation version
id: get_docs_version
run: |
CURRENT_DOCS_VERSION=$(grep -oP 'PROWLER_UI_VERSION="\K[^"]+' docs/getting-started/installation/prowler-app.mdx)
echo "current_docs_version=${CURRENT_DOCS_VERSION}" >> "${GITHUB_OUTPUT}"
echo "Current documentation version: $CURRENT_DOCS_VERSION"
- name: Detect release type and parse version
id: detect
run: |
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
MAJOR_VERSION=${BASH_REMATCH[1]}
MINOR_VERSION=${BASH_REMATCH[2]}
PATCH_VERSION=${BASH_REMATCH[3]}
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
if (( MAJOR_VERSION != 5 )); then
echo "::error::Releasing another Prowler major version, aborting..."
exit 1
fi
if (( PATCH_VERSION == 0 )); then
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
echo "✓ Minor release detected: $PROWLER_VERSION"
else
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
echo "✓ Patch release detected: $PROWLER_VERSION"
fi
else
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
exit 1
fi
bump-minor-version:
needs: detect-release-type
if: needs.detect-release-type.outputs.is_minor == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Calculate next minor version
run: |
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}"
NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}"
echo "Current documentation version: $CURRENT_DOCS_VERSION"
echo "Current release version: $PROWLER_VERSION"
echo "Next minor version: $NEXT_MINOR_VERSION"
- name: Bump versions in documentation for master
run: |
set -e
# Update prowler-app.mdx with current release version
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
echo "Files modified:"
git --no-pager diff
- name: Create PR for documentation update to master
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
base: master
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
labels: no-changelog,skip-sync
body: |
### Description
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
### Files Updated
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
- All `*.mdx` files with `<VersionBadge>` components
### License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
- name: Checkout version branch
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
- name: Calculate first patch version
run: |
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}"
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
echo "First patch version: $FIRST_PATCH_VERSION"
echo "Version branch: $VERSION_BRANCH"
- name: Bump versions in documentation for version branch
run: |
set -e
# Update prowler-app.mdx with current release version
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
echo "Files modified:"
git --no-pager diff
- name: Create PR for documentation update to version branch
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
base: ${{ env.VERSION_BRANCH }}
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}-branch
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
labels: no-changelog,skip-sync
body: |
### Description
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
### Files Updated
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
### License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
bump-patch-version:
needs: detect-release-type
if: needs.detect-release-type.outputs.is_patch == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Calculate next patch version
run: |
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }}
CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}"
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
echo "Current documentation version: $CURRENT_DOCS_VERSION"
echo "Current release version: $PROWLER_VERSION"
echo "Next patch version: $NEXT_PATCH_VERSION"
echo "Target branch: $VERSION_BRANCH"
- name: Bump versions in documentation for patch version
run: |
set -e
# Update prowler-app.mdx with current release version
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
echo "Files modified:"
git --no-pager diff
- name: Create PR for documentation update to version branch
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
base: ${{ env.VERSION_BRANCH }}
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
labels: no-changelog,skip-sync
body: |
### Description
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
### Files Updated
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
### License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
+2 -2
View File
@@ -23,11 +23,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
fetch-depth: 0
- name: Scan for secrets with TruffleHog
uses: trufflesecurity/trufflehog@ef6e76c3c4023279497fab4721ffa071a722fd05 # v3.92.4
uses: trufflesecurity/trufflehog@b84c3d14d189e16da175e2c27fa8136603783ffc # v3.90.12
with:
extra_args: '--results=verified,unknown'
@@ -56,7 +56,7 @@ jobs:
message-ts: ${{ steps.slack-notification.outputs.ts }}
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Notify container push started
id: slack-notification
@@ -91,7 +91,7 @@ jobs:
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Login to DockerHub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
@@ -100,7 +100,7 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build and push MCP container for ${{ matrix.arch }}
id: container-push
@@ -126,18 +126,18 @@ jobs:
# Create and push multi-architecture manifest
create-manifest:
needs: [setup, container-build-push]
if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Create and push manifests for push event
if: github.event_name == 'push'
@@ -176,7 +176,7 @@ jobs:
timeout-minutes: 5
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Determine overall outcome
id: outcome
@@ -204,8 +204,8 @@ jobs:
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
trigger-deployment:
if: github.event_name == 'push'
needs: [setup, container-build-push]
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
@@ -213,7 +213,7 @@ jobs:
steps:
- name: Trigger MCP deployment
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
repository: ${{ secrets.CLOUD_DISPATCH }}
+6 -8
View File
@@ -20,7 +20,6 @@ env:
jobs:
mcp-dockerfile-lint:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
@@ -28,11 +27,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check if Dockerfile changed
id: dockerfile-changed
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: mcp_server/Dockerfile
@@ -43,7 +42,6 @@ jobs:
dockerfile: mcp_server/Dockerfile
mcp-container-build-and-scan:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ${{ matrix.runner }}
strategy:
matrix:
@@ -62,11 +60,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for MCP changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: mcp_server/**
files_ignore: |
@@ -75,7 +73,7 @@ jobs:
- name: Set up Docker Buildx
if: steps.check-changes.outputs.any_changed == 'true'
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build MCP container for ${{ matrix.arch }}
if: steps.check-changes.outputs.any_changed == 'true'
@@ -90,7 +88,7 @@ jobs:
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
- name: Scan MCP container with Trivy for ${{ matrix.arch }}
if: steps.check-changes.outputs.any_changed == 'true'
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
uses: ./.github/actions/trivy-scan
with:
image-name: ${{ env.IMAGE_NAME }}
-81
View File
@@ -1,81 +0,0 @@
name: "MCP: PyPI Release"
on:
release:
types:
- "published"
concurrency:
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
cancel-in-progress: false
env:
RELEASE_TAG: ${{ github.event.release.tag_name }}
PYTHON_VERSION: "3.12"
WORKING_DIRECTORY: ./mcp_server
jobs:
validate-release:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
prowler_version: ${{ steps.parse-version.outputs.version }}
major_version: ${{ steps.parse-version.outputs.major }}
steps:
- name: Parse and validate version
id: parse-version
run: |
PROWLER_VERSION="${{ env.RELEASE_TAG }}"
echo "version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}"
# Extract major version
MAJOR_VERSION="${PROWLER_VERSION%%.*}"
echo "major=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
# Validate major version (only Prowler 3, 4, 5 supported)
case ${MAJOR_VERSION} in
3|4|5)
echo "✓ Releasing Prowler MCP for tag ${PROWLER_VERSION}"
;;
*)
echo "::error::Unsupported Prowler major version: ${MAJOR_VERSION}"
exit 1
;;
esac
publish-prowler-mcp:
needs: validate-release
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
id-token: write
environment:
name: pypi-prowler-mcp
url: https://pypi.org/project/prowler-mcp/
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Install uv
uses: astral-sh/setup-uv@v7
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Build prowler-mcp package
working-directory: ${{ env.WORKING_DIRECTORY }}
run: uv build
- name: Publish prowler-mcp package to PyPI
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
with:
packages-dir: ${{ env.WORKING_DIRECTORY }}/dist/
print-hash: true
+2 -2
View File
@@ -29,13 +29,13 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
fetch-depth: 0
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
api/**
+2 -2
View File
@@ -25,14 +25,14 @@ jobs:
steps:
- name: Checkout PR head
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: '**'
+2 -5
View File
@@ -13,10 +13,7 @@ concurrency:
jobs:
trigger-cloud-pull-request:
if: |
github.event.pull_request.merged == true &&
github.repository == 'prowler-cloud/prowler' &&
!contains(github.event.pull_request.labels.*.name, 'skip-sync')
if: github.event.pull_request.merged == true && github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
@@ -29,7 +26,7 @@ jobs:
echo "SHORT_SHA=${SHORT_SHA::7}" >> $GITHUB_ENV
- name: Trigger Cloud repository pull request
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
repository: ${{ secrets.CLOUD_DISPATCH }}
+4 -4
View File
@@ -27,13 +27,13 @@ jobs:
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
fetch-depth: 0
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
- name: Set up Python
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: '3.12'
@@ -344,7 +344,7 @@ jobs:
- name: Create PR for API dependency update
if: ${{ env.PATCH_VERSION == '0' }}
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
commit-message: 'chore(api): update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}'
@@ -374,7 +374,7 @@ jobs:
no-changelog
- name: Create draft release
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0
uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2.4.1
with:
tag_name: ${{ env.PROWLER_VERSION }}
name: Prowler ${{ env.PROWLER_VERSION }}
+12 -9
View File
@@ -67,7 +67,7 @@ jobs:
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Calculate next minor version
run: |
@@ -86,12 +86,13 @@ jobs:
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_MINOR_VERSION}\"|" pyproject.toml
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_MINOR_VERSION}\"|" prowler/config/config.py
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env
echo "Files modified:"
git --no-pager diff
- name: Create PR for next minor version to master
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
@@ -99,7 +100,7 @@ jobs:
commit-message: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
branch: version-bump-to-v${{ env.NEXT_MINOR_VERSION }}
title: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
labels: no-changelog,skip-sync
labels: no-changelog
body: |
### Description
@@ -110,7 +111,7 @@ jobs:
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
- name: Checkout version branch
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
@@ -134,12 +135,13 @@ jobs:
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${FIRST_PATCH_VERSION}\"|" pyproject.toml
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${FIRST_PATCH_VERSION}\"|" prowler/config/config.py
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env
echo "Files modified:"
git --no-pager diff
- name: Create PR for first patch version to version branch
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
@@ -147,7 +149,7 @@ jobs:
commit-message: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
branch: version-bump-to-v${{ env.FIRST_PATCH_VERSION }}
title: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
labels: no-changelog,skip-sync
labels: no-changelog
body: |
### Description
@@ -167,7 +169,7 @@ jobs:
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Calculate next patch version
run: |
@@ -191,12 +193,13 @@ jobs:
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_PATCH_VERSION}\"|" pyproject.toml
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_PATCH_VERSION}\"|" prowler/config/config.py
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env
echo "Files modified:"
git --no-pager diff
- name: Create PR for next patch version to version branch
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
@@ -204,7 +207,7 @@ jobs:
commit-message: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
branch: version-bump-to-v${{ env.NEXT_PATCH_VERSION }}
title: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
labels: no-changelog,skip-sync
labels: no-changelog
body: |
### Description
+5 -7
View File
@@ -31,11 +31,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for SDK changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ./**
files_ignore: |
@@ -47,7 +47,6 @@ jobs:
ui/**
dashboard/**
mcp_server/**
skills/**
README.md
mkdocs.yml
.backportrc.json
@@ -56,7 +55,6 @@ jobs:
examples/**
.gitignore
contrib/**
**/AGENTS.md
- name: Install Poetry
if: steps.check-changes.outputs.any_changed == 'true'
@@ -64,7 +62,7 @@ jobs:
- name: Set up Python ${{ matrix.python-version }}
if: steps.check-changes.outputs.any_changed == 'true'
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: ${{ matrix.python-version }}
cache: 'poetry'
@@ -81,11 +79,11 @@ jobs:
- name: Lint with flake8
if: steps.check-changes.outputs.any_changed == 'true'
run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api,skills
run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api
- name: Check format with black
if: steps.check-changes.outputs.any_changed == 'true'
run: poetry run black --exclude "api|ui|skills" --check .
run: poetry run black --exclude api ui --check .
- name: Lint with pylint
if: steps.check-changes.outputs.any_changed == 'true'
+3 -3
View File
@@ -49,15 +49,15 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Initialize CodeQL
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
with:
languages: ${{ matrix.language }}
config-file: ./.github/codeql/sdk-codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
with:
category: '/language:${{ matrix.language }}'
+13 -13
View File
@@ -61,10 +61,10 @@ jobs:
stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }}
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: ${{ env.PYTHON_VERSION }}
@@ -115,7 +115,7 @@ jobs:
message-ts: ${{ steps.slack-notification.outputs.ts }}
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Notify container push started
id: slack-notification
@@ -151,7 +151,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Login to DockerHub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
@@ -169,7 +169,7 @@ jobs:
AWS_REGION: ${{ env.AWS_REGION }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build and push SDK container for ${{ matrix.arch }}
id: container-push
@@ -188,18 +188,18 @@ jobs:
# Create and push multi-architecture manifest
create-manifest:
needs: [setup, container-build-push]
if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Public ECR
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
registry: public.ecr.aws
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
@@ -208,7 +208,7 @@ jobs:
AWS_REGION: ${{ env.AWS_REGION }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Create and push manifests for push event
if: github.event_name == 'push'
@@ -252,7 +252,7 @@ jobs:
timeout-minutes: 5
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Determine overall outcome
id: outcome
@@ -280,8 +280,8 @@ jobs:
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
dispatch-v3-deployment:
if: needs.setup.outputs.prowler_version_major == '3'
needs: [setup, container-build-push]
if: always() && needs.setup.outputs.prowler_version_major == '3' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
@@ -294,7 +294,7 @@ jobs:
- name: Dispatch v3 deployment (latest)
if: github.event_name == 'push'
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
@@ -303,7 +303,7 @@ jobs:
- name: Dispatch v3 deployment (release)
if: github.event_name == 'release'
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
+6 -8
View File
@@ -27,11 +27,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check if Dockerfile changed
id: dockerfile-changed
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: Dockerfile
@@ -62,11 +62,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for SDK changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ./**
files_ignore: |
@@ -78,7 +78,6 @@ jobs:
ui/**
dashboard/**
mcp_server/**
skills/**
README.md
mkdocs.yml
.backportrc.json
@@ -87,11 +86,10 @@ jobs:
examples/**
.gitignore
contrib/**
**/AGENTS.md
- name: Set up Docker Buildx
if: steps.check-changes.outputs.any_changed == 'true'
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build SDK container for ${{ matrix.arch }}
if: steps.check-changes.outputs.any_changed == 'true'
@@ -106,7 +104,7 @@ jobs:
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
- name: Scan SDK container with Trivy for ${{ matrix.arch }}
if: steps.check-changes.outputs.any_changed == 'true'
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
uses: ./.github/actions/trivy-scan
with:
image-name: ${{ env.IMAGE_NAME }}
+4 -4
View File
@@ -59,13 +59,13 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Install Poetry
run: pipx install poetry==2.1.1
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'poetry'
@@ -91,13 +91,13 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Install Poetry
run: pipx install poetry==2.1.1
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'poetry'
@@ -25,12 +25,12 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
ref: 'master'
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
@@ -39,7 +39,7 @@ jobs:
run: pip install boto3
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
with:
aws-region: ${{ env.AWS_REGION }}
role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }}
@@ -50,7 +50,7 @@ jobs:
- name: Create pull request
id: create-pr
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
author: 'prowler-bot <179230569+prowler-bot@users.noreply.github.com>'
+5 -9
View File
@@ -24,15 +24,13 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for SDK changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files:
./**
.github/workflows/sdk-security.yml
files: ./**
files_ignore: |
.github/**
prowler/CHANGELOG.md
@@ -42,7 +40,6 @@ jobs:
ui/**
dashboard/**
mcp_server/**
skills/**
README.md
mkdocs.yml
.backportrc.json
@@ -51,7 +48,6 @@ jobs:
examples/**
.gitignore
contrib/**
**/AGENTS.md
- name: Install Poetry
if: steps.check-changes.outputs.any_changed == 'true'
@@ -59,7 +55,7 @@ jobs:
- name: Set up Python 3.12
if: steps.check-changes.outputs.any_changed == 'true'
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: '3.12'
cache: 'poetry'
@@ -74,7 +70,7 @@ jobs:
- name: Security scan with Safety
if: steps.check-changes.outputs.any_changed == 'true'
run: poetry run safety check -r pyproject.toml
run: poetry run safety check --ignore 70612 -r pyproject.toml
- name: Dead code detection with Vulture
if: steps.check-changes.outputs.any_changed == 'true'
+27 -29
View File
@@ -31,11 +31,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for SDK changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ./**
files_ignore: |
@@ -47,7 +47,6 @@ jobs:
ui/**
dashboard/**
mcp_server/**
skills/**
README.md
mkdocs.yml
.backportrc.json
@@ -56,7 +55,6 @@ jobs:
examples/**
.gitignore
contrib/**
**/AGENTS.md
- name: Install Poetry
if: steps.check-changes.outputs.any_changed == 'true'
@@ -64,7 +62,7 @@ jobs:
- name: Set up Python ${{ matrix.python-version }}
if: steps.check-changes.outputs.any_changed == 'true'
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: ${{ matrix.python-version }}
cache: 'poetry'
@@ -77,7 +75,7 @@ jobs:
- name: Check if AWS files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-aws
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/aws/**
@@ -191,7 +189,7 @@ jobs:
- name: Upload AWS coverage to Codecov
if: steps.changed-aws.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -202,7 +200,7 @@ jobs:
- name: Check if Azure files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-azure
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/azure/**
@@ -215,7 +213,7 @@ jobs:
- name: Upload Azure coverage to Codecov
if: steps.changed-azure.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -226,7 +224,7 @@ jobs:
- name: Check if GCP files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-gcp
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/gcp/**
@@ -239,7 +237,7 @@ jobs:
- name: Upload GCP coverage to Codecov
if: steps.changed-gcp.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -250,7 +248,7 @@ jobs:
- name: Check if Kubernetes files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-kubernetes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/kubernetes/**
@@ -263,7 +261,7 @@ jobs:
- name: Upload Kubernetes coverage to Codecov
if: steps.changed-kubernetes.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -274,7 +272,7 @@ jobs:
- name: Check if GitHub files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-github
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/github/**
@@ -287,7 +285,7 @@ jobs:
- name: Upload GitHub coverage to Codecov
if: steps.changed-github.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -298,7 +296,7 @@ jobs:
- name: Check if NHN files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-nhn
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/nhn/**
@@ -311,7 +309,7 @@ jobs:
- name: Upload NHN coverage to Codecov
if: steps.changed-nhn.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -322,7 +320,7 @@ jobs:
- name: Check if M365 files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-m365
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/m365/**
@@ -335,7 +333,7 @@ jobs:
- name: Upload M365 coverage to Codecov
if: steps.changed-m365.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -346,7 +344,7 @@ jobs:
- name: Check if IaC files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-iac
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/iac/**
@@ -359,7 +357,7 @@ jobs:
- name: Upload IaC coverage to Codecov
if: steps.changed-iac.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -370,7 +368,7 @@ jobs:
- name: Check if MongoDB Atlas files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-mongodbatlas
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/mongodbatlas/**
@@ -383,7 +381,7 @@ jobs:
- name: Upload MongoDB Atlas coverage to Codecov
if: steps.changed-mongodbatlas.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -394,7 +392,7 @@ jobs:
- name: Check if OCI files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-oraclecloud
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/**/oraclecloud/**
@@ -407,7 +405,7 @@ jobs:
- name: Upload OCI coverage to Codecov
if: steps.changed-oraclecloud.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -418,7 +416,7 @@ jobs:
- name: Check if Lib files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-lib
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/lib/**
@@ -431,7 +429,7 @@ jobs:
- name: Upload Lib coverage to Codecov
if: steps.changed-lib.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
@@ -442,7 +440,7 @@ jobs:
- name: Check if Config files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-config
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
./prowler/config/**
@@ -455,7 +453,7 @@ jobs:
- name: Upload Config coverage to Codecov
if: steps.changed-config.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
-221
View File
@@ -1,221 +0,0 @@
name: 'UI: Bump Version'
on:
release:
types:
- 'published'
concurrency:
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
cancel-in-progress: false
env:
PROWLER_VERSION: ${{ github.event.release.tag_name }}
BASE_BRANCH: master
jobs:
detect-release-type:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
is_minor: ${{ steps.detect.outputs.is_minor }}
is_patch: ${{ steps.detect.outputs.is_patch }}
major_version: ${{ steps.detect.outputs.major_version }}
minor_version: ${{ steps.detect.outputs.minor_version }}
patch_version: ${{ steps.detect.outputs.patch_version }}
steps:
- name: Detect release type and parse version
id: detect
run: |
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
MAJOR_VERSION=${BASH_REMATCH[1]}
MINOR_VERSION=${BASH_REMATCH[2]}
PATCH_VERSION=${BASH_REMATCH[3]}
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
if (( MAJOR_VERSION != 5 )); then
echo "::error::Releasing another Prowler major version, aborting..."
exit 1
fi
if (( PATCH_VERSION == 0 )); then
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
echo "✓ Minor release detected: $PROWLER_VERSION"
else
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
echo "✓ Patch release detected: $PROWLER_VERSION"
fi
else
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
exit 1
fi
bump-minor-version:
needs: detect-release-type
if: needs.detect-release-type.outputs.is_minor == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Calculate next minor version
run: |
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}"
echo "Current version: $PROWLER_VERSION"
echo "Next minor version: $NEXT_MINOR_VERSION"
- name: Bump UI version in .env for master
run: |
set -e
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env
echo "Files modified:"
git --no-pager diff
- name: Create PR for next minor version to master
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
base: master
commit-message: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
branch: ui-version-bump-to-v${{ env.NEXT_MINOR_VERSION }}
title: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
labels: no-changelog,skip-sync
body: |
### Description
Bump Prowler UI version to v${{ env.NEXT_MINOR_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
### Files Updated
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
### License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
- name: Checkout version branch
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
- name: Calculate first patch version
run: |
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
echo "First patch version: $FIRST_PATCH_VERSION"
echo "Version branch: $VERSION_BRANCH"
- name: Bump UI version in .env for version branch
run: |
set -e
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env
echo "Files modified:"
git --no-pager diff
- name: Create PR for first patch version to version branch
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
base: ${{ env.VERSION_BRANCH }}
commit-message: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
branch: ui-version-bump-to-v${{ env.FIRST_PATCH_VERSION }}
title: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
labels: no-changelog,skip-sync
body: |
### Description
Bump Prowler UI version to v${{ env.FIRST_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
### Files Updated
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
### License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
bump-patch-version:
needs: detect-release-type
if: needs.detect-release-type.outputs.is_patch == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Calculate next patch version
run: |
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }}
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
echo "Current version: $PROWLER_VERSION"
echo "Next patch version: $NEXT_PATCH_VERSION"
echo "Target branch: $VERSION_BRANCH"
- name: Bump UI version in .env for version branch
run: |
set -e
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env
echo "Files modified:"
git --no-pager diff
- name: Create PR for next patch version to version branch
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
with:
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
base: ${{ env.VERSION_BRANCH }}
commit-message: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
branch: ui-version-bump-to-v${{ env.NEXT_PATCH_VERSION }}
title: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
labels: no-changelog,skip-sync
body: |
### Description
Bump Prowler UI version to v${{ env.NEXT_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
### Files Updated
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
### License
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
+3 -3
View File
@@ -45,15 +45,15 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Initialize CodeQL
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
with:
languages: ${{ matrix.language }}
config-file: ./.github/codeql/ui-codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
with:
category: '/language:${{ matrix.language }}'
@@ -59,7 +59,7 @@ jobs:
message-ts: ${{ steps.slack-notification.outputs.ts }}
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Notify container push started
id: slack-notification
@@ -95,7 +95,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Login to DockerHub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
@@ -104,7 +104,7 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build and push UI container for ${{ matrix.arch }}
id: container-push
@@ -125,18 +125,18 @@ jobs:
# Create and push multi-architecture manifest
create-manifest:
needs: [setup, container-build-push]
if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Create and push manifests for push event
if: github.event_name == 'push'
@@ -175,7 +175,7 @@ jobs:
timeout-minutes: 5
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Determine overall outcome
id: outcome
@@ -203,8 +203,8 @@ jobs:
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
trigger-deployment:
if: github.event_name == 'push'
needs: [setup, container-build-push]
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
@@ -212,7 +212,7 @@ jobs:
steps:
- name: Trigger UI deployment
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
repository: ${{ secrets.CLOUD_DISPATCH }}
+6 -9
View File
@@ -20,7 +20,6 @@ env:
jobs:
ui-dockerfile-lint:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
@@ -28,11 +27,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check if Dockerfile changed
id: dockerfile-changed
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ui/Dockerfile
@@ -44,7 +43,6 @@ jobs:
ignore: DL3018
ui-container-build-and-scan:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ${{ matrix.runner }}
strategy:
matrix:
@@ -63,21 +61,20 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for UI changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: ui/**
files_ignore: |
ui/CHANGELOG.md
ui/README.md
ui/AGENTS.md
- name: Set up Docker Buildx
if: steps.check-changes.outputs.any_changed == 'true'
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- name: Build UI container for ${{ matrix.arch }}
if: steps.check-changes.outputs.any_changed == 'true'
@@ -95,7 +92,7 @@ jobs:
NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX
- name: Scan UI container with Trivy for ${{ matrix.arch }}
if: steps.check-changes.outputs.any_changed == 'true'
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
uses: ./.github/actions/trivy-scan
with:
image-name: ${{ env.IMAGE_NAME }}
+6 -6
View File
@@ -54,7 +54,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Create k8s Kind Cluster
uses: helm/kind-action@v1
with:
@@ -114,9 +114,9 @@ jobs:
echo "All database fixtures loaded successfully!"
'
- name: Setup Node.js environment
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
with:
node-version: '24.13.0'
node-version: '20.x'
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
@@ -126,7 +126,7 @@ jobs:
shell: bash
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
- name: Setup pnpm cache
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ${{ env.STORE_PATH }}
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }}
@@ -139,7 +139,7 @@ jobs:
working-directory: ./ui
run: pnpm run build
- name: Cache Playwright browsers
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
id: playwright-cache
with:
path: ~/.cache/ms-playwright
@@ -154,7 +154,7 @@ jobs:
working-directory: ./ui
run: pnpm run test:e2e
- name: Upload test reports
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: failure()
with:
name: playwright-report
+5 -6
View File
@@ -16,7 +16,7 @@ concurrency:
env:
UI_WORKING_DIR: ./ui
NODE_VERSION: '24.13.0'
NODE_VERSION: '20.x'
jobs:
ui-tests:
@@ -30,11 +30,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- name: Check for UI changes
id: check-changes
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
with:
files: |
ui/**
@@ -42,11 +42,10 @@ jobs:
files_ignore: |
ui/CHANGELOG.md
ui/README.md
ui/AGENTS.md
- name: Setup Node.js ${{ env.NODE_VERSION }}
if: steps.check-changes.outputs.any_changed == 'true'
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
with:
node-version: ${{ env.NODE_VERSION }}
@@ -64,7 +63,7 @@ jobs:
- name: Setup pnpm cache
if: steps.check-changes.outputs.any_changed == 'true'
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ${{ env.STORE_PATH }}
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }}
+1 -12
View File
@@ -82,9 +82,6 @@ continue.json
.continuerc
.continuerc.json
# AI Coding Assistants - OpenCode
opencode.json
# AI Coding Assistants - GitHub Copilot
.copilot/
.github/copilot/
@@ -150,16 +147,8 @@ node_modules
# Persistent data
_data/
# AI Instructions (generated by skills/setup.sh from AGENTS.md)
# Claude
CLAUDE.md
GEMINI.md
.github/copilot-instructions.md
# Compliance report
*.pdf
# AI Skills symlinks (generated by skills/setup.sh)
.claude/skills
.codex/skills
.github/skills
.gemini/skills
+3 -6
View File
@@ -34,7 +34,6 @@ repos:
rev: v2.3.1
hooks:
- id: autoflake
exclude: ^skills/
args:
[
"--in-place",
@@ -46,20 +45,18 @@ repos:
rev: 5.13.2
hooks:
- id: isort
exclude: ^skills/
args: ["--profile", "black"]
- repo: https://github.com/psf/black
rev: 24.4.2
hooks:
- id: black
exclude: ^skills/
- repo: https://github.com/pycqa/flake8
rev: 7.0.0
hooks:
- id: flake8
exclude: (contrib|^skills/)
exclude: contrib
args: ["--ignore=E266,W503,E203,E501,W605"]
- repo: https://github.com/python-poetry/poetry
@@ -112,7 +109,7 @@ repos:
- id: bandit
name: bandit
description: "Bandit is a tool for finding common security issues in Python code"
entry: bash -c 'bandit -q -lll -x '*_test.py,./contrib/,./.venv/,./skills/' -r .'
entry: bash -c 'bandit -q -lll -x '*_test.py,./contrib/,./.venv/' -r .'
language: system
files: '.*\.py'
@@ -126,7 +123,7 @@ repos:
- id: vulture
name: vulture
description: "Vulture finds unused code in Python programs."
entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/,skills/" --min-confidence 100 .'
entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/" --min-confidence 100 .'
language: system
files: '.*\.py'
+88 -114
View File
@@ -2,135 +2,109 @@
## How to Use This Guide
- Start here for cross-project norms. Prowler is a monorepo with several components.
- Each component has an `AGENTS.md` file with specific guidelines (e.g., `api/AGENTS.md`, `ui/AGENTS.md`).
- Component docs override this file when guidance conflicts.
## Available Skills
Use these skills for detailed patterns on-demand:
### Generic Skills (Any Project)
| Skill | Description | URL |
|-------|-------------|-----|
| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) |
| `react-19` | No useMemo/useCallback, React Compiler | [SKILL.md](skills/react-19/SKILL.md) |
| `nextjs-15` | App Router, Server Actions, streaming | [SKILL.md](skills/nextjs-15/SKILL.md) |
| `tailwind-4` | cn() utility, no var() in className | [SKILL.md](skills/tailwind-4/SKILL.md) |
| `playwright` | Page Object Model, MCP workflow, selectors | [SKILL.md](skills/playwright/SKILL.md) |
| `pytest` | Fixtures, mocking, markers, parametrize | [SKILL.md](skills/pytest/SKILL.md) |
| `django-drf` | ViewSets, Serializers, Filters | [SKILL.md](skills/django-drf/SKILL.md) |
| `zod-4` | New API (z.email(), z.uuid()) | [SKILL.md](skills/zod-4/SKILL.md) |
| `zustand-5` | Persist, selectors, slices | [SKILL.md](skills/zustand-5/SKILL.md) |
| `ai-sdk-5` | UIMessage, streaming, LangChain | [SKILL.md](skills/ai-sdk-5/SKILL.md) |
### Prowler-Specific Skills
| Skill | Description | URL |
|-------|-------------|-----|
| `prowler` | Project overview, component navigation | [SKILL.md](skills/prowler/SKILL.md) |
| `prowler-api` | Django + RLS + JSON:API patterns | [SKILL.md](skills/prowler-api/SKILL.md) |
| `prowler-ui` | Next.js + shadcn conventions | [SKILL.md](skills/prowler-ui/SKILL.md) |
| `prowler-sdk-check` | Create new security checks | [SKILL.md](skills/prowler-sdk-check/SKILL.md) |
| `prowler-mcp` | MCP server tools and models | [SKILL.md](skills/prowler-mcp/SKILL.md) |
| `prowler-test-sdk` | SDK testing (pytest + moto) | [SKILL.md](skills/prowler-test-sdk/SKILL.md) |
| `prowler-test-api` | API testing (pytest-django + RLS) | [SKILL.md](skills/prowler-test-api/SKILL.md) |
| `prowler-test-ui` | E2E testing (Playwright) | [SKILL.md](skills/prowler-test-ui/SKILL.md) |
| `prowler-compliance` | Compliance framework structure | [SKILL.md](skills/prowler-compliance/SKILL.md) |
| `prowler-compliance-review` | Review compliance framework PRs | [SKILL.md](skills/prowler-compliance-review/SKILL.md) |
| `prowler-provider` | Add new cloud providers | [SKILL.md](skills/prowler-provider/SKILL.md) |
| `prowler-ci` | CI checks and PR gates (GitHub Actions) | [SKILL.md](skills/prowler-ci/SKILL.md) |
| `prowler-pr` | Pull request conventions | [SKILL.md](skills/prowler-pr/SKILL.md) |
| `prowler-docs` | Documentation style guide | [SKILL.md](skills/prowler-docs/SKILL.md) |
| `skill-creator` | Create new AI agent skills | [SKILL.md](skills/skill-creator/SKILL.md) |
### Auto-invoke Skills
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
| Action | Skill |
|--------|-------|
| Adding new providers | `prowler-provider` |
| Adding services to existing providers | `prowler-provider` |
| After creating/modifying a skill | `skill-sync` |
| App Router / Server Actions | `nextjs-15` |
| Building AI chat features | `ai-sdk-5` |
| Create a PR with gh pr create | `prowler-pr` |
| Creating Zod schemas | `zod-4` |
| Creating new checks | `prowler-sdk-check` |
| Creating new skills | `skill-creator` |
| Creating/modifying Prowler UI components | `prowler-ui` |
| Creating/modifying models, views, serializers | `prowler-api` |
| Creating/updating compliance frameworks | `prowler-compliance` |
| Debug why a GitHub Actions job is failing | `prowler-ci` |
| Fill .github/pull_request_template.md (Context/Description/Steps to review/Checklist) | `prowler-pr` |
| General Prowler development questions | `prowler` |
| Generic DRF patterns | `django-drf` |
| Inspect PR CI checks and gates (.github/workflows/*) | `prowler-ci` |
| Inspect PR CI workflows (.github/workflows/*): conventional-commit, pr-check-changelog, pr-conflict-checker, labeler | `prowler-pr` |
| Mapping checks to compliance controls | `prowler-compliance` |
| Mocking AWS with moto in tests | `prowler-test-sdk` |
| Regenerate AGENTS.md Auto-invoke tables (sync.sh) | `skill-sync` |
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
| Reviewing compliance framework PRs | `prowler-compliance-review` |
| Testing RLS tenant isolation | `prowler-test-api` |
| Troubleshoot why a skill is missing from AGENTS.md auto-invoke | `skill-sync` |
| Understand CODEOWNERS/labeler-based automation | `prowler-ci` |
| Understand PR title conventional-commit validation | `prowler-ci` |
| Understand changelog gate and no-changelog label behavior | `prowler-ci` |
| Understand review ownership with CODEOWNERS | `prowler-pr` |
| Updating existing checks and metadata | `prowler-sdk-check` |
| Using Zustand stores | `zustand-5` |
| Working on MCP server tools | `prowler-mcp` |
| Working on Prowler UI structure (actions/adapters/types/hooks) | `prowler-ui` |
| Working with Prowler UI test helpers/pages | `prowler-test-ui` |
| Working with Tailwind classes | `tailwind-4` |
| Writing Playwright E2E tests | `playwright` |
| Writing Prowler API tests | `prowler-test-api` |
| Writing Prowler SDK tests | `prowler-test-sdk` |
| Writing Prowler UI E2E tests | `prowler-test-ui` |
| Writing Python tests with pytest | `pytest` |
| Writing React components | `react-19` |
| Writing TypeScript types/interfaces | `typescript` |
| Writing documentation | `prowler-docs` |
---
- Start here for cross-project norms, Prowler is a monorepo with several components. Every component should have an `AGENTS.md` file that contains the guidelines for the agents in that component. The file is located beside the code you are touching (e.g. `api/AGENTS.md`, `ui/AGENTS.md`, `prowler/AGENTS.md`).
- Follow the stricter rule when guidance conflicts; component docs override this file for their scope.
- Keep instructions synchronized. When you add new workflows or scripts, update both, the relevant component `AGENTS.md` and this file if they apply broadly.
## Project Overview
Prowler is an open-source cloud security assessment tool supporting AWS, Azure, GCP, Kubernetes, GitHub, M365, and more.
Prowler is an open-source cloud security assessment tool that supports multiple cloud providers (AWS, Azure, GCP, Kubernetes, GitHub, M365, etc.). The project consists in a monorepo with the following main components:
| Component | Location | Tech Stack |
|-----------|----------|------------|
| SDK | `prowler/` | Python 3.9+, Poetry |
| API | `api/` | Django 5.1, DRF, Celery |
| UI | `ui/` | Next.js 15, React 19, Tailwind 4 |
| MCP Server | `mcp_server/` | FastMCP, Python 3.12+ |
| Dashboard | `dashboard/` | Dash, Plotly |
- **Prowler SDK**: Python SDK, includes the Prowler CLI, providers, services, checks, compliances, config, etc. (`prowler/`)
- **Prowler API**: Django-based REST API backend (`api/`)
- **Prowler UI**: Next.js frontend application (`ui/`)
- **Prowler MCP Server**: Model Context Protocol server that gives access to the entire Prowler ecosystem for LLMs (`mcp_server/`)
- **Prowler Dashboard**: Prowler CLI feature that allows to visualize the results of the scans in a simple dashboard (`dashboard/`)
---
### Project Structure (Key Folders & Files)
- `prowler/`: Main source code for Prowler SDK (CLI, providers, services, checks, compliances, config, etc.)
- `api/`: Django-based REST API backend components
- `ui/`: Next.js frontend application
- `mcp_server/`: Model Context Protocol server that gives access to the entire Prowler ecosystem for LLMs
- `dashboard/`: Prowler CLI feature that allows to visualize the results of the scans in a simple dashboard
- `docs/`: Documentation
- `examples/`: Example output formats for providers and scripts
- `permissions/`: Permission-related files and policies
- `contrib/`: Community-contributed scripts or modules
- `tests/`: Prowler SDK test suite
- `docker-compose.yml`: Docker compose file to run the Prowler App (API + UI) production environment
- `docker-compose-dev.yml`: Docker compose file to run the Prowler App (API + UI) development environment
- `pyproject.toml`: Poetry Prowler SDK project file
- `.pre-commit-config.yaml`: Pre-commit hooks configuration
- `Makefile`: Makefile to run the project
- `LICENSE`: License file
- `README.md`: README file
- `CONTRIBUTING.md`: Contributing guide
## Python Development
```bash
# Setup
poetry install --with dev
poetry run pre-commit install
Most of the code is written in Python, so the main files in the root are focused on Python code.
# Code quality
### Poetry Dev Environment
For developing in Python we recommend using `poetry` to manage the dependencies. The minimal version is `2.1.1`. So it is recommended to run all commands using `poetry run ...`.
To install the core dependencies to develop it is needed to run `poetry install --with dev`.
### Pre-commit hooks
The project has pre-commit hooks to lint and format the code. They are installed by running `poetry run pre-commit install`.
When commiting a change, the hooks will be run automatically. Some of them are:
- Code formatting (black, isort)
- Linting (flake8, pylint)
- Security checks (bandit, safety, trufflehog)
- YAML/JSON validation
- Poetry lock file validation
### Linting and Formatting
We use the following tools to lint and format the code:
- `flake8`: for linting the code
- `black`: for formatting the code
- `pylint`: for linting the code
You can run all using the `make` command:
```bash
poetry run make lint
poetry run make format
poetry run pre-commit run --all-files
```
---
Or they will be run automatically when you commit your changes using pre-commit hooks.
## Commit & Pull Request Guidelines
Follow conventional-commit style: `<type>[scope]: <description>`
For the commit messages and pull requests name follow the conventional-commit style.
**Types:** `feat`, `fix`, `docs`, `chore`, `perf`, `refactor`, `style`, `test`
Befire creating a pull request, complete the checklist in `.github/pull_request_template.md`. Summaries should explain deployment impact, highlight review steps, and note changelog or permission updates. Run all relevant tests and linters before requesting review and link screenshots for UI or dashboard changes.
Before creating a PR:
1. Complete checklist in `.github/pull_request_template.md`
2. Run all relevant tests and linters
3. Link screenshots for UI changes
### Conventional Commit Style
The Conventional Commits specification is a lightweight convention on top of commit messages. It provides an easy set of rules for creating an explicit commit history; which makes it easier to write automated tools on top of.
The commit message should be structured as follows:
```
<type>[optional scope]: <description>
<BLANK LINE>
[optional body]
<BLANK LINE>
[optional footer(s)]
```
Any line of the commit message cannot be longer 100 characters! This allows the message to be easier to read on GitHub as well as in various git tools
#### Commit Types
- **feat**: code change introuce new functionality to the application
- **fix**: code change that solve a bug in the codebase
- **docs**: documentation only changes
- **chore**: changes related to the build process or auxiliary tools and libraries, that do not affect the application's functionality
- **perf**: code change that improves performance
- **refactor**: code change that neither fixes a bug nor adds a feature
- **style**: changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
- **test**: adding missing tests or correcting existing tests
+4 -4
View File
@@ -47,12 +47,12 @@ help: ## Show this help.
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
##@ Build no cache
build-no-cache-dev:
docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat mcp-server
build-no-cache-dev:
docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat
##@ Development Environment
run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, MCP, and workers
docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat mcp-server
run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, and workers
docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat
##@ Development Environment
build-and-run-api-dev: build-no-cache-dev run-api-dev
+3 -60
View File
@@ -80,23 +80,6 @@ prowler dashboard
```
![Prowler Dashboard](docs/images/products/dashboard.png)
## Attack Paths
Attack Paths automatically extends every completed AWS scan with a Neo4j graph that combines Cartography's cloud inventory with Prowler findings. The feature runs in the API worker after each scan and therefore requires:
- An accessible Neo4j instance (the Docker Compose files already ships a `neo4j` service).
- The following environment variables so Django and Celery can connect:
| Variable | Description | Default |
| --- | --- | --- |
| `NEO4J_HOST` | Hostname used by the API containers. | `neo4j` |
| `NEO4J_PORT` | Bolt port exposed by Neo4j. | `7687` |
| `NEO4J_USER` / `NEO4J_PASSWORD` | Credentials with rights to create per-tenant databases. | `neo4j` / `neo4j_password` |
Every AWS provider scan will enqueue an Attack Paths ingestion job automatically. Other cloud providers will be added in future iterations.
# Prowler at a Glance
> [!Tip]
> For the most accurate and up-to-date information about checks, services, frameworks, and categories, visit [**Prowler Hub**](https://hub.prowler.com).
@@ -165,9 +148,9 @@ If your workstation's architecture is incompatible, you can resolve this by:
### Common Issues with Docker Pull Installation
> [!Note]
If you want to use AWS role assumption (e.g., with the "Connect assuming IAM Role" option), you may need to mount your local `.aws` directory into the container as a volume (e.g., `- "${HOME}/.aws:/home/prowler/.aws:ro"`). There are several ways to configure credentials for Docker containers. See the [Troubleshooting](./docs/troubleshooting.mdx) section for more details and examples.
If you want to use AWS role assumption (e.g., with the "Connect assuming IAM Role" option), you may need to mount your local `.aws` directory into the container as a volume (e.g., `- "${HOME}/.aws:/home/prowler/.aws:ro"`). There are several ways to configure credentials for Docker containers. See the [Troubleshooting](./docs/troubleshooting.md) section for more details and examples.
You can find more information in the [Troubleshooting](./docs/troubleshooting.mdx) section.
You can find more information in the [Troubleshooting](./docs/troubleshooting.md) section.
### From GitHub
@@ -294,12 +277,11 @@ python prowler-cli.py -v
# ✏️ High level architecture
## Prowler App
**Prowler App** is composed of four key components:
**Prowler App** is composed of three key components:
- **Prowler UI**: A web-based interface, built with Next.js, providing a user-friendly experience for executing Prowler scans and visualizing results.
- **Prowler API**: A backend service, developed with Django REST Framework, responsible for running Prowler scans and storing the generated results.
- **Prowler SDK**: A Python SDK designed to extend the functionality of the Prowler CLI for advanced capabilities.
- **Prowler MCP Server**: A Model Context Protocol server that provides AI tools for Lighthouse, the AI-powered security assistant. This is a critical dependency for Lighthouse functionality.
![Prowler App Architecture](docs/products/img/prowler-app-architecture.png)
@@ -327,45 +309,6 @@ And many more environments.
![Architecture](docs/img/architecture.png)
# 🤖 AI Skills for Development
Prowler includes a comprehensive set of **AI Skills** that help AI coding assistants understand Prowler's codebase patterns and conventions.
## What are AI Skills?
Skills are structured instructions that give AI assistants the context they need to write code that follows Prowler's standards. They include:
- **Coding patterns** for each component (SDK, API, UI, MCP Server)
- **Testing conventions** (pytest, Playwright)
- **Architecture guidelines** (Clean Architecture, RLS patterns)
- **Framework-specific rules** (React 19, Next.js 15, Django DRF, Tailwind 4)
## Available Skills
| Category | Skills |
|----------|--------|
| **Generic** | `typescript`, `react-19`, `nextjs-15`, `tailwind-4`, `playwright`, `pytest`, `django-drf`, `zod-4`, `zustand-5`, `ai-sdk-5` |
| **Prowler** | `prowler`, `prowler-api`, `prowler-ui`, `prowler-mcp`, `prowler-sdk-check`, `prowler-test-ui`, `prowler-test-api`, `prowler-test-sdk`, `prowler-compliance`, `prowler-provider`, `prowler-pr`, `prowler-docs` |
## Setup
```bash
./skills/setup.sh
```
This configures skills for AI coding assistants that follow the [agentskills.io](https://agentskills.io) standard:
| Tool | Configuration |
|------|---------------|
| **Claude Code** | `.claude/skills/` (symlink) |
| **OpenCode** | `.claude/skills/` (symlink) |
| **Codex (OpenAI)** | `.codex/skills/` (symlink) |
| **GitHub Copilot** | `.github/skills/` (symlink) |
| **Gemini CLI** | `.gemini/skills/` (symlink) |
> **Note:** Restart your AI coding assistant after running setup to load the skills.
> Gemini CLI requires `experimental.skills` enabled in settings.
# 📖 Documentation
For installation instructions, usage details, tutorials, and the Developer Guide, visit https://docs.prowler.com/
-151
View File
@@ -1,151 +0,0 @@
# Prowler API - AI Agent Ruleset
> **Skills Reference**: For detailed patterns, use these skills:
> - [`prowler-api`](../skills/prowler-api/SKILL.md) - Models, Serializers, Views, RLS patterns
> - [`prowler-test-api`](../skills/prowler-test-api/SKILL.md) - Testing patterns (pytest-django)
> - [`django-drf`](../skills/django-drf/SKILL.md) - Generic DRF patterns
> - [`pytest`](../skills/pytest/SKILL.md) - Generic pytest patterns
### Auto-invoke Skills
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
| Action | Skill |
|--------|-------|
| Creating/modifying models, views, serializers | `prowler-api` |
| Generic DRF patterns | `django-drf` |
| Testing RLS tenant isolation | `prowler-test-api` |
| Writing Prowler API tests | `prowler-test-api` |
| Writing Python tests with pytest | `pytest` |
---
## CRITICAL RULES - NON-NEGOTIABLE
### Models
- ALWAYS: UUIDv4 PKs, `inserted_at`/`updated_at` timestamps, `JSONAPIMeta` class
- ALWAYS: Inherit from `RowLevelSecurityProtectedModel` for tenant-scoped data
- NEVER: Auto-increment integer PKs, models without tenant isolation
### Serializers
- ALWAYS: Separate serializers for Create/Update operations
- ALWAYS: Inherit from `RLSSerializer` for tenant-scoped models
- NEVER: Write logic in serializers (use services/utils)
### Views
- ALWAYS: Inherit from `BaseRLSViewSet` for tenant-scoped resources
- ALWAYS: Define `filterset_class`, use `@extend_schema` for OpenAPI
- NEVER: Raw SQL queries, business logic in views
### Row-Level Security (RLS)
- ALWAYS: Use `rls_transaction(tenant_id)` context manager
- NEVER: Query across tenants, trust client-provided tenant_id
### Celery Tasks
- ALWAYS: `@shared_task` with `name`, `queue`, `RLSTask` base class
- NEVER: Long-running ops in views, request context in tasks
---
## DECISION TREES
### Serializer Selection
```
Read → <Model>Serializer
Create → <Model>CreateSerializer
Update → <Model>UpdateSerializer
Nested read → <Model>IncludeSerializer
```
### Task vs View
```
< 100ms → View
> 100ms or external API → Celery task
Needs retry → Celery task
```
---
## TECH STACK
Django 5.1.x | DRF 3.15.x | djangorestframework-jsonapi 7.x | Celery 5.4.x | PostgreSQL 16 | pytest 8.x
---
## PROJECT STRUCTURE
```
api/src/backend/
├── api/ # Main Django app
│ ├── v1/ # API version 1 (views, serializers, urls)
│ ├── models.py # Django models
│ ├── filters.py # FilterSet classes
│ ├── base_views.py # Base ViewSet classes
│ ├── rls.py # Row-Level Security
│ └── tests/ # Unit tests
├── config/ # Django configuration
└── tasks/ # Celery tasks
```
---
## COMMANDS
```bash
# Development
poetry run python src/backend/manage.py runserver
poetry run celery -A config.celery worker -l INFO
# Database
poetry run python src/backend/manage.py makemigrations
poetry run python src/backend/manage.py migrate
# Testing & Linting
poetry run pytest -x --tb=short
poetry run make lint
```
---
## QA CHECKLIST
- [ ] `poetry run pytest` passes
- [ ] `poetry run make lint` passes
- [ ] Migrations created if models changed
- [ ] New endpoints have `@extend_schema` decorators
- [ ] RLS properly applied for tenant data
- [ ] Tests cover success and error cases
---
## NAMING CONVENTIONS
| Entity | Pattern | Example |
|--------|---------|---------|
| Serializer (read) | `<Model>Serializer` | `ProviderSerializer` |
| Serializer (create) | `<Model>CreateSerializer` | `ProviderCreateSerializer` |
| Serializer (update) | `<Model>UpdateSerializer` | `ProviderUpdateSerializer` |
| Filter | `<Model>Filter` | `ProviderFilter` |
| ViewSet | `<Model>ViewSet` | `ProviderViewSet` |
| Task | `<action>_<entity>_task` | `sync_provider_resources_task` |
---
## API CONVENTIONS (JSON:API)
```json
{
"data": {
"type": "providers",
"id": "uuid",
"attributes": { "name": "value" },
"relationships": { "tenant": { "data": { "type": "tenants", "id": "uuid" } } }
}
}
```
- Content-Type: `application/vnd.api+json`
- Pagination: `?page[number]=1&page[size]=20`
- Filtering: `?filter[field]=value`, `?filter[field__in]=val1,val2`
- Sorting: `?sort=field`, `?sort=-field`
- Including: `?include=provider,findings`
+1 -48
View File
@@ -2,61 +2,14 @@
All notable changes to the **Prowler API** are documented in this file.
## [1.18.0] (Prowler UNRELEASED)
### Added
- `/api/v1/overviews/compliance-watchlist` to retrieve the compliance watchlist [(#9596)](https://github.com/prowler-cloud/prowler/pull/9596)
- Support AlibabaCloud provider [(#9485)](https://github.com/prowler-cloud/prowler/pull/9485)
- `/api/v1/overviews/resource-groups` to retrieve an overview of the resource groups based on finding severities [(#9694)](https://github.com/prowler-cloud/prowler/pull/9694)
- Endpoints `GET /findings` and `GET /findings/metadata/latest` now support the `group` filter [(#9694)](https://github.com/prowler-cloud/prowler/pull/9694)
- `provider_id` and `provider_id__in` filter aliases for findings endpoints to enable consistent frontend parameter naming [(#9701)](https://github.com/prowler-cloud/prowler/pull/9701)
- Attack Paths scans for AWS providers: [(#)](https://github.com/prowler-cloud/prowler/pull/)
- A new Neo4j Docker Compose service
- A new task for the Attack Paths scan is executed when a regular scan is executed
- `AttackPathsScan` model and Attack Paths related serializers
- 4 endpoints at `/api/v1/attack-paths-scans`
- `/`: retrieve a list of Attack Paths scans
- `/:id`: retrieve full details for an Attack Paths scan
- `/:id/queries`: retrieve the catalog of Attack Paths queries for an Attack Paths scan
- `/:id/queries/run`: execute the selected Attack Paths query in an Attack Paths scan
---
## [1.17.2] (Prowler v5.16.2)
### Security
- Updated dependencies to patch security vulnerabilities: Django 5.1.15 (CVE-2025-64460, CVE-2025-13372), Werkzeug 3.1.4 (CVE-2025-66221), sqlparse 0.5.5 (PVE-2025-82038), fonttools 4.60.2 (CVE-2025-66034) [(#9730)](https://github.com/prowler-cloud/prowler/pull/9730)
---
## [1.17.1] (Prowler v5.16.1)
### Added
- Attack Paths backend support [(#9344)](https://github.com/prowler-cloud/prowler/pull/9344)
### Changed
- Security Hub integration error when no regions [(#9635)](https://github.com/prowler-cloud/prowler/pull/9635)
### Fixed
- Orphan scheduled scans caused by transaction isolation during provider creation [(#9633)](https://github.com/prowler-cloud/prowler/pull/9633)
---
## [1.17.0] (Prowler v5.16.0)
## [1.17.0] (Prowler UNRELEASED)
### Added
- New endpoint to retrieve and overview of the categories based on finding severities [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
- Endpoints `GET /findings` and `GET /findings/latests` can now use the category filter [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
- Account id, alias and provider name to PDF reporting table [(#9574)](https://github.com/prowler-cloud/prowler/pull/9574)
### Changed
- Endpoint `GET /overviews/attack-surfaces` no longer returns the related check IDs [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
- OpenAI provider to only load chat-compatible models with tool calling support [(#9523)](https://github.com/prowler-cloud/prowler/pull/9523)
- Increased execution delay for the first scheduled scan tasks to 5 seconds[(#9558)](https://github.com/prowler-cloud/prowler/pull/9558)
### Fixed
- Made `scan_id` a required filter in the compliance overview endpoint [(#9560)](https://github.com/prowler-cloud/prowler/pull/9560)
- Reduced unnecessary UPDATE resources operations by only saving when tag mappings change, lowering write load during scans [(#9569)](https://github.com/prowler-cloud/prowler/pull/9569)
---
+305 -2114
View File
File diff suppressed because it is too large Load Diff
+3 -8
View File
@@ -7,7 +7,7 @@ authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}]
dependencies = [
"celery[pytest] (>=5.4.0,<6.0.0)",
"dj-rest-auth[with_social,jwt] (==7.0.1)",
"django (==5.1.15)",
"django (==5.1.14)",
"django-allauth[saml] (>=65.8.0,<66.0.0)",
"django-celery-beat (>=2.7.0,<3.0.0)",
"django-celery-results (>=2.5.1,<3.0.0)",
@@ -36,12 +36,7 @@ dependencies = [
"drf-simple-apikey (==2.2.1)",
"matplotlib (>=3.10.6,<4.0.0)",
"reportlab (>=4.4.4,<5.0.0)",
"neo4j (<6.0.0)",
"cartography @ git+https://github.com/prowler-cloud/cartography@master",
"gevent (>=25.9.1,<26.0.0)",
"werkzeug (>=3.1.4)",
"sqlparse (>=0.5.4)",
"fonttools (>=4.60.2)"
"gevent (>=25.9.1,<26.0.0)"
]
description = "Prowler's API (Django/DRF)"
license = "Apache-2.0"
@@ -49,7 +44,7 @@ name = "prowler-api"
package-mode = false
# Needed for the SDK compatibility
requires-python = ">=3.11,<3.13"
version = "1.18.0"
version = "1.16.0"
[project.scripts]
celery = "src.backend.config.settings.celery"
+1 -7
View File
@@ -1,5 +1,4 @@
import logging
import atexit
import os
import sys
from pathlib import Path
@@ -31,7 +30,6 @@ class ApiConfig(AppConfig):
def ready(self):
from api import schema_extensions # noqa: F401
from api import signals # noqa: F401
from api.attack_paths import database as graph_database
from api.compliance import load_prowler_compliance
# Generate required cryptographic keys if not present, but only if:
@@ -41,10 +39,6 @@ class ApiConfig(AppConfig):
if "manage.py" not in sys.argv or os.environ.get("RUN_MAIN"):
self._ensure_crypto_keys()
if not getattr(settings, "TESTING", False):
graph_database.init_driver()
atexit.register(graph_database.close_driver)
load_prowler_compliance()
def _ensure_crypto_keys(self):
@@ -60,7 +54,7 @@ class ApiConfig(AppConfig):
global _keys_initialized
# Skip key generation if running tests
if getattr(settings, "TESTING", False):
if hasattr(settings, "TESTING") and settings.TESTING:
return
# Skip if already initialized in this process
@@ -1,13 +0,0 @@
from api.attack_paths.query_definitions import (
AttackPathsQueryDefinition,
AttackPathsQueryParameterDefinition,
get_queries_for_provider,
get_query_by_id,
)
__all__ = [
"AttackPathsQueryDefinition",
"AttackPathsQueryParameterDefinition",
"get_queries_for_provider",
"get_query_by_id",
]
@@ -1,144 +0,0 @@
import logging
import threading
from contextlib import contextmanager
from typing import Iterator
from uuid import UUID
import neo4j
import neo4j.exceptions
from django.conf import settings
from api.attack_paths.retryable_session import RetryableSession
# Without this Celery goes crazy with Neo4j logging
logging.getLogger("neo4j").setLevel(logging.ERROR)
logging.getLogger("neo4j").propagate = False
SERVICE_UNAVAILABLE_MAX_RETRIES = 3
# Module-level process-wide driver singleton
_driver: neo4j.Driver | None = None
_lock = threading.Lock()
# Base Neo4j functions
def get_uri() -> str:
host = settings.DATABASES["neo4j"]["HOST"]
port = settings.DATABASES["neo4j"]["PORT"]
return f"bolt://{host}:{port}"
def init_driver() -> neo4j.Driver:
global _driver
if _driver is not None:
return _driver
with _lock:
if _driver is None:
uri = get_uri()
config = settings.DATABASES["neo4j"]
_driver = neo4j.GraphDatabase.driver(
uri, auth=(config["USER"], config["PASSWORD"])
)
_driver.verify_connectivity()
return _driver
def get_driver() -> neo4j.Driver:
return init_driver()
def close_driver() -> None: # TODO: Use it
global _driver
with _lock:
if _driver is not None:
try:
_driver.close()
finally:
_driver = None
@contextmanager
def get_session(database: str | None = None) -> Iterator[RetryableSession]:
session_wrapper: RetryableSession | None = None
try:
session_wrapper = RetryableSession(
session_factory=lambda: get_driver().session(database=database),
close_driver=close_driver, # Just to avoid circular imports
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
)
yield session_wrapper
except neo4j.exceptions.Neo4jError as exc:
raise GraphDatabaseQueryException(message=exc.message, code=exc.code)
finally:
if session_wrapper is not None:
session_wrapper.close()
def create_database(database: str) -> None:
query = "CREATE DATABASE $database IF NOT EXISTS"
parameters = {"database": database}
with get_session() as session:
session.run(query, parameters)
def drop_database(database: str) -> None:
query = f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA"
with get_session() as session:
session.run(query)
def drop_subgraph(database: str, root_node_label: str, root_node_id: str) -> int:
query = """
MATCH (a:__ROOT_NODE_LABEL__ {id: $root_node_id})
CALL apoc.path.subgraphNodes(a, {})
YIELD node
DETACH DELETE node
RETURN COUNT(node) AS deleted_nodes_count
""".replace("__ROOT_NODE_LABEL__", root_node_label)
parameters = {"root_node_id": root_node_id}
with get_session(database) as session:
result = session.run(query, parameters)
try:
return result.single()["deleted_nodes_count"]
except neo4j.exceptions.ResultConsumedError:
return 0 # As there are no nodes to delete, the result is empty
# Neo4j functions related to Prowler + Cartography
DATABASE_NAME_TEMPLATE = "db-{attack_paths_scan_id}"
def get_database_name(attack_paths_scan_id: UUID) -> str:
attack_paths_scan_id_str = str(attack_paths_scan_id).lower()
return DATABASE_NAME_TEMPLATE.format(attack_paths_scan_id=attack_paths_scan_id_str)
# Exceptions
class GraphDatabaseQueryException(Exception):
def __init__(self, message: str, code: str | None = None) -> None:
super().__init__(message)
self.message = message
self.code = code
def __str__(self) -> str:
if self.code:
return f"{self.code}: {self.message}"
return self.message
@@ -1,514 +0,0 @@
from dataclasses import dataclass, field
# Dataclases for handling API's Attack Path query definitions and their parameters
@dataclass
class AttackPathsQueryParameterDefinition:
"""
Metadata describing a parameter that must be provided to an Attack Paths query.
"""
name: str
label: str
data_type: str = "string"
cast: type = str
description: str | None = None
placeholder: str | None = None
@dataclass
class AttackPathsQueryDefinition:
"""
Immutable representation of an Attack Path query.
"""
id: str
name: str
description: str
provider: str
cypher: str
parameters: list[AttackPathsQueryParameterDefinition] = field(default_factory=list)
# Accessor functions for API's Attack Paths query definitions
def get_queries_for_provider(provider: str) -> list[AttackPathsQueryDefinition]:
return _QUERY_DEFINITIONS.get(provider, [])
def get_query_by_id(query_id: str) -> AttackPathsQueryDefinition | None:
return _QUERIES_BY_ID.get(query_id)
# API's Attack Paths query definitions
_QUERY_DEFINITIONS: dict[str, list[AttackPathsQueryDefinition]] = {
"aws": [
# Custom query for detecting internet-exposed EC2 instances with sensitive S3 access
AttackPathsQueryDefinition(
id="aws-internet-exposed-ec2-sensitive-s3-access",
name="Identify internet-exposed EC2 instances with sensitive S3 access",
description="Detect EC2 instances with SSH exposed to the internet that can assume higher-privileged roles to read tagged sensitive S3 buckets despite bucket-level public access blocks.",
provider="aws",
cypher="""
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
YIELD node AS internet
MATCH path_s3 = (aws:AWSAccount {id: $provider_uid})--(s3:S3Bucket)--(t:AWSTag)
WHERE toLower(t.key) = toLower($tag_key) AND toLower(t.value) = toLower($tag_value)
MATCH path_ec2 = (aws)--(ec2:EC2Instance)--(sg:EC2SecurityGroup)--(ipi:IpPermissionInbound)
WHERE ec2.exposed_internet = true
AND ipi.toport = 22
MATCH path_role = (r:AWSRole)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
WHERE ANY(x IN stmt.resource WHERE x CONTAINS s3.name)
AND ANY(x IN stmt.action WHERE toLower(x) =~ 's3:(listbucket|getobject).*')
MATCH path_assume_role = (ec2)-[p:STS_ASSUMEROLE_ALLOW*1..9]-(r:AWSRole)
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, ec2)
YIELD rel AS can_access
UNWIND nodes(path_s3) + nodes(path_ec2) + nodes(path_role) + nodes(path_assume_role) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path_s3, path_ec2, path_role, path_assume_role, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
""",
parameters=[
AttackPathsQueryParameterDefinition(
name="tag_key",
label="Tag key",
description="Tag key to filter the S3 bucket, e.g. DataClassification.",
placeholder="DataClassification",
),
AttackPathsQueryParameterDefinition(
name="tag_value",
label="Tag value",
description="Tag value to filter the S3 bucket, e.g. Sensitive.",
placeholder="Sensitive",
),
],
),
# Regular Cartography Attack Paths queries
AttackPathsQueryDefinition(
id="aws-rds-instances",
name="Identify provisioned RDS instances",
description="List the selected AWS account alongside the RDS instances it owns.",
provider="aws",
cypher="""
MATCH path = (aws:AWSAccount {id: $provider_uid})--(rds:RDSInstance)
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-rds-unencrypted-storage",
name="Identify RDS instances without storage encryption",
description="Find RDS instances with storage encryption disabled within the selected account.",
provider="aws",
cypher="""
MATCH path = (aws:AWSAccount {id: $provider_uid})--(rds:RDSInstance)
WHERE rds.storage_encrypted = false
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-s3-anonymous-access-buckets",
name="Identify S3 buckets with anonymous access",
description="Find S3 buckets that allow anonymous access within the selected account.",
provider="aws",
cypher="""
MATCH path = (aws:AWSAccount {id: $provider_uid})--(s3:S3Bucket)
WHERE s3.anonymous_access = true
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-iam-statements-allow-all-actions",
name="Identify IAM statements that allow all actions",
description="Find IAM policy statements that allow all actions via '*' within the selected account.",
provider="aws",
cypher="""
MATCH path = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
WHERE stmt.effect = 'Allow'
AND any(x IN stmt.action WHERE x = '*')
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-iam-statements-allow-delete-policy",
name="Identify IAM statements that allow iam:DeletePolicy",
description="Find IAM policy statements that allow the iam:DeletePolicy action within the selected account.",
provider="aws",
cypher="""
MATCH path = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
WHERE stmt.effect = 'Allow'
AND any(x IN stmt.action WHERE x = "iam:DeletePolicy")
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-iam-statements-allow-create-actions",
name="Identify IAM statements that allow create actions",
description="Find IAM policy statements that allow actions containing 'create' within the selected account.",
provider="aws",
cypher="""
MATCH path = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
WHERE stmt.effect = "Allow"
AND any(x IN stmt.action WHERE toLower(x) CONTAINS "create")
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-ec2-instances-internet-exposed",
name="Identify internet-exposed EC2 instances",
description="Find EC2 instances flagged as exposed to the internet within the selected account.",
provider="aws",
cypher="""
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
YIELD node AS internet
MATCH path = (aws:AWSAccount {id: $provider_uid})--(ec2:EC2Instance)
WHERE ec2.exposed_internet = true
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, ec2)
YIELD rel AS can_access
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-security-groups-open-internet-facing",
name="Identify internet-facing resources with open security groups",
description="Find internet-facing resources associated with security groups that allow inbound access from '0.0.0.0/0'.",
provider="aws",
cypher="""
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
YIELD node AS internet
MATCH path_open = (aws:AWSAccount {id: $provider_uid})-[r0]-(open)
MATCH path_sg = (open)-[r1:MEMBER_OF_EC2_SECURITY_GROUP]-(sg:EC2SecurityGroup)
MATCH path_ip = (sg)-[r2:MEMBER_OF_EC2_SECURITY_GROUP]-(ipi:IpPermissionInbound)
MATCH path_ipi = (ipi)-[r3]-(ir:IpRange)
WHERE ir.range = "0.0.0.0/0"
OPTIONAL MATCH path_dns = (dns:AWSDNSRecord)-[:DNS_POINTS_TO]->(lb)
WHERE open.scheme = 'internet-facing'
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, open)
YIELD rel AS can_access
UNWIND nodes(path_open) + nodes(path_sg) + nodes(path_ip) + nodes(path_ipi) + nodes(path_dns) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path_open, path_sg, path_ip, path_ipi, path_dns, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-classic-elb-internet-exposed",
name="Identify internet-exposed Classic Load Balancers",
description="Find Classic Load Balancers exposed to the internet along with their listeners.",
provider="aws",
cypher="""
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
YIELD node AS internet
MATCH path = (aws:AWSAccount {id: $provider_uid})--(elb:LoadBalancer)--(listener:ELBListener)
WHERE elb.exposed_internet = true
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, elb)
YIELD rel AS can_access
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-elbv2-internet-exposed",
name="Identify internet-exposed ELBv2 load balancers",
description="Find ELBv2 load balancers exposed to the internet along with their listeners.",
provider="aws",
cypher="""
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
YIELD node AS internet
MATCH path = (aws:AWSAccount {id: $provider_uid})--(elbv2:LoadBalancerV2)--(listener:ELBV2Listener)
WHERE elbv2.exposed_internet = true
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, elbv2)
YIELD rel AS can_access
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-public-ip-resource-lookup",
name="Identify resources by public IP address",
description="Given a public IP address, find the related AWS resource and its adjacent node within the selected account.",
provider="aws",
cypher="""
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
YIELD node AS internet
CALL () {
MATCH path = (aws:AWSAccount {id: $provider_uid})-[r]-(x:EC2PrivateIp)-[q]-(y)
WHERE x.public_ip = $ip
RETURN path, x
UNION MATCH path = (aws:AWSAccount {id: $provider_uid})-[r]-(x:EC2Instance)-[q]-(y)
WHERE x.publicipaddress = $ip
RETURN path, x
UNION MATCH path = (aws:AWSAccount {id: $provider_uid})-[r]-(x:NetworkInterface)-[q]-(y)
WHERE x.public_ip = $ip
RETURN path, x
UNION MATCH path = (aws:AWSAccount {id: $provider_uid})-[r]-(x:ElasticIPAddress)-[q]-(y)
WHERE x.public_ip = $ip
RETURN path, x
}
WITH path, x, internet
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, x)
YIELD rel AS can_access
UNWIND nodes(path) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
""",
parameters=[
AttackPathsQueryParameterDefinition(
name="ip",
label="IP address",
description="Public IP address, e.g. 192.0.2.0.",
placeholder="192.0.2.0",
),
],
),
# Privilege Escalation Queries (based on pathfinding.cloud research): https://github.com/DataDog/pathfinding.cloud
AttackPathsQueryDefinition(
id="aws-iam-privesc-passrole-ec2",
name="Privilege Escalation: iam:PassRole + ec2:RunInstances",
description="Detect principals who can launch EC2 instances with privileged IAM roles attached. This allows gaining the permissions of the passed role by accessing the EC2 instance metadata service. This is a new-passrole escalation path (pathfinding.cloud: ec2-001).",
provider="aws",
cypher="""
// Create a single shared virtual EC2 instance node
CALL apoc.create.vNode(['EC2Instance'], {
id: 'potential-ec2-passrole',
name: 'New EC2 Instance',
description: 'Attacker-controlled EC2 with privileged role'
})
YIELD node AS ec2_node
// Create a single shared virtual escalation outcome node (styled like a finding)
CALL apoc.create.vNode(['PrivilegeEscalation'], {
id: 'effective-administrator-passrole-ec2',
check_title: 'Privilege Escalation',
name: 'Effective Administrator',
status: 'FAIL',
severity: 'critical'
})
YIELD node AS escalation_outcome
WITH ec2_node, escalation_outcome
// Find principals in the account
MATCH path_principal = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)
// Find statements granting iam:PassRole
MATCH path_passrole = (principal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
WHERE stmt_passrole.effect = 'Allow'
AND any(action IN stmt_passrole.action WHERE
toLower(action) = 'iam:passrole'
OR toLower(action) = 'iam:*'
OR action = '*'
)
// Find statements granting ec2:RunInstances
MATCH path_ec2 = (principal)--(ec2_policy:AWSPolicy)--(stmt_ec2:AWSPolicyStatement)
WHERE stmt_ec2.effect = 'Allow'
AND any(action IN stmt_ec2.action WHERE
toLower(action) = 'ec2:runinstances'
OR toLower(action) = 'ec2:*'
OR action = '*'
)
// Find roles that trust EC2 service (can be passed to EC2)
MATCH path_target = (aws)--(target_role:AWSRole)
WHERE target_role.arn CONTAINS $provider_uid
// Check if principal can pass this role
AND any(resource IN stmt_passrole.resource WHERE
resource = '*'
OR target_role.arn CONTAINS resource
OR resource CONTAINS target_role.name
)
// Check if target role has elevated permissions (optional, for severity assessment)
OPTIONAL MATCH (target_role)--(role_policy:AWSPolicy)--(role_stmt:AWSPolicyStatement)
WHERE role_stmt.effect = 'Allow'
AND (
any(action IN role_stmt.action WHERE action = '*')
OR any(action IN role_stmt.action WHERE toLower(action) = 'iam:*')
)
CALL apoc.create.vRelationship(principal, 'CAN_LAUNCH', {
via: 'ec2:RunInstances + iam:PassRole'
}, ec2_node)
YIELD rel AS launch_rel
CALL apoc.create.vRelationship(ec2_node, 'ASSUMES_ROLE', {}, target_role)
YIELD rel AS assumes_rel
CALL apoc.create.vRelationship(target_role, 'GRANTS_ACCESS', {
reference: 'https://pathfinding.cloud/paths/ec2-001'
}, escalation_outcome)
YIELD rel AS grants_rel
UNWIND nodes(path_principal) + nodes(path_passrole) + nodes(path_ec2) + nodes(path_target) as n
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
WHERE pf.status = 'FAIL'
RETURN path_principal, path_passrole, path_ec2, path_target,
ec2_node, escalation_outcome, launch_rel, assumes_rel, grants_rel,
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
),
AttackPathsQueryDefinition(
id="aws-glue-privesc-passrole-dev-endpoint",
name="Privilege Escalation: Glue Dev Endpoint with PassRole",
description="Detect principals that can escalate privileges by passing a role to a Glue development endpoint. The attacker creates a dev endpoint with an arbitrary role attached, then accesses those credentials through the endpoint.",
provider="aws",
cypher="""
CALL apoc.create.vNode(['PrivilegeEscalation'], {
id: 'effective-administrator-glue',
check_title: 'Privilege Escalation',
name: 'Effective Administrator (Glue)',
status: 'FAIL',
severity: 'critical'
})
YIELD node AS escalation_outcome
WITH escalation_outcome
// Find principals in the account
MATCH path_principal = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)
// Principal can assume roles (up to 2 hops)
OPTIONAL MATCH path_assume = (principal)-[:STS_ASSUMEROLE_ALLOW*0..2]->(acting_as:AWSRole)
WITH escalation_outcome, principal, path_principal, path_assume,
CASE WHEN path_assume IS NULL THEN principal ELSE acting_as END AS effective_principal
// Find iam:PassRole permission
MATCH path_passrole = (effective_principal)--(passrole_policy:AWSPolicy)--(passrole_stmt:AWSPolicyStatement)
WHERE passrole_stmt.effect = 'Allow'
AND any(action IN passrole_stmt.action WHERE toLower(action) = 'iam:passrole' OR action = '*')
// Find Glue CreateDevEndpoint permission
MATCH (effective_principal)--(glue_policy:AWSPolicy)--(glue_stmt:AWSPolicyStatement)
WHERE glue_stmt.effect = 'Allow'
AND any(action IN glue_stmt.action WHERE toLower(action) = 'glue:createdevendpoint' OR action = '*' OR toLower(action) = 'glue:*')
// Find target role with elevated permissions
MATCH (aws)--(target_role:AWSRole)--(target_policy:AWSPolicy)--(target_stmt:AWSPolicyStatement)
WHERE target_stmt.effect = 'Allow'
AND (
any(action IN target_stmt.action WHERE action = '*')
OR any(action IN target_stmt.action WHERE toLower(action) = 'iam:*')
)
// Deduplicate before creating virtual nodes
WITH DISTINCT escalation_outcome, aws, principal, effective_principal, target_role
// Create virtual Glue endpoint node (one per unique principal->target pair)
CALL apoc.create.vNode(['GlueDevEndpoint'], {
name: 'New Dev Endpoint',
description: 'Glue endpoint with target role attached',
id: effective_principal.arn + '->' + target_role.arn
})
YIELD node AS glue_endpoint
CALL apoc.create.vRelationship(effective_principal, 'CREATES_ENDPOINT', {
permissions: ['iam:PassRole', 'glue:CreateDevEndpoint'],
technique: 'new-passrole'
}, glue_endpoint)
YIELD rel AS create_rel
CALL apoc.create.vRelationship(glue_endpoint, 'RUNS_AS', {}, target_role)
YIELD rel AS runs_rel
CALL apoc.create.vRelationship(target_role, 'GRANTS_ACCESS', {
reference: 'https://pathfinding.cloud/paths/glue-001'
}, escalation_outcome)
YIELD rel AS grants_rel
// Re-match paths for visualization
MATCH path_principal = (aws)--(principal)
MATCH path_target = (aws)--(target_role)
RETURN path_principal, path_target,
glue_endpoint, escalation_outcome, create_rel, runs_rel, grants_rel
""",
parameters=[],
),
],
}
_QUERIES_BY_ID: dict[str, AttackPathsQueryDefinition] = {
definition.id: definition
for definitions in _QUERY_DEFINITIONS.values()
for definition in definitions
}
@@ -1,87 +0,0 @@
import logging
from collections.abc import Callable
from typing import Any
import neo4j
import neo4j.exceptions
logger = logging.getLogger(__name__)
class RetryableSession:
"""
Wrapper around `neo4j.Session` that retries `neo4j.exceptions.ServiceUnavailable` errors.
"""
def __init__(
self,
session_factory: Callable[[], neo4j.Session],
close_driver: Callable[[], None], # Just to avoid circular imports
max_retries: int,
) -> None:
self._session_factory = session_factory
self._close_driver = close_driver
self._max_retries = max(0, max_retries)
self._session = self._session_factory()
def close(self) -> None:
if self._session is not None:
self._session.close()
self._session = None
def __enter__(self) -> "RetryableSession":
return self
def __exit__(self, exc_type: Any, exc: Any, exc_tb: Any) -> None:
self.close()
def run(self, *args: Any, **kwargs: Any) -> Any:
return self._call_with_retry("run", *args, **kwargs)
def write_transaction(self, *args: Any, **kwargs: Any) -> Any:
return self._call_with_retry("write_transaction", *args, **kwargs)
def read_transaction(self, *args: Any, **kwargs: Any) -> Any:
return self._call_with_retry("read_transaction", *args, **kwargs)
def execute_write(self, *args: Any, **kwargs: Any) -> Any:
return self._call_with_retry("execute_write", *args, **kwargs)
def execute_read(self, *args: Any, **kwargs: Any) -> Any:
return self._call_with_retry("execute_read", *args, **kwargs)
def __getattr__(self, item: str) -> Any:
return getattr(self._session, item)
def _call_with_retry(self, method_name: str, *args: Any, **kwargs: Any) -> Any:
attempt = 0
last_exc: neo4j.exceptions.ServiceUnavailable | None = None
while attempt <= self._max_retries:
try:
method = getattr(self._session, method_name)
return method(*args, **kwargs)
except (
neo4j.exceptions.ServiceUnavailable
) as exc: # pragma: no cover - depends on infra
last_exc = exc
attempt += 1
if attempt > self._max_retries:
raise
logger.warning(
f"Neo4j session {method_name} failed with ServiceUnavailable ({attempt}/{self._max_retries} attempts). Retrying..."
)
self._refresh_session()
raise last_exc if last_exc else RuntimeError("Unexpected retry loop exit")
def _refresh_session(self) -> None:
if self._session is not None:
self._session.close()
self._close_driver()
self._session = self._session_factory()
@@ -1,143 +0,0 @@
import logging
from typing import Any
from rest_framework.exceptions import APIException, ValidationError
from api.attack_paths import database as graph_database, AttackPathsQueryDefinition
from api.models import AttackPathsScan
from config.custom_logging import BackendLogger
logger = logging.getLogger(BackendLogger.API)
def normalize_run_payload(raw_data):
if not isinstance(raw_data, dict): # Let the serializer handle this
return raw_data
if "data" in raw_data and isinstance(raw_data.get("data"), dict):
data_section = raw_data.get("data") or {}
attributes = data_section.get("attributes") or {}
payload = {
"id": attributes.get("id", data_section.get("id")),
"parameters": attributes.get("parameters"),
}
# Remove `None` parameters to allow defaults downstream
if payload.get("parameters") is None:
payload.pop("parameters")
return payload
return raw_data
def prepare_query_parameters(
definition: AttackPathsQueryDefinition,
provided_parameters: dict[str, Any],
provider_uid: str,
) -> dict[str, Any]:
parameters = dict(provided_parameters or {})
expected_names = {parameter.name for parameter in definition.parameters}
provided_names = set(parameters.keys())
unexpected = provided_names - expected_names
if unexpected:
raise ValidationError(
{"parameters": f"Unknown parameter(s): {', '.join(sorted(unexpected))}"}
)
missing = expected_names - provided_names
if missing:
raise ValidationError(
{
"parameters": f"Missing required parameter(s): {', '.join(sorted(missing))}"
}
)
clean_parameters = {
"provider_uid": str(provider_uid),
}
for definition_parameter in definition.parameters:
raw_value = provided_parameters[definition_parameter.name]
try:
casted_value = definition_parameter.cast(raw_value)
except (ValueError, TypeError) as exc:
raise ValidationError(
{
"parameters": (
f"Invalid value for parameter `{definition_parameter.name}`: {str(exc)}"
)
}
)
clean_parameters[definition_parameter.name] = casted_value
return clean_parameters
def execute_attack_paths_query(
attack_paths_scan: AttackPathsScan,
definition: AttackPathsQueryDefinition,
parameters: dict[str, Any],
) -> dict[str, Any]:
try:
with graph_database.get_session(attack_paths_scan.graph_database) as session:
result = session.run(definition.cypher, parameters)
return _serialize_graph(result.graph())
except graph_database.GraphDatabaseQueryException as exc:
logger.error(f"Query failed for Attack Paths query `{definition.id}`: {exc}")
raise APIException(
"Attack Paths query execution failed due to a database error"
)
def _serialize_graph(graph):
nodes = []
for node in graph.nodes:
nodes.append(
{
"id": node.element_id,
"labels": list(node.labels),
"properties": _serialize_properties(node._properties),
},
)
relationships = []
for relationship in graph.relationships:
relationships.append(
{
"id": relationship.element_id,
"label": relationship.type,
"source": relationship.start_node.element_id,
"target": relationship.end_node.element_id,
"properties": _serialize_properties(relationship._properties),
},
)
return {
"nodes": nodes,
"relationships": relationships,
}
def _serialize_properties(properties: dict[str, Any]) -> dict[str, Any]:
"""Convert Neo4j property values into JSON-serializable primitives."""
def _serialize_value(value: Any) -> Any:
# Neo4j temporal and spatial values expose `to_native` returning Python primitives
if hasattr(value, "to_native") and callable(value.to_native):
return _serialize_value(value.to_native())
if isinstance(value, (list, tuple)):
return [_serialize_value(item) for item in value]
if isinstance(value, dict):
return {key: _serialize_value(val) for key, val in value.items()}
return value
return {key: _serialize_value(val) for key, val in properties.items()}
+19 -110
View File
@@ -29,7 +29,6 @@ from api.models import (
Finding,
Integration,
Invitation,
AttackPathsScan,
LighthouseProviderConfiguration,
LighthouseProviderModels,
Membership,
@@ -38,7 +37,6 @@ from api.models import (
PermissionChoices,
Processor,
Provider,
ProviderComplianceScore,
ProviderGroup,
ProviderSecret,
Resource,
@@ -46,7 +44,6 @@ from api.models import (
Role,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
SeverityChoices,
StateChoices,
@@ -95,62 +92,10 @@ class ChoiceInFilter(BaseInFilter, ChoiceFilter):
pass
class BaseProviderFilter(FilterSet):
"""
Abstract base filter for models with direct FK to Provider.
Provides standard provider_id and provider_type filters.
Subclasses must define Meta.model.
"""
provider_id = UUIDFilter(field_name="provider__id", lookup_expr="exact")
provider_id__in = UUIDInFilter(field_name="provider__id", lookup_expr="in")
provider_type = ChoiceFilter(
field_name="provider__provider", choices=Provider.ProviderChoices.choices
)
provider_type__in = ChoiceInFilter(
field_name="provider__provider",
choices=Provider.ProviderChoices.choices,
lookup_expr="in",
)
class Meta:
abstract = True
fields = {}
class BaseScanProviderFilter(FilterSet):
"""
Abstract base filter for models with FK to Scan (and Scan has FK to Provider).
Provides standard provider_id and provider_type filters via scan relationship.
Subclasses must define Meta.model.
"""
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
provider_type = ChoiceFilter(
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
)
provider_type__in = ChoiceInFilter(
field_name="scan__provider__provider",
choices=Provider.ProviderChoices.choices,
lookup_expr="in",
)
class Meta:
abstract = True
fields = {}
class CommonFindingFilters(FilterSet):
# We filter providers from the scan in findings
# Both 'provider' and 'provider_id' parameters are supported for API consistency
# Frontend uses 'provider_id' uniformly across all endpoints
provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
provider_type = ChoiceFilter(
choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider"
)
@@ -216,9 +161,6 @@ class CommonFindingFilters(FilterSet):
category = CharFilter(method="filter_category")
category__in = CharInFilter(field_name="categories", lookup_expr="overlap")
resource_groups = CharFilter(field_name="resource_groups", lookup_expr="exact")
resource_groups__in = CharInFilter(field_name="resource_groups", lookup_expr="in")
# Temporarily disabled until we implement tag filtering in the UI
# resource_tag_key = CharFilter(field_name="resources__tags__key")
# resource_tag_key__in = CharInFilter(
@@ -397,23 +339,6 @@ class ScanFilter(ProviderRelationshipFilterSet):
}
class AttackPathsScanFilter(ProviderRelationshipFilterSet):
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
completed_at = DateFilter(field_name="completed_at", lookup_expr="date")
started_at = DateFilter(field_name="started_at", lookup_expr="date")
state = ChoiceFilter(choices=StateChoices.choices)
state__in = ChoiceInFilter(
field_name="state", choices=StateChoices.choices, lookup_expr="in"
)
class Meta:
model = AttackPathsScan
fields = {
"provider": ["exact", "in"],
"scan": ["exact", "in"],
}
class TaskFilter(FilterSet):
name = CharFilter(field_name="task_runner_task__task_name", lookup_expr="exact")
name__icontains = CharFilter(
@@ -461,8 +386,6 @@ class ResourceFilter(ProviderRelationshipFilterSet):
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
scan = UUIDFilter(field_name="provider__scan", lookup_expr="exact")
scan__in = UUIDInFilter(field_name="provider__scan", lookup_expr="in")
groups = CharFilter(method="filter_groups")
groups__in = CharInFilter(field_name="groups", lookup_expr="overlap")
class Meta:
model = Resource
@@ -477,9 +400,6 @@ class ResourceFilter(ProviderRelationshipFilterSet):
"updated_at": ["gte", "lte"],
}
def filter_groups(self, queryset, name, value):
return queryset.filter(groups__contains=[value])
def filter_queryset(self, queryset):
if not (self.data.get("scan") or self.data.get("scan__in")) and not (
self.data.get("updated_at")
@@ -544,8 +464,6 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
tag_value = CharFilter(method="filter_tag_value")
tag = CharFilter(method="filter_tag")
tags = CharFilter(method="filter_tag")
groups = CharFilter(method="filter_groups")
groups__in = CharInFilter(field_name="groups", lookup_expr="overlap")
class Meta:
model = Resource
@@ -558,9 +476,6 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
"type": ["exact", "icontains", "in"],
}
def filter_groups(self, queryset, name, value):
return queryset.filter(groups__contains=[value])
def filter_tag_key(self, queryset, name, value):
return queryset.filter(Q(tags__key=value) | Q(tags__key__icontains=value))
@@ -854,7 +769,7 @@ class RoleFilter(FilterSet):
class ComplianceOverviewFilter(FilterSet):
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
scan_id = UUIDFilter(field_name="scan_id", required=True)
scan_id = UUIDFilter(field_name="scan_id")
region = CharFilter(field_name="region")
class Meta:
@@ -1171,25 +1086,9 @@ class ThreatScoreSnapshotFilter(FilterSet):
}
class AttackSurfaceOverviewFilter(BaseScanProviderFilter):
class AttackSurfaceOverviewFilter(FilterSet):
"""Filter for attack surface overview aggregations by provider."""
class Meta(BaseScanProviderFilter.Meta):
model = AttackSurfaceOverview
class CategoryOverviewFilter(BaseScanProviderFilter):
"""Filter for category overview aggregations by provider."""
category = CharFilter(field_name="category", lookup_expr="exact")
category__in = CharInFilter(field_name="category", lookup_expr="in")
class Meta(BaseScanProviderFilter.Meta):
model = ScanCategorySummary
fields = {}
class ResourceGroupOverviewFilter(FilterSet):
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
provider_type = ChoiceFilter(
@@ -1200,16 +1099,26 @@ class ResourceGroupOverviewFilter(FilterSet):
choices=Provider.ProviderChoices.choices,
lookup_expr="in",
)
resource_group = CharFilter(field_name="resource_group", lookup_expr="exact")
resource_group__in = CharInFilter(field_name="resource_group", lookup_expr="in")
class Meta:
model = ScanGroupSummary
model = AttackSurfaceOverview
fields = {}
class ComplianceWatchlistFilter(BaseProviderFilter):
"""Filter for compliance watchlist overview by provider."""
class CategoryOverviewFilter(FilterSet):
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
provider_type = ChoiceFilter(
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
)
provider_type__in = ChoiceInFilter(
field_name="scan__provider__provider",
choices=Provider.ProviderChoices.choices,
lookup_expr="in",
)
category = CharFilter(field_name="category", lookup_expr="exact")
category__in = CharInFilter(field_name="category", lookup_expr="in")
class Meta(BaseProviderFilter.Meta):
model = ProviderComplianceScore
class Meta:
model = ScanCategorySummary
fields = {}
@@ -1,41 +0,0 @@
[
{
"model": "api.attackpathsscan",
"pk": "a7f0f6de-6f8e-4b3a-8cbe-3f6dd9012345",
"fields": {
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
"provider": "b85601a8-4b45-4194-8135-03fb980ef428",
"scan": "01920573-aa9c-73c9-bcda-f2e35c9b19d2",
"state": "completed",
"progress": 100,
"update_tag": 1693586667,
"graph_database": "db-a7f0f6de-6f8e-4b3a-8cbe-3f6dd9012345",
"is_graph_database_deleted": false,
"task": null,
"inserted_at": "2024-09-01T17:24:37Z",
"updated_at": "2024-09-01T17:44:37Z",
"started_at": "2024-09-01T17:34:37Z",
"completed_at": "2024-09-01T17:44:37Z",
"duration": 269,
"ingestion_exceptions": {}
}
},
{
"model": "api.attackpathsscan",
"pk": "4a2fb2af-8a60-4d7d-9cae-4ca65e098765",
"fields": {
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
"provider": "15fce1fa-ecaa-433f-a9dc-62553f3a2555",
"scan": "01929f3b-ed2e-7623-ad63-7c37cd37828f",
"state": "executing",
"progress": 48,
"update_tag": 1697625000,
"graph_database": "db-4a2fb2af-8a60-4d7d-9cae-4ca65e098765",
"is_graph_database_deleted": false,
"task": null,
"inserted_at": "2024-10-18T10:55:57Z",
"updated_at": "2024-10-18T10:56:15Z",
"started_at": "2024-10-18T10:56:05Z"
}
}
]
@@ -26,11 +26,8 @@ class Migration(migrations.Migration):
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="api.tenant",
),
"tenant_id",
models.UUIDField(db_index=True, editable=False),
),
(
"inserted_at",
@@ -59,6 +56,7 @@ class Migration(migrations.Migration):
("low", "Low"),
("informational", "Informational"),
],
max_length=50,
),
),
(
@@ -84,7 +82,6 @@ class Migration(migrations.Migration):
],
options={
"db_table": "scan_category_summaries",
"abstract": False,
},
),
migrations.AddIndex(
@@ -16,7 +16,6 @@ class Migration(migrations.Migration):
blank=True,
null=True,
size=None,
help_text="Categories from check metadata for efficient filtering",
),
),
]
@@ -1,37 +0,0 @@
# Generated by Django migration for Alibaba Cloud provider support
from django.db import migrations
import api.db_utils
class Migration(migrations.Migration):
dependencies = [
("api", "0064_finding_categories"),
]
operations = [
migrations.AlterField(
model_name="provider",
name="provider",
field=api.db_utils.ProviderEnumField(
choices=[
("aws", "AWS"),
("azure", "Azure"),
("gcp", "GCP"),
("kubernetes", "Kubernetes"),
("m365", "M365"),
("github", "GitHub"),
("mongodbatlas", "MongoDB Atlas"),
("iac", "IaC"),
("oraclecloud", "Oracle Cloud Infrastructure"),
("alibabacloud", "Alibaba Cloud"),
],
default="aws",
),
),
migrations.RunSQL(
"ALTER TYPE provider ADD VALUE IF NOT EXISTS 'alibabacloud';",
reverse_sql=migrations.RunSQL.noop,
),
]
@@ -1,94 +0,0 @@
import uuid
import django.db.models.deletion
from django.db import migrations, models
import api.db_utils
import api.rls
class Migration(migrations.Migration):
dependencies = [
("api", "0065_alibabacloud_provider"),
]
operations = [
migrations.CreateModel(
name="ProviderComplianceScore",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("compliance_id", models.TextField()),
("requirement_id", models.TextField()),
(
"requirement_status",
api.db_utils.StatusEnumField(
choices=[
("FAIL", "Fail"),
("PASS", "Pass"),
("MANUAL", "Manual"),
]
),
),
("scan_completed_at", models.DateTimeField()),
(
"provider",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="compliance_scores",
related_query_name="compliance_score",
to="api.provider",
),
),
(
"scan",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="compliance_scores",
related_query_name="compliance_score",
to="api.scan",
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="api.tenant",
),
),
],
options={
"db_table": "provider_compliance_scores",
"abstract": False,
},
),
migrations.AddConstraint(
model_name="providercompliancescore",
constraint=models.UniqueConstraint(
fields=("tenant_id", "provider_id", "compliance_id", "requirement_id"),
name="unique_provider_compliance_req",
),
),
migrations.AddConstraint(
model_name="providercompliancescore",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_providercompliancescore",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
migrations.AddIndex(
model_name="providercompliancescore",
index=models.Index(
fields=["tenant_id", "provider_id", "compliance_id"],
name="pcs_tenant_prov_comp_idx",
),
),
]
@@ -1,61 +0,0 @@
import uuid
import django.db.models.deletion
from django.db import migrations, models
import api.rls
class Migration(migrations.Migration):
dependencies = [
("api", "0066_provider_compliance_score"),
]
operations = [
migrations.CreateModel(
name="TenantComplianceSummary",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("compliance_id", models.TextField()),
("requirements_passed", models.IntegerField(default=0)),
("requirements_failed", models.IntegerField(default=0)),
("requirements_manual", models.IntegerField(default=0)),
("total_requirements", models.IntegerField(default=0)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="api.tenant",
),
),
],
options={
"db_table": "tenant_compliance_summaries",
"abstract": False,
},
),
migrations.AddConstraint(
model_name="tenantcompliancesummary",
constraint=models.UniqueConstraint(
fields=("tenant_id", "compliance_id"),
name="unique_tenant_compliance_summary",
),
),
migrations.AddConstraint(
model_name="tenantcompliancesummary",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_tenantcompliancesummary",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
]
@@ -1,126 +0,0 @@
import uuid
import django.db.models.deletion
from django.db import migrations, models
import api.db_utils
import api.rls
class Migration(migrations.Migration):
dependencies = [
("api", "0067_tenant_compliance_summary"),
]
operations = [
migrations.AddField(
model_name="finding",
name="resource_groups",
field=models.TextField(
blank=True,
help_text="Resource group from check metadata for efficient filtering",
null=True,
),
),
migrations.CreateModel(
name="ScanGroupSummary",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="api.tenant",
),
),
(
"inserted_at",
models.DateTimeField(auto_now_add=True),
),
(
"scan",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="resource_group_summaries",
related_query_name="resource_group_summary",
to="api.scan",
),
),
(
"resource_group",
models.CharField(max_length=50),
),
(
"severity",
api.db_utils.SeverityEnumField(
choices=[
("critical", "Critical"),
("high", "High"),
("medium", "Medium"),
("low", "Low"),
("informational", "Informational"),
],
),
),
(
"total_findings",
models.IntegerField(
default=0, help_text="Non-muted findings (PASS + FAIL)"
),
),
(
"failed_findings",
models.IntegerField(
default=0,
help_text="Non-muted FAIL findings (subset of total_findings)",
),
),
(
"new_failed_findings",
models.IntegerField(
default=0,
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
),
),
(
"resources_count",
models.IntegerField(
default=0, help_text="Count of distinct resource_uid values"
),
),
],
options={
"db_table": "scan_resource_group_summaries",
"abstract": False,
},
),
migrations.AddIndex(
model_name="scangroupsummary",
index=models.Index(
fields=["tenant_id", "scan"], name="srgs_tenant_scan_idx"
),
),
migrations.AddConstraint(
model_name="scangroupsummary",
constraint=models.UniqueConstraint(
fields=("tenant_id", "scan_id", "resource_group", "severity"),
name="unique_resource_group_severity_per_scan",
),
),
migrations.AddConstraint(
model_name="scangroupsummary",
constraint=api.rls.RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_scangroupsummary",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
]
@@ -1,21 +0,0 @@
from django.contrib.postgres.fields import ArrayField
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("api", "0068_finding_resource_group_scangroupsummary"),
]
operations = [
migrations.AddField(
model_name="resource",
name="groups",
field=ArrayField(
models.CharField(max_length=100),
blank=True,
help_text="Groups for categorization (e.g., compute, storage, IAM)",
null=True,
),
),
]
@@ -1,154 +0,0 @@
# Generated by Django 5.1.13 on 2025-11-06 16:20
import django.db.models.deletion
from django.db import migrations, models
from uuid6 import uuid7
import api.rls
class Migration(migrations.Migration):
dependencies = [
("api", "0069_resource_resource_group"),
]
operations = [
migrations.CreateModel(
name="AttackPathsScan",
fields=[
(
"id",
models.UUIDField(
default=uuid7,
editable=False,
primary_key=True,
serialize=False,
),
),
("inserted_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"state",
api.db_utils.StateEnumField(
choices=[
("available", "Available"),
("scheduled", "Scheduled"),
("executing", "Executing"),
("completed", "Completed"),
("failed", "Failed"),
("cancelled", "Cancelled"),
],
default="available",
),
),
("progress", models.IntegerField(default=0)),
("started_at", models.DateTimeField(blank=True, null=True)),
("completed_at", models.DateTimeField(blank=True, null=True)),
(
"duration",
models.IntegerField(
blank=True, help_text="Duration in seconds", null=True
),
),
(
"update_tag",
models.BigIntegerField(
blank=True,
help_text="Cartography update tag (epoch)",
null=True,
),
),
(
"graph_database",
models.CharField(blank=True, max_length=63, null=True),
),
(
"is_graph_database_deleted",
models.BooleanField(default=False),
),
(
"ingestion_exceptions",
models.JSONField(blank=True, default=dict, null=True),
),
(
"provider",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="attack_paths_scans",
related_query_name="attack_paths_scan",
to="api.provider",
),
),
(
"scan",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="attack_paths_scans",
related_query_name="attack_paths_scan",
to="api.scan",
),
),
(
"task",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="attack_paths_scans",
related_query_name="attack_paths_scan",
to="api.task",
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
),
),
],
options={
"db_table": "attack_paths_scans",
"abstract": False,
"indexes": [
models.Index(
fields=["tenant_id", "provider_id", "-inserted_at"],
name="aps_prov_ins_desc_idx",
),
models.Index(
fields=["tenant_id", "state", "-inserted_at"],
name="aps_state_ins_desc_idx",
),
models.Index(
fields=["tenant_id", "scan_id"],
name="aps_scan_lookup_idx",
),
models.Index(
fields=["tenant_id", "provider_id"],
name="aps_active_graph_idx",
include=["graph_database", "id"],
condition=models.Q(("is_graph_database_deleted", False)),
),
models.Index(
fields=["tenant_id", "provider_id", "-completed_at"],
name="aps_completed_graph_idx",
include=["graph_database", "id"],
condition=models.Q(
("state", "completed"),
("is_graph_database_deleted", False),
),
),
],
},
),
migrations.AddConstraint(
model_name="attackpathsscan",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_attackpathsscan",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
]
+4 -275
View File
@@ -287,7 +287,6 @@ class Provider(RowLevelSecurityProtectedModel):
MONGODBATLAS = "mongodbatlas", _("MongoDB Atlas")
IAC = "iac", _("IaC")
ORACLECLOUD = "oraclecloud", _("Oracle Cloud Infrastructure")
ALIBABACLOUD = "alibabacloud", _("Alibaba Cloud")
@staticmethod
def validate_aws_uid(value):
@@ -392,15 +391,6 @@ class Provider(RowLevelSecurityProtectedModel):
pointer="/data/attributes/uid",
)
@staticmethod
def validate_alibabacloud_uid(value):
if not re.match(r"^\d{16}$", value):
raise ModelValidationError(
detail="Alibaba Cloud account ID must be exactly 16 digits.",
code="alibabacloud-uid",
pointer="/data/attributes/uid",
)
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
@@ -626,101 +616,6 @@ class Scan(RowLevelSecurityProtectedModel):
resource_name = "scans"
class AttackPathsScan(RowLevelSecurityProtectedModel):
objects = ActiveProviderManager()
all_objects = models.Manager()
id = models.UUIDField(primary_key=True, default=uuid7, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
state = StateEnumField(choices=StateChoices.choices, default=StateChoices.AVAILABLE)
progress = models.IntegerField(default=0)
# Timing
started_at = models.DateTimeField(null=True, blank=True)
completed_at = models.DateTimeField(null=True, blank=True)
duration = models.IntegerField(
null=True, blank=True, help_text="Duration in seconds"
)
# Relationship to the provider and optional prowler Scan and celery Task
provider = models.ForeignKey(
"Provider",
on_delete=models.CASCADE,
related_name="attack_paths_scans",
related_query_name="attack_paths_scan",
)
scan = models.ForeignKey(
"Scan",
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name="attack_paths_scans",
related_query_name="attack_paths_scan",
)
task = models.ForeignKey(
"Task",
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name="attack_paths_scans",
related_query_name="attack_paths_scan",
)
# Cartography specific metadata
update_tag = models.BigIntegerField(
null=True, blank=True, help_text="Cartography update tag (epoch)"
)
graph_database = models.CharField(max_length=63, null=True, blank=True)
is_graph_database_deleted = models.BooleanField(default=False)
ingestion_exceptions = models.JSONField(default=dict, null=True, blank=True)
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "attack_paths_scans"
constraints = [
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
indexes = [
models.Index(
fields=["tenant_id", "provider_id", "-inserted_at"],
name="aps_prov_ins_desc_idx",
),
models.Index(
fields=["tenant_id", "state", "-inserted_at"],
name="aps_state_ins_desc_idx",
),
models.Index(
fields=["tenant_id", "scan_id"],
name="aps_scan_lookup_idx",
),
models.Index(
fields=["tenant_id", "provider_id"],
name="aps_active_graph_idx",
include=["graph_database", "id"],
condition=Q(is_graph_database_deleted=False),
),
models.Index(
fields=["tenant_id", "provider_id", "-completed_at"],
name="aps_completed_graph_idx",
include=["graph_database", "id"],
condition=Q(
state=StateChoices.COMPLETED,
is_graph_database_deleted=False,
),
),
]
class JSONAPIMeta:
resource_name = "attack-paths-scans"
class ResourceTag(RowLevelSecurityProtectedModel):
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
@@ -799,12 +694,6 @@ class Resource(RowLevelSecurityProtectedModel):
metadata = models.TextField(blank=True, null=True)
details = models.TextField(blank=True, null=True)
partition = models.TextField(blank=True, null=True)
groups = ArrayField(
models.CharField(max_length=100),
blank=True,
null=True,
help_text="Groups for categorization (e.g., compute, storage, IAM)",
)
failed_findings_count = models.IntegerField(default=0)
@@ -827,19 +716,14 @@ class Resource(RowLevelSecurityProtectedModel):
self.clear_tags()
return
# Add new relationships with the tenant_id field; avoid touching the
# Resource row unless a mapping is actually created to prevent noisy
# updates during scans.
mapping_created = False
# Add new relationships with the tenant_id field
for tag in tags:
_, created = ResourceTagMapping.objects.update_or_create(
ResourceTagMapping.objects.update_or_create(
tag=tag, resource=self, tenant_id=self.tenant_id
)
mapping_created = mapping_created or created
if mapping_created:
# Only bump updated_at when the tag set truly changed
self.save(update_fields=["updated_at"])
# Save the instance
self.save()
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "resources"
@@ -991,11 +875,6 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
null=True,
help_text="Categories from check metadata for efficient filtering",
)
resource_groups = models.TextField(
blank=True,
null=True,
help_text="Resource group from check metadata for efficient filtering",
)
# Relationships
scan = models.ForeignKey(to=Scan, related_name="findings", on_delete=models.CASCADE)
@@ -2138,67 +2017,6 @@ class ScanCategorySummary(RowLevelSecurityProtectedModel):
resource_name = "scan-category-summaries"
class ScanGroupSummary(RowLevelSecurityProtectedModel):
"""
Pre-aggregated resource group metrics per scan by severity.
Stores one row per (resource_group, severity) combination per scan for efficient
overview queries. Resource groups come from check_metadata.Group.
Count relationships (each is a subset of the previous):
- total_findings >= failed_findings >= new_failed_findings
"""
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
scan = models.ForeignKey(
Scan,
on_delete=models.CASCADE,
related_name="resource_group_summaries",
related_query_name="resource_group_summary",
)
resource_group = models.CharField(max_length=50)
severity = SeverityEnumField(choices=SeverityChoices)
total_findings = models.IntegerField(
default=0, help_text="Non-muted findings (PASS + FAIL)"
)
failed_findings = models.IntegerField(
default=0, help_text="Non-muted FAIL findings (subset of total_findings)"
)
new_failed_findings = models.IntegerField(
default=0,
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
)
resources_count = models.IntegerField(
default=0, help_text="Count of distinct resource_uid values"
)
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "scan_resource_group_summaries"
indexes = [
models.Index(fields=["tenant_id", "scan"], name="srgs_tenant_scan_idx"),
]
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "scan_id", "resource_group", "severity"),
name="unique_resource_group_severity_per_scan",
),
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
class JSONAPIMeta:
resource_name = "scan-resource-group-summaries"
class LighthouseConfiguration(RowLevelSecurityProtectedModel):
"""
Stores configuration and API keys for LLM services.
@@ -2772,92 +2590,3 @@ class AttackSurfaceOverview(RowLevelSecurityProtectedModel):
class JSONAPIMeta:
resource_name = "attack-surface-overviews"
class ProviderComplianceScore(RowLevelSecurityProtectedModel):
"""
Compliance requirement status from latest completed scan per provider.
Used for efficient compliance watchlist queries with FAIL-dominant aggregation
across multiple providers. Updated via atomic upsert after each scan completion.
"""
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
scan = models.ForeignKey(
Scan,
on_delete=models.CASCADE,
related_name="compliance_scores",
related_query_name="compliance_score",
)
provider = models.ForeignKey(
Provider,
on_delete=models.CASCADE,
related_name="compliance_scores",
related_query_name="compliance_score",
)
compliance_id = models.TextField()
requirement_id = models.TextField()
requirement_status = StatusEnumField(choices=StatusChoices)
scan_completed_at = models.DateTimeField()
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "provider_compliance_scores"
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "provider_id", "compliance_id", "requirement_id"),
name="unique_provider_compliance_req",
),
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
indexes = [
models.Index(
fields=["tenant_id", "provider_id", "compliance_id"],
name="pcs_tenant_prov_comp_idx",
),
]
class TenantComplianceSummary(RowLevelSecurityProtectedModel):
"""
Pre-aggregated compliance counts per tenant with FAIL-dominant logic applied.
One row per (tenant, compliance_id). Used for fast watchlist queries when
no provider filter is applied. Recalculated after each scan by aggregating
across all providers with FAIL-dominant logic at requirement level.
"""
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
compliance_id = models.TextField()
requirements_passed = models.IntegerField(default=0)
requirements_failed = models.IntegerField(default=0)
requirements_manual = models.IntegerField(default=0)
total_requirements = models.IntegerField(default=0)
updated_at = models.DateTimeField(auto_now=True)
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "tenant_compliance_summaries"
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "compliance_id"),
name="unique_tenant_compliance_summary",
),
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
File diff suppressed because it is too large Load Diff
@@ -1,172 +0,0 @@
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from rest_framework.exceptions import APIException, ValidationError
from api.attack_paths import database as graph_database
from api.attack_paths import views_helpers
def test_normalize_run_payload_extracts_attributes_section():
payload = {
"data": {
"id": "ignored",
"attributes": {
"id": "aws-rds",
"parameters": {"ip": "192.0.2.0"},
},
}
}
result = views_helpers.normalize_run_payload(payload)
assert result == {"id": "aws-rds", "parameters": {"ip": "192.0.2.0"}}
def test_normalize_run_payload_passthrough_for_non_dict():
sentinel = "not-a-dict"
assert views_helpers.normalize_run_payload(sentinel) is sentinel
def test_prepare_query_parameters_includes_provider_and_casts(
attack_paths_query_definition_factory,
):
definition = attack_paths_query_definition_factory(cast_type=int)
result = views_helpers.prepare_query_parameters(
definition,
{"limit": "5"},
provider_uid="123456789012",
)
assert result["provider_uid"] == "123456789012"
assert result["limit"] == 5
@pytest.mark.parametrize(
"provided,expected_message",
[
({}, "Missing required parameter"),
({"limit": 10, "extra": True}, "Unknown parameter"),
],
)
def test_prepare_query_parameters_validates_names(
attack_paths_query_definition_factory, provided, expected_message
):
definition = attack_paths_query_definition_factory()
with pytest.raises(ValidationError) as exc:
views_helpers.prepare_query_parameters(definition, provided, provider_uid="1")
assert expected_message in str(exc.value)
def test_prepare_query_parameters_validates_cast(
attack_paths_query_definition_factory,
):
definition = attack_paths_query_definition_factory(cast_type=int)
with pytest.raises(ValidationError) as exc:
views_helpers.prepare_query_parameters(
definition,
{"limit": "not-an-int"},
provider_uid="1",
)
assert "Invalid value" in str(exc.value)
def test_execute_attack_paths_query_serializes_graph(
attack_paths_query_definition_factory, attack_paths_graph_stub_classes
):
definition = attack_paths_query_definition_factory(
id="aws-rds",
name="RDS",
description="",
cypher="MATCH (n) RETURN n",
parameters=[],
)
parameters = {"provider_uid": "123"}
attack_paths_scan = SimpleNamespace(graph_database="tenant-db")
node = attack_paths_graph_stub_classes.Node(
element_id="node-1",
labels=["AWSAccount"],
properties={
"name": "account",
"complex": {
"items": [
attack_paths_graph_stub_classes.NativeValue("value"),
{"nested": 1},
]
},
},
)
relationship = attack_paths_graph_stub_classes.Relationship(
element_id="rel-1",
rel_type="OWNS",
start_node=node,
end_node=attack_paths_graph_stub_classes.Node("node-2", ["RDSInstance"], {}),
properties={"weight": 1},
)
graph = SimpleNamespace(nodes=[node], relationships=[relationship])
run_result = MagicMock()
run_result.graph.return_value = graph
session = MagicMock()
session.run.return_value = run_result
session_ctx = MagicMock()
session_ctx.__enter__.return_value = session
session_ctx.__exit__.return_value = False
with patch(
"api.attack_paths.views_helpers.graph_database.get_session",
return_value=session_ctx,
) as mock_get_session:
result = views_helpers.execute_attack_paths_query(
attack_paths_scan, definition, parameters
)
mock_get_session.assert_called_once_with("tenant-db")
session.run.assert_called_once_with(definition.cypher, parameters)
assert result["nodes"][0]["id"] == "node-1"
assert result["nodes"][0]["properties"]["complex"]["items"][0] == "value"
assert result["relationships"][0]["label"] == "OWNS"
def test_execute_attack_paths_query_wraps_graph_errors(
attack_paths_query_definition_factory,
):
definition = attack_paths_query_definition_factory(
id="aws-rds",
name="RDS",
description="",
cypher="MATCH (n) RETURN n",
parameters=[],
)
attack_paths_scan = SimpleNamespace(graph_database="tenant-db")
parameters = {"provider_uid": "123"}
class ExplodingContext:
def __enter__(self):
raise graph_database.GraphDatabaseQueryException("boom")
def __exit__(self, exc_type, exc, tb):
return False
with (
patch(
"api.attack_paths.views_helpers.graph_database.get_session",
return_value=ExplodingContext(),
),
patch("api.attack_paths.views_helpers.logger") as mock_logger,
):
with pytest.raises(APIException):
views_helpers.execute_attack_paths_query(
attack_paths_scan, definition, parameters
)
mock_logger.error.assert_called_once()
+1 -169
View File
@@ -1,21 +1,9 @@
from datetime import datetime, timezone
import pytest
from allauth.socialaccount.models import SocialApp
from django.core.exceptions import ValidationError
from django.db import IntegrityError
from api.db_router import MainRouter
from api.models import (
ProviderComplianceScore,
Resource,
ResourceTag,
SAMLConfiguration,
SAMLDomainIndex,
StateChoices,
StatusChoices,
TenantComplianceSummary,
)
from api.models import Resource, ResourceTag, SAMLConfiguration, SAMLDomainIndex
@pytest.mark.django_db
@@ -336,159 +324,3 @@ class TestSAMLConfigurationModel:
errors = exc_info.value.message_dict
assert "metadata_xml" in errors
assert "There is a problem with your metadata." in errors["metadata_xml"][0]
@pytest.mark.django_db
class TestProviderComplianceScoreModel:
def test_create_provider_compliance_score(self, providers_fixture, scans_fixture):
provider = providers_fixture[0]
scan = scans_fixture[0]
scan.completed_at = datetime.now(timezone.utc)
scan.save()
score = ProviderComplianceScore.objects.create(
tenant_id=provider.tenant_id,
provider=provider,
scan=scan,
compliance_id="aws_cis_2.0",
requirement_id="req_1",
requirement_status=StatusChoices.PASS,
scan_completed_at=scan.completed_at,
)
assert score.compliance_id == "aws_cis_2.0"
assert score.requirement_id == "req_1"
assert score.requirement_status == StatusChoices.PASS
def test_unique_constraint_per_provider_compliance_requirement(
self, providers_fixture, scans_fixture
):
provider = providers_fixture[0]
scan = scans_fixture[0]
scan.completed_at = datetime.now(timezone.utc)
scan.save()
ProviderComplianceScore.objects.create(
tenant_id=provider.tenant_id,
provider=provider,
scan=scan,
compliance_id="aws_cis_2.0",
requirement_id="req_1",
requirement_status=StatusChoices.PASS,
scan_completed_at=scan.completed_at,
)
with pytest.raises(IntegrityError):
ProviderComplianceScore.objects.create(
tenant_id=provider.tenant_id,
provider=provider,
scan=scan,
compliance_id="aws_cis_2.0",
requirement_id="req_1",
requirement_status=StatusChoices.FAIL,
scan_completed_at=scan.completed_at,
)
def test_different_providers_same_requirement_allowed(
self, providers_fixture, scans_fixture
):
provider1, provider2, *_ = providers_fixture
scan1 = scans_fixture[0]
scan1.completed_at = datetime.now(timezone.utc)
scan1.save()
scan2 = scans_fixture[2]
scan2.state = StateChoices.COMPLETED
scan2.completed_at = datetime.now(timezone.utc)
scan2.save()
score1 = ProviderComplianceScore.objects.create(
tenant_id=provider1.tenant_id,
provider=provider1,
scan=scan1,
compliance_id="aws_cis_2.0",
requirement_id="req_1",
requirement_status=StatusChoices.PASS,
scan_completed_at=scan1.completed_at,
)
score2 = ProviderComplianceScore.objects.create(
tenant_id=provider2.tenant_id,
provider=provider2,
scan=scan2,
compliance_id="aws_cis_2.0",
requirement_id="req_1",
requirement_status=StatusChoices.FAIL,
scan_completed_at=scan2.completed_at,
)
assert score1.id != score2.id
assert score1.requirement_status != score2.requirement_status
@pytest.mark.django_db
class TestTenantComplianceSummaryModel:
def test_create_tenant_compliance_summary(self, tenants_fixture):
tenant = tenants_fixture[0]
summary = TenantComplianceSummary.objects.create(
tenant_id=tenant.id,
compliance_id="aws_cis_2.0",
requirements_passed=5,
requirements_failed=2,
requirements_manual=1,
total_requirements=8,
)
assert summary.compliance_id == "aws_cis_2.0"
assert summary.requirements_passed == 5
assert summary.requirements_failed == 2
assert summary.requirements_manual == 1
assert summary.total_requirements == 8
assert summary.updated_at is not None
def test_unique_constraint_per_tenant_compliance(self, tenants_fixture):
tenant = tenants_fixture[0]
TenantComplianceSummary.objects.create(
tenant_id=tenant.id,
compliance_id="aws_cis_2.0",
requirements_passed=5,
requirements_failed=2,
requirements_manual=1,
total_requirements=8,
)
with pytest.raises(IntegrityError):
TenantComplianceSummary.objects.create(
tenant_id=tenant.id,
compliance_id="aws_cis_2.0",
requirements_passed=3,
requirements_failed=4,
requirements_manual=1,
total_requirements=8,
)
def test_different_tenants_same_compliance_allowed(self, tenants_fixture):
tenant1, tenant2, *_ = tenants_fixture
summary1 = TenantComplianceSummary.objects.create(
tenant_id=tenant1.id,
compliance_id="aws_cis_2.0",
requirements_passed=5,
requirements_failed=2,
requirements_manual=1,
total_requirements=8,
)
summary2 = TenantComplianceSummary.objects.create(
tenant_id=tenant2.id,
compliance_id="aws_cis_2.0",
requirements_passed=3,
requirements_failed=4,
requirements_manual=1,
total_requirements=8,
)
assert summary1.id != summary2.id
assert summary1.requirements_passed != summary2.requirements_passed
-2
View File
@@ -16,7 +16,6 @@ from api.utils import (
return_prowler_provider,
validate_invitation,
)
from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider
from prowler.providers.aws.aws_provider import AwsProvider
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
from prowler.providers.azure.azure_provider import AzureProvider
@@ -117,7 +116,6 @@ class TestReturnProwlerProvider:
(Provider.ProviderChoices.MONGODBATLAS.value, MongodbatlasProvider),
(Provider.ProviderChoices.ORACLECLOUD.value, OraclecloudProvider),
(Provider.ProviderChoices.IAC.value, IacProvider),
(Provider.ProviderChoices.ALIBABACLOUD.value, AlibabacloudProvider),
],
)
def test_return_prowler_provider(self, provider_type, expected_provider):
File diff suppressed because it is too large Load Diff
+8 -22
View File
@@ -11,7 +11,6 @@ from api.exceptions import InvitationTokenExpiredException
from api.models import Integration, Invitation, Processor, Provider, Resource
from api.v1.serializers import FindingMetadataSerializer
from prowler.lib.outputs.jira.jira import Jira, JiraBasicAuthError
from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider
from prowler.providers.aws.aws_provider import AwsProvider
from prowler.providers.aws.lib.s3.s3 import S3
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub
@@ -64,9 +63,8 @@ def merge_dicts(default_dict: dict, replacement_dict: dict) -> dict:
def return_prowler_provider(
provider: Provider,
) -> (
AlibabacloudProvider
| AwsProvider
) -> [
AwsProvider
| AzureProvider
| GcpProvider
| GithubProvider
@@ -75,14 +73,14 @@ def return_prowler_provider(
| M365Provider
| MongodbatlasProvider
| OraclecloudProvider
):
]:
"""Return the Prowler provider class based on the given provider type.
Args:
provider (Provider): The provider object containing the provider type and associated secrets.
Returns:
AlibabacloudProvider | AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: The corresponding provider class.
AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: The corresponding provider class.
Raises:
ValueError: If the provider type specified in `provider.provider` is not supported.
@@ -106,8 +104,6 @@ def return_prowler_provider(
prowler_provider = IacProvider
case Provider.ProviderChoices.ORACLECLOUD.value:
prowler_provider = OraclecloudProvider
case Provider.ProviderChoices.ALIBABACLOUD.value:
prowler_provider = AlibabacloudProvider
case _:
raise ValueError(f"Provider type {provider.provider} not supported")
return prowler_provider
@@ -173,8 +169,7 @@ def initialize_prowler_provider(
provider: Provider,
mutelist_processor: Processor | None = None,
) -> (
AlibabacloudProvider
| AwsProvider
AwsProvider
| AzureProvider
| GcpProvider
| GithubProvider
@@ -191,8 +186,9 @@ def initialize_prowler_provider(
mutelist_processor (Processor): The mutelist processor object containing the mutelist configuration.
Returns:
AlibabacloudProvider | AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: An instance of the corresponding provider class
initialized with the provider's secrets.
AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: An instance of the corresponding provider class
(`AwsProvider`, `AzureProvider`, `GcpProvider`, `GithubProvider`, `IacProvider`, `KubernetesProvider`, `M365Provider`, `OraclecloudProvider` or `MongodbatlasProvider`) initialized with the
provider's secrets.
"""
prowler_provider = return_prowler_provider(provider)
prowler_provider_kwargs = get_prowler_provider_kwargs(provider, mutelist_processor)
@@ -393,21 +389,11 @@ def get_findings_metadata_no_aggregations(tenant_id: str, filtered_queryset):
categories_set.update(categories_list)
categories = sorted(categories_set)
# Aggregate groups from findings
groups = list(
filtered_queryset.exclude(resource_groups__isnull=True)
.exclude(resource_groups__exact="")
.values_list("resource_groups", flat=True)
.distinct()
.order_by("resource_groups")
)
result = {
"services": services,
"regions": regions,
"resource_types": resource_types,
"categories": categories,
"groups": groups,
}
serializer = FindingMetadataSerializer(data=result)
@@ -304,48 +304,6 @@ from rest_framework_json_api import serializers
},
"required": ["atlas_public_key", "atlas_private_key"],
},
{
"type": "object",
"title": "Alibaba Cloud Static Credentials",
"properties": {
"access_key_id": {
"type": "string",
"description": "The Alibaba Cloud access key ID for authentication.",
},
"access_key_secret": {
"type": "string",
"description": "The Alibaba Cloud access key secret for authentication.",
},
"security_token": {
"type": "string",
"description": "The STS security token for temporary credentials (optional).",
},
},
"required": ["access_key_id", "access_key_secret"],
},
{
"type": "object",
"title": "Alibaba Cloud RAM Role Assumption",
"properties": {
"role_arn": {
"type": "string",
"description": "The ARN of the RAM role to assume (e.g., acs:ram::1234567890123456:role/ProwlerRole).",
},
"access_key_id": {
"type": "string",
"description": "The Alibaba Cloud access key ID of the RAM user that will assume the role.",
},
"access_key_secret": {
"type": "string",
"description": "The Alibaba Cloud access key secret of the RAM user that will assume the role.",
},
"role_session_name": {
"type": "string",
"description": "An identifier for the role session (optional, defaults to 'ProwlerSession').",
},
},
"required": ["role_arn", "access_key_id", "access_key_secret"],
},
]
}
)
+1 -181
View File
@@ -21,7 +21,6 @@ from rest_framework_simplejwt.tokens import RefreshToken
from api.db_router import MainRouter
from api.exceptions import ConflictException
from api.models import (
AttackPathsScan,
Finding,
Integration,
IntegrationProviderRelationship,
@@ -1133,109 +1132,6 @@ class ScanComplianceReportSerializer(BaseSerializerV1):
fields = ["id", "name"]
class AttackPathsScanSerializer(RLSSerializer):
state = StateEnumSerializerField(read_only=True)
provider_alias = serializers.SerializerMethodField(read_only=True)
provider_type = serializers.SerializerMethodField(read_only=True)
provider_uid = serializers.SerializerMethodField(read_only=True)
class Meta:
model = AttackPathsScan
fields = [
"id",
"state",
"progress",
"provider",
"provider_alias",
"provider_type",
"provider_uid",
"scan",
"task",
"inserted_at",
"started_at",
"completed_at",
"duration",
]
included_serializers = {
"provider": "api.v1.serializers.ProviderIncludeSerializer",
"scan": "api.v1.serializers.ScanIncludeSerializer",
"task": "api.v1.serializers.TaskSerializer",
}
def get_provider_alias(self, obj):
provider = getattr(obj, "provider", None)
return provider.alias if provider else None
def get_provider_type(self, obj):
provider = getattr(obj, "provider", None)
return provider.provider if provider else None
def get_provider_uid(self, obj):
provider = getattr(obj, "provider", None)
return provider.uid if provider else None
class AttackPathsQueryParameterSerializer(BaseSerializerV1):
name = serializers.CharField()
label = serializers.CharField()
data_type = serializers.CharField(default="string")
description = serializers.CharField(allow_null=True, required=False)
placeholder = serializers.CharField(allow_null=True, required=False)
class JSONAPIMeta:
resource_name = "attack-paths-query-parameters"
class AttackPathsQuerySerializer(BaseSerializerV1):
id = serializers.CharField()
name = serializers.CharField()
description = serializers.CharField()
provider = serializers.CharField()
parameters = AttackPathsQueryParameterSerializer(many=True)
class JSONAPIMeta:
resource_name = "attack-paths-queries"
class AttackPathsQueryRunRequestSerializer(BaseSerializerV1):
id = serializers.CharField()
parameters = serializers.DictField(
child=serializers.JSONField(), allow_empty=True, required=False
)
class JSONAPIMeta:
resource_name = "attack-paths-query-run-requests"
class AttackPathsNodeSerializer(BaseSerializerV1):
id = serializers.CharField()
labels = serializers.ListField(child=serializers.CharField())
properties = serializers.DictField(child=serializers.JSONField())
class JSONAPIMeta:
resource_name = "attack-paths-query-result-nodes"
class AttackPathsRelationshipSerializer(BaseSerializerV1):
id = serializers.CharField()
label = serializers.CharField()
source = serializers.CharField()
target = serializers.CharField()
properties = serializers.DictField(child=serializers.JSONField())
class JSONAPIMeta:
resource_name = "attack-paths-query-result-relationships"
class AttackPathsQueryResultSerializer(BaseSerializerV1):
nodes = AttackPathsNodeSerializer(many=True)
relationships = AttackPathsRelationshipSerializer(many=True)
class JSONAPIMeta:
resource_name = "attack-paths-query-results"
class ResourceTagSerializer(RLSSerializer):
"""
Serializer for the ResourceTag model
@@ -1279,7 +1175,6 @@ class ResourceSerializer(RLSSerializer):
"metadata",
"details",
"partition",
"groups",
]
extra_kwargs = {
"id": {"read_only": True},
@@ -1288,7 +1183,6 @@ class ResourceSerializer(RLSSerializer):
"metadata": {"read_only": True},
"details": {"read_only": True},
"partition": {"read_only": True},
"groups": {"read_only": True},
}
included_serializers = {
@@ -1382,7 +1276,6 @@ class ResourceMetadataSerializer(BaseSerializerV1):
services = serializers.ListField(child=serializers.CharField(), allow_empty=True)
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
types = serializers.ListField(child=serializers.CharField(), allow_empty=True)
groups = serializers.ListField(child=serializers.CharField(), allow_empty=True)
# Temporarily disabled until we implement tag filtering in the UI
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
@@ -1409,7 +1302,6 @@ class FindingSerializer(RLSSerializer):
"check_id",
"check_metadata",
"categories",
"resource_groups",
"raw_result",
"inserted_at",
"updated_at",
@@ -1466,9 +1358,6 @@ class FindingMetadataSerializer(BaseSerializerV1):
child=serializers.CharField(), allow_empty=True
)
categories = serializers.ListField(child=serializers.CharField(), allow_empty=True)
groups = serializers.ListField(
child=serializers.CharField(), allow_empty=True, required=False, default=list
)
# Temporarily disabled until we implement tag filtering in the UI
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
@@ -1501,23 +1390,12 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
serializer = OracleCloudProviderSecret(data=secret)
elif provider_type == Provider.ProviderChoices.MONGODBATLAS.value:
serializer = MongoDBAtlasProviderSecret(data=secret)
elif provider_type == Provider.ProviderChoices.ALIBABACLOUD.value:
serializer = AlibabaCloudProviderSecret(data=secret)
else:
raise serializers.ValidationError(
{"provider": f"Provider type not supported {provider_type}"}
)
elif secret_type == ProviderSecret.TypeChoices.ROLE:
if provider_type == Provider.ProviderChoices.AWS.value:
serializer = AWSRoleAssumptionProviderSecret(data=secret)
elif provider_type == Provider.ProviderChoices.ALIBABACLOUD.value:
serializer = AlibabaCloudRoleAssumptionProviderSecret(data=secret)
else:
raise serializers.ValidationError(
{
"secret_type": f"Role assumption not supported for provider type: {provider_type}"
}
)
serializer = AWSRoleAssumptionProviderSecret(data=secret)
elif secret_type == ProviderSecret.TypeChoices.SERVICE_ACCOUNT:
serializer = GCPServiceAccountProviderSecret(data=secret)
else:
@@ -1654,34 +1532,6 @@ class OracleCloudProviderSecret(serializers.Serializer):
resource_name = "provider-secrets"
class AlibabaCloudProviderSecret(serializers.Serializer):
access_key_id = serializers.CharField()
access_key_secret = serializers.CharField()
security_token = serializers.CharField(required=False)
class Meta:
resource_name = "provider-secrets"
class AlibabaCloudRoleAssumptionProviderSecret(serializers.Serializer):
role_arn = serializers.CharField(
help_text="Access Key ID of the RAM user that will assume the role"
)
access_key_id = serializers.CharField(
help_text="Access Key ID of the RAM user that will assume the role"
)
access_key_secret = serializers.CharField(
help_text="Access Key Secret of the RAM user that will assume the role"
)
role_session_name = serializers.CharField(
required=False,
help_text="Session name for the assumed role session (optional, defaults to 'ProwlerSession')",
)
class Meta:
resource_name = "provider-secrets"
class AWSRoleAssumptionProviderSecret(serializers.Serializer):
role_arn = serializers.CharField()
external_id = serializers.CharField()
@@ -2414,36 +2264,6 @@ class CategoryOverviewSerializer(BaseSerializerV1):
resource_name = "category-overviews"
class ResourceGroupOverviewSerializer(BaseSerializerV1):
"""Serializer for resource group overview aggregations."""
id = serializers.CharField(source="resource_group")
total_findings = serializers.IntegerField()
failed_findings = serializers.IntegerField()
new_failed_findings = serializers.IntegerField()
resources_count = serializers.IntegerField()
severity = serializers.JSONField(
help_text="Severity breakdown: {informational, low, medium, high, critical}"
)
class JSONAPIMeta:
resource_name = "resource-group-overviews"
class ComplianceWatchlistOverviewSerializer(BaseSerializerV1):
"""Serializer for compliance watchlist overview with FAIL-dominant aggregation."""
id = serializers.CharField(source="compliance_id")
compliance_id = serializers.CharField()
requirements_passed = serializers.IntegerField()
requirements_failed = serializers.IntegerField()
requirements_manual = serializers.IntegerField()
total_requirements = serializers.IntegerField()
class JSONAPIMeta:
resource_name = "compliance-watchlist-overviews"
class OverviewRegionSerializer(serializers.Serializer):
id = serializers.SerializerMethodField()
provider_type = serializers.CharField()
-4
View File
@@ -4,7 +4,6 @@ from drf_spectacular.views import SpectacularRedocView
from rest_framework_nested import routers
from api.v1.views import (
AttackPathsScanViewSet,
ComplianceOverviewViewSet,
CustomSAMLLoginView,
CustomTokenObtainView,
@@ -54,9 +53,6 @@ router.register(r"tenants", TenantViewSet, basename="tenant")
router.register(r"providers", ProviderViewSet, basename="provider")
router.register(r"provider-groups", ProviderGroupViewSet, basename="providergroup")
router.register(r"scans", ScanViewSet, basename="scan")
router.register(
r"attack-paths-scans", AttackPathsScanViewSet, basename="attack-paths-scans"
)
router.register(r"tasks", TaskViewSet, basename="task")
router.register(r"resources", ResourceViewSet, basename="resource")
router.register(r"findings", FindingViewSet, basename="finding")
+21 -486
View File
@@ -3,7 +3,6 @@ import glob
import json
import logging
import os
from collections import defaultdict
from copy import deepcopy
from datetime import datetime, timedelta, timezone
@@ -11,7 +10,6 @@ from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
from urllib.parse import urljoin
import sentry_sdk
from allauth.socialaccount.models import SocialAccount, SocialApp
from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter
@@ -43,9 +41,8 @@ from django.db.models import (
Sum,
Value,
When,
Window,
)
from django.db.models.functions import Coalesce, RowNumber
from django.db.models.functions import Coalesce
from django.http import HttpResponse, QueryDict
from django.shortcuts import redirect
from django.urls import reverse
@@ -75,12 +72,23 @@ from rest_framework.generics import GenericAPIView, get_object_or_404
from rest_framework.permissions import SAFE_METHODS
from rest_framework_json_api.views import RelationshipView, Response
from rest_framework_simplejwt.exceptions import InvalidToken, TokenError
from api.attack_paths import (
get_queries_for_provider,
get_query_by_id,
views_helpers as attack_paths_views_helpers,
from tasks.beat import schedule_provider_scan
from tasks.jobs.export import get_s3_client
from tasks.tasks import (
backfill_compliance_summaries_task,
backfill_scan_resource_summaries_task,
check_integration_connection_task,
check_lighthouse_connection_task,
check_lighthouse_provider_connection_task,
check_provider_connection_task,
delete_provider_task,
delete_tenant_task,
jira_integration_task,
mute_historical_findings_task,
perform_scan_task,
refresh_lighthouse_provider_models_task,
)
from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset
from api.compliance import (
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
@@ -93,7 +101,6 @@ from api.filters import (
AttackSurfaceOverviewFilter,
CategoryOverviewFilter,
ComplianceOverviewFilter,
ComplianceWatchlistFilter,
CustomDjangoFilterBackend,
DailySeveritySummaryFilter,
FindingFilter,
@@ -102,7 +109,6 @@ from api.filters import (
InvitationFilter,
LatestFindingFilter,
LatestResourceFilter,
AttackPathsScanFilter,
LighthouseProviderConfigFilter,
LighthouseProviderModelsFilter,
MembershipFilter,
@@ -112,7 +118,6 @@ from api.filters import (
ProviderGroupFilter,
ProviderSecretFilter,
ResourceFilter,
ResourceGroupOverviewFilter,
RoleFilter,
ScanFilter,
ScanSummaryFilter,
@@ -131,7 +136,6 @@ from api.models import (
Finding,
Integration,
Invitation,
AttackPathsScan,
LighthouseConfiguration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
@@ -140,7 +144,6 @@ from api.models import (
MuteRule,
Processor,
Provider,
ProviderComplianceScore,
ProviderGroup,
ProviderGroupMembership,
ProviderSecret,
@@ -155,13 +158,11 @@ from api.models import (
SAMLToken,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
SeverityChoices,
StateChoices,
Task,
TenantAPIKey,
TenantComplianceSummary,
ThreatScoreSnapshot,
User,
UserRoleRelationship,
@@ -177,10 +178,6 @@ from api.utils import (
from api.uuid_utils import datetime_to_uuid7, uuid7_start
from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin
from api.v1.serializers import (
AttackPathsQueryRunRequestSerializer,
AttackPathsQuerySerializer,
AttackPathsQueryResultSerializer,
AttackPathsScanSerializer,
AttackSurfaceOverviewSerializer,
CategoryOverviewSerializer,
ComplianceOverviewAttributesSerializer,
@@ -188,7 +185,6 @@ from api.v1.serializers import (
ComplianceOverviewDetailThreatscoreSerializer,
ComplianceOverviewMetadataSerializer,
ComplianceOverviewSerializer,
ComplianceWatchlistOverviewSerializer,
FindingDynamicFilterSerializer,
FindingMetadataSerializer,
FindingSerializer,
@@ -233,7 +229,6 @@ from api.v1.serializers import (
ProviderSecretUpdateSerializer,
ProviderSerializer,
ProviderUpdateSerializer,
ResourceGroupOverviewSerializer,
ResourceMetadataSerializer,
ResourceSerializer,
RoleCreateSerializer,
@@ -263,23 +258,6 @@ from api.v1.serializers import (
UserSerializer,
UserUpdateSerializer,
)
from tasks.beat import schedule_provider_scan
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
from tasks.jobs.export import get_s3_client
from tasks.tasks import (
backfill_compliance_summaries_task,
backfill_scan_resource_summaries_task,
check_integration_connection_task,
check_lighthouse_connection_task,
check_lighthouse_provider_connection_task,
check_provider_connection_task,
delete_provider_task,
delete_tenant_task,
jira_integration_task,
mute_historical_findings_task,
perform_scan_task,
refresh_lighthouse_provider_models_task,
)
logger = logging.getLogger(BackendLogger.API)
@@ -381,7 +359,7 @@ class SchemaView(SpectacularAPIView):
def get(self, request, *args, **kwargs):
spectacular_settings.TITLE = "Prowler API"
spectacular_settings.VERSION = "1.18.0"
spectacular_settings.VERSION = "1.17.0"
spectacular_settings.DESCRIPTION = (
"Prowler API specification.\n\nThis file is auto-generated."
)
@@ -423,10 +401,6 @@ class SchemaView(SpectacularAPIView):
"name": "Scan",
"description": "Endpoints for triggering manual scans and viewing scan results.",
},
{
"name": "Attack Paths",
"description": "Endpoints for Attack Paths scan status and executing Attack Paths queries.",
},
{
"name": "Schedule",
"description": "Endpoints for managing scan schedules, allowing configuration of automated "
@@ -2177,12 +2151,6 @@ class ScanViewSet(BaseRLSViewSet):
},
)
attack_paths_db_utils.create_attack_paths_scan(
tenant_id=self.request.tenant_id,
scan_id=str(scan.id),
provider_id=str(scan.provider_id),
)
prowler_task = Task.objects.get(id=task.id)
scan.task_id = task.id
scan.save(update_fields=["task_id"])
@@ -2263,187 +2231,6 @@ class TaskViewSet(BaseRLSViewSet):
)
@extend_schema_view(
list=extend_schema(
tags=["Attack Paths"],
summary="List Attack Paths scans",
description="Retrieve Attack Paths scans for the tenant with support for filtering, ordering, and pagination.",
),
retrieve=extend_schema(
tags=["Attack Paths"],
summary="Retrieve Attack Paths scan details",
description="Fetch full details for a specific Attack Paths scan.",
),
attack_paths_queries=extend_schema(
tags=["Attack Paths"],
summary="List attack paths queries",
description="Retrieve the catalog of Attack Paths queries available for this Attack Paths scan.",
responses={
200: OpenApiResponse(AttackPathsQuerySerializer(many=True)),
404: OpenApiResponse(
description="No queries found for the selected provider"
),
},
),
run_attack_paths_query=extend_schema(
tags=["Attack Paths"],
summary="Execute an Attack Paths query",
description="Execute the selected Attack Paths query against the Attack Paths graph and return the resulting subgraph.",
request=AttackPathsQueryRunRequestSerializer,
responses={
200: OpenApiResponse(AttackPathsQueryResultSerializer),
400: OpenApiResponse(
description="Bad request (e.g., Unknown Attack Paths query for the selected provider)"
),
404: OpenApiResponse(
description="No attack paths found for the given query and parameters"
),
500: OpenApiResponse(
description="Attack Paths query execution failed due to a database error"
),
},
),
)
class AttackPathsScanViewSet(BaseRLSViewSet):
queryset = AttackPathsScan.objects.all()
serializer_class = AttackPathsScanSerializer
http_method_names = ["get", "post"]
filterset_class = AttackPathsScanFilter
ordering = ["-inserted_at"]
ordering_fields = [
"inserted_at",
"started_at",
]
# RBAC required permissions
required_permissions = [Permissions.MANAGE_SCANS]
def set_required_permissions(self):
if self.request.method in SAFE_METHODS:
self.required_permissions = []
else:
self.required_permissions = [Permissions.MANAGE_SCANS]
def get_serializer_class(self):
if self.action == "run_attack_paths_query":
return AttackPathsQueryRunRequestSerializer
return super().get_serializer_class()
def get_queryset(self):
user_roles = get_role(self.request.user)
base_queryset = AttackPathsScan.objects.filter(tenant_id=self.request.tenant_id)
if user_roles.unlimited_visibility:
queryset = base_queryset
else:
queryset = base_queryset.filter(provider__in=get_providers(user_roles))
return queryset.select_related("provider", "scan", "task")
def list(self, request, *args, **kwargs):
queryset = self.filter_queryset(self.get_queryset())
latest_per_provider = queryset.annotate(
latest_scan_rank=Window(
expression=RowNumber(),
partition_by=[F("provider_id")],
order_by=[F("inserted_at").desc()],
)
).filter(latest_scan_rank=1)
page = self.paginate_queryset(latest_per_provider)
if page is not None:
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
serializer = self.get_serializer(latest_per_provider, many=True)
return Response(serializer.data)
@extend_schema(exclude=True)
def create(self, request, *args, **kwargs):
raise MethodNotAllowed(method="POST")
@extend_schema(exclude=True)
def destroy(self, request, *args, **kwargs):
raise MethodNotAllowed(method="DELETE")
@action(
detail=True,
methods=["get"],
url_path="queries",
url_name="queries",
)
def attack_paths_queries(self, request, pk=None):
attack_paths_scan = self.get_object()
queries = get_queries_for_provider(attack_paths_scan.provider.provider)
if not queries:
return Response(
{"detail": "No queries found for the selected provider"},
status=status.HTTP_404_NOT_FOUND,
)
serializer = AttackPathsQuerySerializer(queries, many=True)
return Response(serializer.data, status=status.HTTP_200_OK)
@action(
detail=True,
methods=["post"],
url_path="queries/run",
url_name="queries-run",
)
def run_attack_paths_query(self, request, pk=None):
attack_paths_scan = self.get_object()
if attack_paths_scan.state != StateChoices.COMPLETED:
raise ValidationError(
{
"detail": "The Attack Paths scan must be completed before running Attack Paths queries"
}
)
if not attack_paths_scan.graph_database:
logger.error(
f"The Attack Paths Scan {attack_paths_scan.id} does not reference a graph database"
)
return Response(
{"detail": "The Attack Paths scan does not reference a graph database"},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
payload = attack_paths_views_helpers.normalize_run_payload(request.data)
serializer = AttackPathsQueryRunRequestSerializer(data=payload)
serializer.is_valid(raise_exception=True)
query_definition = get_query_by_id(serializer.validated_data["id"])
if (
query_definition is None
or query_definition.provider != attack_paths_scan.provider.provider
):
raise ValidationError(
{"id": "Unknown Attack Paths query for the selected provider"}
)
parameters = attack_paths_views_helpers.prepare_query_parameters(
query_definition,
serializer.validated_data.get("parameters", {}),
attack_paths_scan.provider.uid,
)
graph = attack_paths_views_helpers.execute_attack_paths_query(
attack_paths_scan, query_definition, parameters
)
status_code = status.HTTP_200_OK
if not graph.get("nodes"):
status_code = status.HTTP_404_NOT_FOUND
response_serializer = AttackPathsQueryResultSerializer(graph)
return Response(response_serializer.data, status=status_code)
@extend_schema_view(
list=extend_schema(
tags=["Resource"],
@@ -2736,20 +2523,10 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
.order_by("resource_type")
)
# Get groups from Resource model (flatten ArrayField)
all_groups = Resource.objects.filter(
tenant_id=tenant_id,
groups__isnull=False,
).values_list("groups", flat=True)
groups = sorted(
set(g for groups_list in all_groups if groups_list for g in groups_list)
)
result = {
"services": services,
"regions": regions,
"types": resource_types,
"groups": groups,
}
serializer = self.get_serializer(data=result)
@@ -2806,20 +2583,10 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
.order_by("resource_type")
)
# Get groups from Resource model for resources in latest scans (flatten ArrayField)
all_groups = Resource.objects.filter(
tenant_id=tenant_id,
groups__isnull=False,
).values_list("groups", flat=True)
groups = sorted(
set(g for groups_list in all_groups if groups_list for g in groups_list)
)
result = {
"services": services,
"regions": regions,
"types": resource_types,
"groups": groups,
}
serializer = self.get_serializer(data=result)
@@ -3248,23 +3015,11 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
categories_set.update(categories_list)
categories = sorted(categories_set)
# Get groups from ScanGroupSummary for latest scans
groups = list(
ScanGroupSummary.objects.filter(
tenant_id=tenant_id,
scan_id__in=latest_scans_queryset.values_list("id", flat=True),
)
.values_list("resource_group", flat=True)
.distinct()
.order_by("resource_group")
)
result = {
"services": services,
"regions": regions,
"resource_types": resource_types,
"categories": categories,
"groups": groups,
}
serializer = self.get_serializer(data=result)
@@ -4338,30 +4093,6 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
filters=True,
responses={200: CategoryOverviewSerializer(many=True)},
),
resource_groups=extend_schema(
summary="Get resource group overview",
description=(
"Retrieve aggregated resource group metrics from latest completed scans per provider. "
"Returns one row per resource group with total, failed, and new failed findings counts, "
"plus a severity breakdown showing failed findings per severity level, "
"and a count of distinct resources evaluated per group."
),
tags=["Overview"],
filters=True,
responses={200: ResourceGroupOverviewSerializer(many=True)},
),
compliance_watchlist=extend_schema(
summary="Get compliance watchlist overview",
description=(
"Retrieve compliance metrics with FAIL-dominant aggregation. "
"Without filters: uses pre-aggregated TenantComplianceSummary. "
"With provider filters: queries ProviderComplianceScore with FAIL-dominant logic "
"where any FAIL in a requirement marks it as failed."
),
tags=["Overview"],
filters=True,
responses={200: ComplianceWatchlistOverviewSerializer(many=True)},
),
)
@method_decorator(CACHE_DECORATOR, name="list")
class OverviewViewSet(BaseRLSViewSet):
@@ -4411,10 +4142,6 @@ class OverviewViewSet(BaseRLSViewSet):
return AttackSurfaceOverviewSerializer
elif self.action == "categories":
return CategoryOverviewSerializer
elif self.action == "resource_groups":
return ResourceGroupOverviewSerializer
elif self.action == "compliance_watchlist":
return ComplianceWatchlistOverviewSerializer
return super().get_serializer_class()
def get_filterset_class(self):
@@ -4428,12 +4155,8 @@ class OverviewViewSet(BaseRLSViewSet):
return DailySeveritySummaryFilter
elif self.action == "categories":
return CategoryOverviewFilter
elif self.action == "resource_groups":
return ResourceGroupOverviewFilter
elif self.action == "attack_surface":
return AttackSurfaceOverviewFilter
elif self.action == "compliance_watchlist":
return ComplianceWatchlistFilter
return None
def filter_queryset(self, queryset):
@@ -4517,8 +4240,6 @@ class OverviewViewSet(BaseRLSViewSet):
self.request.query_params, exclude_keys=set(exclude_keys or [])
)
filterset = filterset_class(normalized_params, queryset=queryset)
if not filterset.is_valid():
raise ValidationError(filterset.errors)
return filterset.qs
def _latest_scan_ids_for_allowed_providers(self, tenant_id, provider_filters=None):
@@ -4535,10 +4256,9 @@ class OverviewViewSet(BaseRLSViewSet):
)
def _extract_provider_filters_from_params(self):
"""Extract and validate provider filters from query params."""
"""Extract provider filters from query params to apply on Scan queryset."""
params = self.request.query_params
filters = {}
valid_provider_types = {c[0] for c in Provider.ProviderChoices.choices}
provider_id = params.get("filter[provider_id]")
if provider_id:
@@ -4550,21 +4270,11 @@ class OverviewViewSet(BaseRLSViewSet):
provider_type = params.get("filter[provider_type]")
if provider_type:
if provider_type not in valid_provider_types:
raise ValidationError(
{"provider_type": f"Invalid choice: {provider_type}"}
)
filters["provider__provider"] = provider_type
provider_type_in = params.get("filter[provider_type__in]")
if provider_type_in:
types = provider_type_in.split(",")
invalid = [t for t in types if t not in valid_provider_types]
if invalid:
raise ValidationError(
{"provider_type__in": f"Invalid choices: {', '.join(invalid)}"}
)
filters["provider__provider__in"] = types
filters["provider__provider__in"] = provider_type_in.split(",")
return filters
@@ -5274,181 +4984,6 @@ class OverviewViewSet(BaseRLSViewSet):
status=status.HTTP_200_OK,
)
@action(
detail=False,
methods=["get"],
url_name="resource-groups",
url_path="resource-groups",
)
def resource_groups(self, request):
tenant_id = request.tenant_id
provider_filters = self._extract_provider_filters_from_params()
latest_scan_ids = self._latest_scan_ids_for_allowed_providers(
tenant_id, provider_filters
)
base_queryset = ScanGroupSummary.objects.filter(
tenant_id=tenant_id, scan_id__in=latest_scan_ids
)
provider_filter_keys = {
"provider_id",
"provider_id__in",
"provider_type",
"provider_type__in",
}
filtered_queryset = self._apply_filterset(
base_queryset,
ResourceGroupOverviewFilter,
exclude_keys=provider_filter_keys,
)
aggregation = (
filtered_queryset.values("resource_group", "severity")
.annotate(
total=Coalesce(Sum("total_findings"), 0),
failed=Coalesce(Sum("failed_findings"), 0),
new_failed=Coalesce(Sum("new_failed_findings"), 0),
)
.order_by("resource_group", "severity")
)
# Get resource_group-level resources_count:
# 1. Max per (scan, resource_group) to deduplicate within-scan severity rows
# 2. Sum across scans for cross-provider aggregation
scan_resource_group_resources = filtered_queryset.values(
"scan_id", "resource_group"
).annotate(resources=Coalesce(Max("resources_count"), 0))
resources_by_resource_group = defaultdict(int)
for row in scan_resource_group_resources:
resources_by_resource_group[row["resource_group"]] += row["resources"]
resource_group_data = defaultdict(
lambda: {
"total_findings": 0,
"failed_findings": 0,
"new_failed_findings": 0,
"resources_count": 0,
"severity": {
"informational": 0,
"low": 0,
"medium": 0,
"high": 0,
"critical": 0,
},
}
)
for row in aggregation:
grp = row["resource_group"]
sev = row["severity"]
resource_group_data[grp]["total_findings"] += row["total"]
resource_group_data[grp]["failed_findings"] += row["failed"]
resource_group_data[grp]["new_failed_findings"] += row["new_failed"]
if sev in resource_group_data[grp]["severity"]:
resource_group_data[grp]["severity"][sev] = row["failed"]
# Set resources_count from resource_group-level aggregation
for grp in resource_group_data:
resource_group_data[grp]["resources_count"] = (
resources_by_resource_group.get(grp, 0)
)
response_data = [
{"resource_group": grp, **data}
for grp, data in sorted(resource_group_data.items())
]
return Response(
self.get_serializer(response_data, many=True).data,
status=status.HTTP_200_OK,
)
@action(
detail=False,
methods=["get"],
url_name="compliance-watchlist",
url_path="compliance-watchlist",
)
def compliance_watchlist(self, request):
"""
Get compliance watchlist overview with FAIL-dominant aggregation.
Without filters: uses pre-aggregated TenantComplianceSummary (~70 rows).
With provider filters: queries ProviderComplianceScore with FAIL-dominant logic.
"""
tenant_id = request.tenant_id
rbac_filter = self._get_provider_filter()
query_params = request.query_params
has_provider_filter = any(
key.startswith("filter[provider") for key in query_params.keys()
)
has_rbac_restriction = bool(rbac_filter)
if not has_provider_filter and not has_rbac_restriction:
response_data = list(
TenantComplianceSummary.objects.filter(tenant_id=tenant_id)
.values(
"compliance_id",
"requirements_passed",
"requirements_failed",
"requirements_manual",
"total_requirements",
)
.order_by("compliance_id")
)
else:
base_queryset = ProviderComplianceScore.objects.filter(
tenant_id=tenant_id, **rbac_filter
)
filtered_queryset = self._apply_filterset(
base_queryset, ComplianceWatchlistFilter
)
aggregation = (
filtered_queryset.values("compliance_id", "requirement_id")
.annotate(
has_fail=Sum(
Case(When(requirement_status="FAIL", then=1), default=0)
),
has_manual=Sum(
Case(When(requirement_status="MANUAL", then=1), default=0)
),
)
.values("compliance_id", "requirement_id", "has_fail", "has_manual")
)
compliance_data = defaultdict(
lambda: {
"requirements_passed": 0,
"requirements_failed": 0,
"requirements_manual": 0,
"total_requirements": 0,
}
)
for row in aggregation:
cid = row["compliance_id"]
compliance_data[cid]["total_requirements"] += 1
if row["has_fail"] and row["has_fail"] > 0:
compliance_data[cid]["requirements_failed"] += 1
elif row["has_manual"] and row["has_manual"] > 0:
compliance_data[cid]["requirements_manual"] += 1
else:
compliance_data[cid]["requirements_passed"] += 1
response_data = [
{"compliance_id": cid, **data}
for cid, data in sorted(compliance_data.items())
]
return Response(
self.get_serializer(response_data, many=True).data,
status=status.HTTP_200_OK,
)
@extend_schema(tags=["Schedule"])
@extend_schema_view(
@@ -6118,7 +5653,7 @@ class TenantApiKeyViewSet(BaseRLSViewSet):
@extend_schema(exclude=True)
def destroy(self, request, *args, **kwargs):
raise MethodNotAllowed(method="DELETE")
raise MethodNotAllowed(method="DESTROY")
@action(detail=True, methods=["delete"])
def revoke(self, request, *args, **kwargs):
-1
View File
@@ -1,7 +1,6 @@
import warnings
from celery import Celery, Task
from config.env import env
# Suppress specific warnings from django-rest-auth: https://github.com/iMerica/dj-rest-auth/issues/684
-6
View File
@@ -44,12 +44,6 @@ DATABASES = {
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
},
"neo4j": {
"HOST": env.str("NEO4J_HOST", "neo4j"),
"PORT": env.str("NEO4J_PORT", "7687"),
"USER": env.str("NEO4J_USER", "neo4j"),
"PASSWORD": env.str("NEO4J_PASSWORD", "neo4j_password"),
},
}
DATABASES["default"] = DATABASES["prowler_user"]
@@ -45,12 +45,6 @@ DATABASES = {
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
},
"neo4j": {
"HOST": env.str("NEO4J_HOST"),
"PORT": env.str("NEO4J_PORT"),
"USER": env.str("NEO4J_USER"),
"PASSWORD": env.str("NEO4J_PASSWORD"),
},
}
DATABASES["default"] = DATABASES["prowler_user"]
+10 -329
View File
@@ -1,11 +1,8 @@
import logging
from types import SimpleNamespace
from datetime import datetime, timedelta, timezone
from unittest.mock import MagicMock, patch
import pytest
from allauth.socialaccount.models import SocialLogin
from django.conf import settings
from django.db import connection as django_connection
@@ -14,14 +11,13 @@ from django.urls import reverse
from django_celery_results.models import TaskResult
from rest_framework import status
from rest_framework.test import APIClient
from api.attack_paths import (
AttackPathsQueryDefinition,
AttackPathsQueryParameterDefinition,
from tasks.jobs.backfill import (
backfill_resource_scan_summaries,
backfill_scan_category_summaries,
)
from api.db_utils import rls_transaction
from api.models import (
AttackPathsScan,
AttackSurfaceOverview,
ComplianceOverview,
ComplianceRequirementOverview,
@@ -34,7 +30,6 @@ from api.models import (
MuteRule,
Processor,
Provider,
ProviderComplianceScore,
ProviderGroup,
ProviderSecret,
Resource,
@@ -45,13 +40,11 @@ from api.models import (
SAMLDomainIndex,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
StateChoices,
StatusChoices,
Task,
TenantAPIKey,
TenantComplianceSummary,
User,
UserRoleRelationship,
)
@@ -59,11 +52,6 @@ from api.rls import Tenant
from api.v1.serializers import TokenSerializer
from prowler.lib.check.models import Severity
from prowler.lib.outputs.finding import Status
from tasks.jobs.backfill import (
backfill_resource_scan_summaries,
backfill_scan_category_summaries,
backfill_scan_resource_group_summaries,
)
TODAY = str(datetime.today().date())
API_JSON_CONTENT_TYPE = "application/vnd.api+json"
@@ -176,20 +164,22 @@ def create_test_user_rbac_no_roles(django_db_setup, django_db_blocker, tenants_f
@pytest.fixture(scope="function")
def create_test_user_rbac_limited(django_db_setup, django_db_blocker, tenants_fixture):
def create_test_user_rbac_limited(django_db_setup, django_db_blocker):
with django_db_blocker.unblock():
user = User.objects.create_user(
name="testing_limited",
email="rbac_limited@rbac.com",
password=TEST_PASSWORD,
)
tenant = tenants_fixture[0]
tenant = Tenant.objects.create(
name="Tenant Test",
)
Membership.objects.create(
user=user,
tenant=tenant,
role=Membership.RoleChoices.OWNER,
)
role = Role.objects.create(
Role.objects.create(
name="limited",
tenant_id=tenant.id,
manage_users=False,
@@ -202,7 +192,7 @@ def create_test_user_rbac_limited(django_db_setup, django_db_blocker, tenants_fi
)
UserRoleRelationship.objects.create(
user=user,
role=role,
role=Role.objects.get(name="limited"),
tenant_id=tenant.id,
)
return user
@@ -527,12 +517,6 @@ def providers_fixture(tenants_fixture):
alias="mongodbatlas_testing",
tenant_id=tenant.id,
)
provider9 = Provider.objects.create(
provider="alibabacloud",
uid="1234567890123456",
alias="alibabacloud_testing",
tenant_id=tenant.id,
)
return (
provider1,
@@ -543,7 +527,6 @@ def providers_fixture(tenants_fixture):
provider6,
provider7,
provider8,
provider9,
)
@@ -747,7 +730,6 @@ def resources_fixture(providers_fixture):
region="us-east-1",
service="ec2",
type="prowler-test",
groups=["compute"],
)
resource1.upsert_or_delete_tags(tags)
@@ -760,7 +742,6 @@ def resources_fixture(providers_fixture):
region="eu-west-1",
service="s3",
type="prowler-test",
groups=["storage"],
)
resource2.upsert_or_delete_tags(tags)
@@ -772,7 +753,6 @@ def resources_fixture(providers_fixture):
region="us-east-1",
service="ec2",
type="test",
groups=["compute"],
)
tags = [
@@ -1394,13 +1374,11 @@ def latest_scan_finding_with_categories(
check_id="genai_iam_check",
check_metadata={"CheckId": "genai_iam_check"},
categories=["gen-ai", "iam"],
resource_groups="ai_ml",
first_seen_at="2024-01-02T00:00:00Z",
)
finding.add_resources([resource])
backfill_resource_scan_summaries(tenant_id, str(scan.id))
backfill_scan_category_summaries(tenant_id, str(scan.id))
backfill_scan_resource_group_summaries(tenant_id, str(scan.id))
return finding
@@ -1603,104 +1581,6 @@ def mute_rules_fixture(tenants_fixture, create_test_user, findings_fixture):
return mute_rule1, mute_rule2
@pytest.fixture
def create_attack_paths_scan():
"""Factory fixture to create Attack Paths scans for tests."""
def _create(
provider,
*,
scan=None,
state=StateChoices.COMPLETED,
progress=0,
graph_database="tenant-db",
**extra_fields,
):
scan_instance = scan or Scan.objects.create(
name=extra_fields.pop("scan_name", "Attack Paths Supporting Scan"),
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=extra_fields.pop("scan_state", StateChoices.COMPLETED),
tenant_id=provider.tenant_id,
)
payload = {
"tenant_id": provider.tenant_id,
"provider": provider,
"scan": scan_instance,
"state": state,
"progress": progress,
"graph_database": graph_database,
}
payload.update(extra_fields)
return AttackPathsScan.objects.create(**payload)
return _create
@pytest.fixture
def attack_paths_query_definition_factory():
"""Factory fixture for building Attack Paths query definitions."""
def _create(**overrides):
cast_type = overrides.pop("cast_type", str)
parameters = overrides.pop(
"parameters",
[
AttackPathsQueryParameterDefinition(
name="limit",
label="Limit",
cast=cast_type,
)
],
)
definition_payload = {
"id": "aws-test",
"name": "Attack Paths Test Query",
"description": "Synthetic Attack Paths definition for tests.",
"provider": "aws",
"cypher": "RETURN 1",
"parameters": parameters,
}
definition_payload.update(overrides)
return AttackPathsQueryDefinition(**definition_payload)
return _create
@pytest.fixture
def attack_paths_graph_stub_classes():
"""Provide lightweight graph element stubs for Attack Paths serialization tests."""
class AttackPathsNativeValue:
def __init__(self, value):
self._value = value
def to_native(self):
return self._value
class AttackPathsNode:
def __init__(self, element_id, labels, properties):
self.element_id = element_id
self.labels = labels
self._properties = properties
class AttackPathsRelationship:
def __init__(self, element_id, rel_type, start_node, end_node, properties):
self.element_id = element_id
self.type = rel_type
self.start_node = start_node
self.end_node = end_node
self._properties = properties
return SimpleNamespace(
NativeValue=AttackPathsNativeValue,
Node=AttackPathsNode,
Relationship=AttackPathsRelationship,
)
@pytest.fixture
def create_attack_surface_overview():
def _create(tenant, scan, attack_surface_type, total=10, failed=5, muted_failed=2):
@@ -1740,209 +1620,10 @@ def create_scan_category_summary():
return _create
@pytest.fixture(scope="function")
def findings_with_group(scans_fixture, resources_fixture):
scan = scans_fixture[0]
resource = resources_fixture[0]
finding = Finding.objects.create(
tenant_id=scan.tenant_id,
uid="finding_with_group_1",
scan=scan,
delta=None,
status=Status.FAIL,
status_extended="test status",
impact=Severity.critical,
impact_extended="test impact",
severity=Severity.critical,
raw_result={"status": Status.FAIL},
check_id="storage_check",
check_metadata={"CheckId": "storage_check"},
resource_groups="storage",
first_seen_at="2024-01-02T00:00:00Z",
)
finding.add_resources([resource])
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
return finding
@pytest.fixture(scope="function")
def findings_with_multiple_groups(scans_fixture, resources_fixture):
scan = scans_fixture[0]
resource1, resource2 = resources_fixture[:2]
finding1 = Finding.objects.create(
tenant_id=scan.tenant_id,
uid="finding_multi_grp_1",
scan=scan,
delta=None,
status=Status.FAIL,
status_extended="test status",
impact=Severity.critical,
impact_extended="test impact",
severity=Severity.critical,
raw_result={"status": Status.FAIL},
check_id="storage_check",
check_metadata={"CheckId": "storage_check"},
resource_groups="storage",
first_seen_at="2024-01-02T00:00:00Z",
)
finding1.add_resources([resource1])
finding2 = Finding.objects.create(
tenant_id=scan.tenant_id,
uid="finding_multi_grp_2",
scan=scan,
delta=None,
status=Status.FAIL,
status_extended="test status 2",
impact=Severity.high,
impact_extended="test impact 2",
severity=Severity.high,
raw_result={"status": Status.FAIL},
check_id="security_check",
check_metadata={"CheckId": "security_check"},
resource_groups="security",
first_seen_at="2024-01-02T00:00:00Z",
)
finding2.add_resources([resource2])
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
return finding1, finding2
@pytest.fixture
def create_scan_resource_group_summary():
def _create(
tenant,
scan,
resource_group,
severity,
total_findings=10,
failed_findings=5,
new_failed_findings=2,
resources_count=3,
):
return ScanGroupSummary.objects.create(
tenant=tenant,
scan=scan,
resource_group=resource_group,
severity=severity,
total_findings=total_findings,
failed_findings=failed_findings,
new_failed_findings=new_failed_findings,
resources_count=resources_count,
)
return _create
def get_authorization_header(access_token: str) -> dict:
return {"Authorization": f"Bearer {access_token}"}
@pytest.fixture
def provider_compliance_scores_fixture(
tenants_fixture, providers_fixture, scans_fixture
):
"""Create ProviderComplianceScore entries for compliance watchlist tests."""
tenant = tenants_fixture[0]
provider1, provider2, *_ = providers_fixture
scan1, _, scan3 = scans_fixture
scan1.completed_at = datetime.now(timezone.utc) - timedelta(hours=1)
scan1.save()
scan3.state = StateChoices.COMPLETED
scan3.completed_at = datetime.now(timezone.utc)
scan3.save()
scores = [
ProviderComplianceScore.objects.create(
tenant_id=tenant.id,
provider=provider1,
scan=scan1,
compliance_id="aws_cis_2.0",
requirement_id="req_1",
requirement_status=StatusChoices.PASS,
scan_completed_at=scan1.completed_at,
),
ProviderComplianceScore.objects.create(
tenant_id=tenant.id,
provider=provider1,
scan=scan1,
compliance_id="aws_cis_2.0",
requirement_id="req_2",
requirement_status=StatusChoices.FAIL,
scan_completed_at=scan1.completed_at,
),
ProviderComplianceScore.objects.create(
tenant_id=tenant.id,
provider=provider1,
scan=scan1,
compliance_id="aws_cis_2.0",
requirement_id="req_3",
requirement_status=StatusChoices.MANUAL,
scan_completed_at=scan1.completed_at,
),
ProviderComplianceScore.objects.create(
tenant_id=tenant.id,
provider=provider2,
scan=scan3,
compliance_id="aws_cis_2.0",
requirement_id="req_1",
requirement_status=StatusChoices.FAIL,
scan_completed_at=scan3.completed_at,
),
ProviderComplianceScore.objects.create(
tenant_id=tenant.id,
provider=provider2,
scan=scan3,
compliance_id="aws_cis_2.0",
requirement_id="req_2",
requirement_status=StatusChoices.PASS,
scan_completed_at=scan3.completed_at,
),
ProviderComplianceScore.objects.create(
tenant_id=tenant.id,
provider=provider1,
scan=scan1,
compliance_id="gdpr_aws",
requirement_id="gdpr_req_1",
requirement_status=StatusChoices.PASS,
scan_completed_at=scan1.completed_at,
),
]
return scores
@pytest.fixture
def tenant_compliance_summary_fixture(tenants_fixture):
"""Create TenantComplianceSummary entries for compliance watchlist tests."""
tenant = tenants_fixture[0]
summaries = [
TenantComplianceSummary.objects.create(
tenant_id=tenant.id,
compliance_id="aws_cis_2.0",
requirements_passed=1,
requirements_failed=2,
requirements_manual=1,
total_requirements=4,
),
TenantComplianceSummary.objects.create(
tenant_id=tenant.id,
compliance_id="gdpr_aws",
requirements_passed=5,
requirements_failed=0,
requirements_manual=2,
total_requirements=7,
),
]
return summaries
def pytest_collection_modifyitems(items):
"""Ensure test_rbac.py is executed first."""
items.sort(key=lambda item: 0 if "test_rbac.py" in item.nodeid else 1)
+1 -8
View File
@@ -7,7 +7,6 @@ from tasks.tasks import perform_scheduled_scan_task
from api.db_utils import rls_transaction
from api.exceptions import ConflictException
from api.models import Provider, Scan, StateChoices
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
def schedule_provider_scan(provider_instance: Provider):
@@ -40,12 +39,6 @@ def schedule_provider_scan(provider_instance: Provider):
scheduled_at=datetime.now(timezone.utc),
)
attack_paths_db_utils.create_attack_paths_scan(
tenant_id=tenant_id,
scan_id=str(scheduled_scan.id),
provider_id=provider_id,
)
# Schedule the task
periodic_task_instance = PeriodicTask.objects.create(
interval=schedule,
@@ -68,5 +61,5 @@ def schedule_provider_scan(provider_instance: Provider):
"tenant_id": str(provider_instance.tenant_id),
"provider_id": provider_id,
},
countdown=5, # Avoid race conditions between the worker and the database
countdown=1, # Avoid race conditions between the worker and the database
)
@@ -1,7 +0,0 @@
from tasks.jobs.attack_paths.db_utils import can_provider_run_attack_paths_scan
from tasks.jobs.attack_paths.scan import run as attack_paths_scan
__all__ = [
"attack_paths_scan",
"can_provider_run_attack_paths_scan",
]
@@ -1,237 +0,0 @@
# Portions of this file are based on code from the Cartography project
# (https://github.com/cartography-cncf/cartography), which is licensed under the Apache 2.0 License.
from typing import Any
import aioboto3
import boto3
import neo4j
from cartography.config import Config as CartographyConfig
from cartography.intel import aws as cartography_aws
from celery.utils.log import get_task_logger
from api.models import (
AttackPathsScan as ProwlerAPIAttackPathsScan,
Provider as ProwlerAPIProvider,
)
from prowler.providers.common.provider import Provider as ProwlerSDKProvider
from tasks.jobs.attack_paths import db_utils, utils
logger = get_task_logger(__name__)
def start_aws_ingestion(
neo4j_session: neo4j.Session,
cartography_config: CartographyConfig,
prowler_api_provider: ProwlerAPIProvider,
prowler_sdk_provider: ProwlerSDKProvider,
attack_paths_scan: ProwlerAPIAttackPathsScan,
) -> dict[str, dict[str, str]]:
"""
Code based on Cartography version 0.122.0, specifically on `cartography.intel.aws.__init__.py`.
For the scan progress updates:
- The caller of this function (`tasks.jobs.attack_paths.scan.run`) has set it to 2.
- When the control returns to the caller, it will be set to 95.
"""
# Initialize variables common to all jobs
common_job_parameters = {
"UPDATE_TAG": cartography_config.update_tag,
"permission_relationships_file": cartography_config.permission_relationships_file,
"aws_guardduty_severity_threshold": cartography_config.aws_guardduty_severity_threshold,
"aws_cloudtrail_management_events_lookback_hours": cartography_config.aws_cloudtrail_management_events_lookback_hours,
"experimental_aws_inspector_batch": cartography_config.experimental_aws_inspector_batch,
}
boto3_session = get_boto3_session(prowler_api_provider, prowler_sdk_provider)
regions: list[str] = list(prowler_sdk_provider._enabled_regions)
requested_syncs = list(cartography_aws.RESOURCE_FUNCTIONS.keys())
sync_args = cartography_aws._build_aws_sync_kwargs(
neo4j_session,
boto3_session,
regions,
prowler_api_provider.uid,
cartography_config.update_tag,
common_job_parameters,
)
# Starting with sync functions
cartography_aws.organizations.sync(
neo4j_session,
{prowler_api_provider.alias: prowler_api_provider.uid},
cartography_config.update_tag,
common_job_parameters,
)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 3)
# Adding an extra field
common_job_parameters["AWS_ID"] = prowler_api_provider.uid
cartography_aws._autodiscover_accounts(
neo4j_session,
boto3_session,
prowler_api_provider.uid,
cartography_config.update_tag,
common_job_parameters,
)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 4)
failed_syncs = sync_aws_account(
prowler_api_provider, requested_syncs, sync_args, attack_paths_scan
)
if "permission_relationships" in requested_syncs:
cartography_aws.RESOURCE_FUNCTIONS["permission_relationships"](**sync_args)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 88)
if "resourcegroupstaggingapi" in requested_syncs:
cartography_aws.RESOURCE_FUNCTIONS["resourcegroupstaggingapi"](**sync_args)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 89)
cartography_aws.run_scoped_analysis_job(
"aws_ec2_iaminstanceprofile.json",
neo4j_session,
common_job_parameters,
)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 90)
cartography_aws.run_analysis_job(
"aws_lambda_ecr.json",
neo4j_session,
common_job_parameters,
)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 91)
cartography_aws.merge_module_sync_metadata(
neo4j_session,
group_type="AWSAccount",
group_id=prowler_api_provider.uid,
synced_type="AWSAccount",
update_tag=cartography_config.update_tag,
stat_handler=cartography_aws.stat_handler,
)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 92)
# Removing the added extra field
del common_job_parameters["AWS_ID"]
cartography_aws.run_cleanup_job(
"aws_post_ingestion_principals_cleanup.json",
neo4j_session,
common_job_parameters,
)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 93)
cartography_aws._perform_aws_analysis(
requested_syncs, neo4j_session, common_job_parameters
)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 94)
return failed_syncs
def get_boto3_session(
prowler_api_provider: ProwlerAPIProvider, prowler_sdk_provider: ProwlerSDKProvider
) -> boto3.Session:
boto3_session = prowler_sdk_provider.session.current_session
aws_accounts_from_session = cartography_aws.organizations.get_aws_account_default(
boto3_session
)
if not aws_accounts_from_session:
raise Exception(
"No valid AWS credentials could be found. No AWS accounts can be synced."
)
aws_account_id_from_session = list(aws_accounts_from_session.values())[0]
if prowler_api_provider.uid != aws_account_id_from_session:
raise Exception(
f"Provider {prowler_api_provider.uid} doesn't match AWS account {aws_account_id_from_session}."
)
if boto3_session.region_name is None:
global_region = prowler_sdk_provider.get_global_region()
boto3_session._session.set_config_variable("region", global_region)
return boto3_session
def get_aioboto3_session(boto3_session: boto3.Session) -> aioboto3.Session:
return aioboto3.Session(botocore_session=boto3_session._session)
def sync_aws_account(
prowler_api_provider: ProwlerAPIProvider,
requested_syncs: list[str],
sync_args: dict[str, Any],
attack_paths_scan: ProwlerAPIAttackPathsScan,
) -> dict[str, str]:
current_progress = 4 # `cartography_aws._autodiscover_accounts`
max_progress = (
87 # `cartography_aws.RESOURCE_FUNCTIONS["permission_relationships"]` - 1
)
n_steps = (
len(requested_syncs) - 2
) # Excluding `permission_relationships` and `resourcegroupstaggingapi`
progress_step = (max_progress - current_progress) / n_steps
failed_syncs = {}
for func_name in requested_syncs:
if func_name in cartography_aws.RESOURCE_FUNCTIONS:
logger.info(
f"Syncing function {func_name} for AWS account {prowler_api_provider.uid}"
)
# Updating progress, not really the right place but good enough
current_progress += progress_step
db_utils.update_attack_paths_scan_progress(
attack_paths_scan, int(current_progress)
)
try:
# `ecr:image_layers` uses `aioboto3_session` instead of `boto3_session`
if func_name == "ecr:image_layers":
cartography_aws.RESOURCE_FUNCTIONS[func_name](
neo4j_session=sync_args.get("neo4j_session"),
aioboto3_session=get_aioboto3_session(
sync_args.get("boto3_session")
),
regions=sync_args.get("regions"),
current_aws_account_id=sync_args.get("current_aws_account_id"),
update_tag=sync_args.get("update_tag"),
common_job_parameters=sync_args.get("common_job_parameters"),
)
# Skip permission relationships and tags for now because they rely on data already being in the graph
elif func_name in [
"permission_relationships",
"resourcegroupstaggingapi",
]:
continue
else:
cartography_aws.RESOURCE_FUNCTIONS[func_name](**sync_args)
except Exception as e:
exception_message = utils.stringify_exception(
e, f"Exception for AWS sync function: {func_name}"
)
failed_syncs[func_name] = exception_message
logger.warning(
f"Caught exception syncing function {func_name} from AWS account {prowler_api_provider.uid}. We "
"are continuing on to the next AWS sync function.",
)
continue
else:
raise ValueError(
f'AWS sync function "{func_name}" was specified but does not exist. Did you misspell it?'
)
return failed_syncs
@@ -1,161 +0,0 @@
from datetime import datetime, timezone
from typing import Any
from cartography.config import Config as CartographyConfig
from api.db_utils import rls_transaction
from api.models import (
AttackPathsScan as ProwlerAPIAttackPathsScan,
Provider as ProwlerAPIProvider,
StateChoices,
)
from tasks.jobs.attack_paths.providers import is_provider_available
def can_provider_run_attack_paths_scan(tenant_id: str, provider_id: int) -> bool:
with rls_transaction(tenant_id):
prowler_api_provider = ProwlerAPIProvider.objects.get(id=provider_id)
return is_provider_available(prowler_api_provider.provider)
def create_attack_paths_scan(
tenant_id: str,
scan_id: str,
provider_id: int,
) -> ProwlerAPIAttackPathsScan | None:
if not can_provider_run_attack_paths_scan(tenant_id, provider_id):
return None
with rls_transaction(tenant_id):
attack_paths_scan = ProwlerAPIAttackPathsScan.objects.create(
tenant_id=tenant_id,
provider_id=provider_id,
scan_id=scan_id,
state=StateChoices.SCHEDULED,
started_at=datetime.now(tz=timezone.utc),
)
attack_paths_scan.save()
return attack_paths_scan
def retrieve_attack_paths_scan(
tenant_id: str,
scan_id: str,
) -> ProwlerAPIAttackPathsScan | None:
try:
with rls_transaction(tenant_id):
attack_paths_scan = ProwlerAPIAttackPathsScan.objects.get(
scan_id=scan_id,
)
return attack_paths_scan
except ProwlerAPIAttackPathsScan.DoesNotExist:
return None
def starting_attack_paths_scan(
attack_paths_scan: ProwlerAPIAttackPathsScan,
task_id: str,
cartography_config: CartographyConfig,
) -> None:
with rls_transaction(attack_paths_scan.tenant_id):
attack_paths_scan.task_id = task_id
attack_paths_scan.state = StateChoices.EXECUTING
attack_paths_scan.started_at = datetime.now(tz=timezone.utc)
attack_paths_scan.update_tag = cartography_config.update_tag
attack_paths_scan.graph_database = cartography_config.neo4j_database
attack_paths_scan.save(
update_fields=[
"task_id",
"state",
"started_at",
"update_tag",
"graph_database",
]
)
def finish_attack_paths_scan(
attack_paths_scan: ProwlerAPIAttackPathsScan,
state: StateChoices,
ingestion_exceptions: dict[str, Any],
) -> None:
with rls_transaction(attack_paths_scan.tenant_id):
now = datetime.now(tz=timezone.utc)
duration = int((now - attack_paths_scan.started_at).total_seconds())
attack_paths_scan.state = state
attack_paths_scan.progress = 100
attack_paths_scan.completed_at = now
attack_paths_scan.duration = duration
attack_paths_scan.ingestion_exceptions = ingestion_exceptions
attack_paths_scan.save(
update_fields=[
"state",
"progress",
"completed_at",
"duration",
"ingestion_exceptions",
]
)
def update_attack_paths_scan_progress(
attack_paths_scan: ProwlerAPIAttackPathsScan,
progress: int,
) -> None:
with rls_transaction(attack_paths_scan.tenant_id):
attack_paths_scan.progress = progress
attack_paths_scan.save(update_fields=["progress"])
def get_old_attack_paths_scans(
tenant_id: str,
provider_id: str,
attack_paths_scan_id: str,
) -> list[ProwlerAPIAttackPathsScan]:
"""
An `old_attack_paths_scan` is any `completed` Attack Paths scan for the same provider,
with its graph database not deleted, excluding the current Attack Paths scan.
"""
with rls_transaction(tenant_id):
completed_scans_qs = (
ProwlerAPIAttackPathsScan.objects.filter(
provider_id=provider_id,
state=StateChoices.COMPLETED,
is_graph_database_deleted=False,
)
.exclude(id=attack_paths_scan_id)
.all()
)
return list(completed_scans_qs)
def update_old_attack_paths_scan(
old_attack_paths_scan: ProwlerAPIAttackPathsScan,
) -> None:
with rls_transaction(old_attack_paths_scan.tenant_id):
old_attack_paths_scan.is_graph_database_deleted = True
old_attack_paths_scan.save(update_fields=["is_graph_database_deleted"])
def get_provider_graph_database_names(tenant_id: str, provider_id: str) -> list[str]:
"""
Return existing graph database names for a tenant/provider.
Note: For accesing the `AttackPathsScan` we need to use `all_objects` manager because the provider is soft-deleted.
"""
with rls_transaction(tenant_id):
graph_databases_names_qs = ProwlerAPIAttackPathsScan.all_objects.filter(
provider_id=provider_id,
is_graph_database_deleted=False,
).values_list("graph_database", flat=True)
return list(graph_databases_names_qs)
@@ -1,23 +0,0 @@
AVAILABLE_PROVIDERS: list[str] = [
"aws",
]
ROOT_NODE_LABELS: dict[str, str] = {
"aws": "AWSAccount",
}
NODE_UID_FIELDS: dict[str, str] = {
"aws": "arn",
}
def is_provider_available(provider_type: str) -> bool:
return provider_type in AVAILABLE_PROVIDERS
def get_root_node_label(provider_type: str) -> str:
return ROOT_NODE_LABELS.get(provider_type, "UnknownProviderAccount")
def get_node_uid_field(provider_type: str) -> str:
return NODE_UID_FIELDS.get(provider_type, "UnknownProviderUID")
@@ -1,205 +0,0 @@
import neo4j
from cartography.client.core.tx import run_write_query
from cartography.config import Config as CartographyConfig
from celery.utils.log import get_task_logger
from api.db_utils import rls_transaction
from api.models import Provider, ResourceFindingMapping
from config.env import env
from prowler.config import config as ProwlerConfig
from tasks.jobs.attack_paths.providers import get_node_uid_field, get_root_node_label
logger = get_task_logger(__name__)
BATCH_SIZE = env.int("NEO4J_INSERT_BATCH_SIZE", 500)
INDEX_STATEMENTS = [
"CREATE INDEX prowler_finding_id IF NOT EXISTS FOR (n:ProwlerFinding) ON (n.id);",
"CREATE INDEX prowler_finding_provider_uid IF NOT EXISTS FOR (n:ProwlerFinding) ON (n.provider_uid);",
"CREATE INDEX prowler_finding_lastupdated IF NOT EXISTS FOR (n:ProwlerFinding) ON (n.lastupdated);",
"CREATE INDEX prowler_finding_check_id IF NOT EXISTS FOR (n:ProwlerFinding) ON (n.status);",
]
INSERT_STATEMENT_TEMPLATE = """
UNWIND $findings_data AS finding_data
MATCH (account:__ROOT_NODE_LABEL__ {id: $provider_uid})
MATCH (account)-->(resource)
WHERE resource.__NODE_UID_FIELD__ = finding_data.resource_uid
OR resource.id = finding_data.resource_uid
MERGE (finding:ProwlerFinding {id: finding_data.id})
ON CREATE SET
finding.id = finding_data.id,
finding.uid = finding_data.uid,
finding.inserted_at = finding_data.inserted_at,
finding.updated_at = finding_data.updated_at,
finding.first_seen_at = finding_data.first_seen_at,
finding.scan_id = finding_data.scan_id,
finding.delta = finding_data.delta,
finding.status = finding_data.status,
finding.status_extended = finding_data.status_extended,
finding.severity = finding_data.severity,
finding.check_id = finding_data.check_id,
finding.check_title = finding_data.check_title,
finding.muted = finding_data.muted,
finding.muted_reason = finding_data.muted_reason,
finding.provider_uid = $provider_uid,
finding.firstseen = timestamp(),
finding.lastupdated = $last_updated,
finding._module_name = 'cartography:prowler',
finding._module_version = $prowler_version
ON MATCH SET
finding.status = finding_data.status,
finding.status_extended = finding_data.status_extended,
finding.lastupdated = $last_updated
MERGE (resource)-[rel:HAS_FINDING]->(finding)
ON CREATE SET
rel.provider_uid = $provider_uid,
rel.firstseen = timestamp(),
rel.lastupdated = $last_updated,
rel._module_name = 'cartography:prowler',
rel._module_version = $prowler_version
ON MATCH SET
rel.lastupdated = $last_updated
"""
CLEANUP_STATEMENT = """
MATCH (finding:ProwlerFinding {provider_uid: $provider_uid})
WHERE finding.lastupdated < $last_updated
WITH finding LIMIT $batch_size
DETACH DELETE finding
RETURN COUNT(finding) AS deleted_findings_count
"""
def create_indexes(neo4j_session: neo4j.Session) -> None:
"""
Code based on Cartography version 0.122.0, specifically on `cartography.intel.create_indexes.run`.
"""
logger.info("Creating indexes for Prowler node types.")
for statement in INDEX_STATEMENTS:
logger.debug("Executing statement: %s", statement)
run_write_query(neo4j_session, statement)
def analysis(
neo4j_session: neo4j.Session,
prowler_api_provider: Provider,
scan_id: str,
config: CartographyConfig,
) -> None:
findings_data = get_provider_last_scan_findings(prowler_api_provider, scan_id)
load_findings(neo4j_session, findings_data, prowler_api_provider, config)
cleanup_findings(neo4j_session, prowler_api_provider, config)
def get_provider_last_scan_findings(
prowler_api_provider: Provider,
scan_id: str,
) -> list[dict[str, str]]:
with rls_transaction(prowler_api_provider.tenant_id):
resource_finding_qs = ResourceFindingMapping.objects.filter(
finding__scan_id=scan_id,
).values(
"resource__uid",
"finding__id",
"finding__uid",
"finding__inserted_at",
"finding__updated_at",
"finding__first_seen_at",
"finding__scan_id",
"finding__delta",
"finding__status",
"finding__status_extended",
"finding__severity",
"finding__check_id",
"finding__check_metadata__checktitle",
"finding__muted",
"finding__muted_reason",
)
findings = []
for resource_finding in resource_finding_qs:
findings.append(
{
"resource_uid": str(resource_finding["resource__uid"]),
"id": str(resource_finding["finding__id"]),
"uid": resource_finding["finding__uid"],
"inserted_at": resource_finding["finding__inserted_at"],
"updated_at": resource_finding["finding__updated_at"],
"first_seen_at": resource_finding["finding__first_seen_at"],
"scan_id": str(resource_finding["finding__scan_id"]),
"delta": resource_finding["finding__delta"],
"status": resource_finding["finding__status"],
"status_extended": resource_finding["finding__status_extended"],
"severity": resource_finding["finding__severity"],
"check_id": str(resource_finding["finding__check_id"]),
"check_title": resource_finding[
"finding__check_metadata__checktitle"
],
"muted": resource_finding["finding__muted"],
"muted_reason": resource_finding["finding__muted_reason"],
}
)
return findings
def load_findings(
neo4j_session: neo4j.Session,
findings_data: list[dict[str, str]],
prowler_api_provider: Provider,
config: CartographyConfig,
) -> None:
replacements = {
"__ROOT_NODE_LABEL__": get_root_node_label(prowler_api_provider.provider),
"__NODE_UID_FIELD__": get_node_uid_field(prowler_api_provider.provider),
}
query = INSERT_STATEMENT_TEMPLATE
for replace_key, replace_value in replacements.items():
query = query.replace(replace_key, replace_value)
parameters = {
"provider_uid": str(prowler_api_provider.uid),
"last_updated": config.update_tag,
"prowler_version": ProwlerConfig.prowler_version,
}
total_length = len(findings_data)
for i in range(0, total_length, BATCH_SIZE):
parameters["findings_data"] = findings_data[i : i + BATCH_SIZE]
logger.info(
f"Loading findings batch {i // BATCH_SIZE + 1} / {(total_length + BATCH_SIZE - 1) // BATCH_SIZE}"
)
neo4j_session.run(query, parameters)
def cleanup_findings(
neo4j_session: neo4j.Session,
prowler_api_provider: Provider,
config: CartographyConfig,
) -> None:
parameters = {
"provider_uid": str(prowler_api_provider.uid),
"last_updated": config.update_tag,
"batch_size": BATCH_SIZE,
}
batch = 1
deleted_count = 1
while deleted_count > 0:
logger.info(f"Cleaning findings batch {batch}")
result = neo4j_session.run(CLEANUP_STATEMENT, parameters)
deleted_count = result.single().get("deleted_findings_count", 0)
batch += 1
@@ -1,183 +0,0 @@
import logging
import time
import asyncio
from typing import Any, Callable
from cartography.config import Config as CartographyConfig
from cartography.intel import analysis as cartography_analysis
from cartography.intel import create_indexes as cartography_create_indexes
from cartography.intel import ontology as cartography_ontology
from celery.utils.log import get_task_logger
from api.attack_paths import database as graph_database
from api.db_utils import rls_transaction
from api.models import (
Provider as ProwlerAPIProvider,
StateChoices,
)
from api.utils import initialize_prowler_provider
from tasks.jobs.attack_paths import aws, db_utils, prowler, utils
# Without this Celery goes crazy with Cartography logging
logging.getLogger("cartography").setLevel(logging.ERROR)
logging.getLogger("neo4j").propagate = False
logger = get_task_logger(__name__)
CARTOGRAPHY_INGESTION_FUNCTIONS: dict[str, Callable] = {
"aws": aws.start_aws_ingestion,
}
def get_cartography_ingestion_function(provider_type: str) -> Callable | None:
return CARTOGRAPHY_INGESTION_FUNCTIONS.get(provider_type)
def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
"""
Code based on Cartography version 0.122.0, specifically on `cartography.cli.main`, `cartography.cli.CLI.main`,
`cartography.sync.run_with_config` and `cartography.sync.Sync.run`.
"""
ingestion_exceptions = {} # This will hold any exceptions raised during ingestion
# Prowler necessary objects
with rls_transaction(tenant_id):
prowler_api_provider = ProwlerAPIProvider.objects.get(scan__pk=scan_id)
prowler_sdk_provider = initialize_prowler_provider(prowler_api_provider)
# Attack Paths Scan necessary objects
cartography_ingestion_function = get_cartography_ingestion_function(
prowler_api_provider.provider
)
attack_paths_scan = db_utils.retrieve_attack_paths_scan(tenant_id, scan_id)
# Checks before starting the scan
if not cartography_ingestion_function:
ingestion_exceptions = {
"global_error": f"Provider {prowler_api_provider.provider} is not supported for Attack Paths scans"
}
if attack_paths_scan:
db_utils.finish_attack_paths_scan(
attack_paths_scan, StateChoices.COMPLETED, ingestion_exceptions
)
logger.warning(
f"Provider {prowler_api_provider.provider} is not supported for Attack Paths scans"
)
return ingestion_exceptions
else:
if not attack_paths_scan:
logger.warning(
f"No Attack Paths Scan found for scan {scan_id} and tenant {tenant_id}, let's create it then"
)
attack_paths_scan = db_utils.create_attack_paths_scan(
tenant_id, scan_id, prowler_api_provider.id
)
# While creating the Cartography configuration, attributes `neo4j_user` and `neo4j_password` are not really needed in this config object
cartography_config = CartographyConfig(
neo4j_uri=graph_database.get_uri(),
neo4j_database=graph_database.get_database_name(attack_paths_scan.id),
update_tag=int(time.time()),
)
# Starting the Attack Paths scan
db_utils.starting_attack_paths_scan(attack_paths_scan, task_id, cartography_config)
try:
logger.info(
f"Creating Neo4j database {cartography_config.neo4j_database} for tenant {prowler_api_provider.tenant_id}"
)
graph_database.create_database(cartography_config.neo4j_database)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 1)
logger.info(
f"Starting Cartography ({attack_paths_scan.id}) for "
f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}"
)
with graph_database.get_session(
cartography_config.neo4j_database
) as neo4j_session:
# Indexes creation
cartography_create_indexes.run(neo4j_session, cartography_config)
prowler.create_indexes(neo4j_session)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 2)
# The real scan, where iterates over cloud services
ingestion_exceptions = _call_within_event_loop(
cartography_ingestion_function,
neo4j_session,
cartography_config,
prowler_api_provider,
prowler_sdk_provider,
attack_paths_scan,
)
# Post-processing: Just keeping it to be more Cartography compliant
cartography_ontology.run(neo4j_session, cartography_config)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 95)
cartography_analysis.run(neo4j_session, cartography_config)
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 96)
# Adding Prowler nodes and relationships
prowler.analysis(
neo4j_session, prowler_api_provider, scan_id, cartography_config
)
logger.info(
f"Completed Cartography ({attack_paths_scan.id}) for "
f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}"
)
# Handling databases changes
old_attack_paths_scans = db_utils.get_old_attack_paths_scans(
prowler_api_provider.tenant_id,
prowler_api_provider.id,
attack_paths_scan.id,
)
for old_attack_paths_scan in old_attack_paths_scans:
graph_database.drop_database(old_attack_paths_scan.graph_database)
db_utils.update_old_attack_paths_scan(old_attack_paths_scan)
db_utils.finish_attack_paths_scan(
attack_paths_scan, StateChoices.COMPLETED, ingestion_exceptions
)
return ingestion_exceptions
except Exception as e:
exception_message = utils.stringify_exception(e, "Cartography failed")
logger.error(exception_message)
ingestion_exceptions["global_cartography_error"] = exception_message
# Handling databases changes
graph_database.drop_database(cartography_config.neo4j_database)
db_utils.finish_attack_paths_scan(
attack_paths_scan, StateChoices.FAILED, ingestion_exceptions
)
raise
def _call_within_event_loop(fn, *args, **kwargs):
"""
Cartography needs a running event loop, so assuming there is none (Celery task or even regular DRF endpoint),
let's create a new one and set it as the current event loop for this thread.
"""
loop = asyncio.new_event_loop()
try:
asyncio.set_event_loop(loop)
return fn(*args, **kwargs)
finally:
try:
loop.run_until_complete(loop.shutdown_asyncgens())
except Exception:
pass
loop.close()
asyncio.set_event_loop(None)
@@ -1,10 +0,0 @@
import traceback
from datetime import datetime, timezone
def stringify_exception(exception: Exception, context: str) -> str:
timestamp = datetime.now(tz=timezone.utc)
exception_traceback = traceback.TracebackException.from_exception(exception)
traceback_string = "".join(exception_traceback.format())
return f"{timestamp} - {context}\n{traceback_string}"
+4 -215
View File
@@ -1,40 +1,26 @@
from collections import defaultdict
from datetime import timedelta
from celery.utils.log import get_task_logger
from django.db.models import OuterRef, Subquery, Sum
from django.db.models import Sum
from django.utils import timezone
from tasks.jobs.queries import (
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL,
)
from tasks.jobs.scan import aggregate_category_counts, aggregate_resource_group_counts
from tasks.jobs.scan import aggregate_category_counts
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.db_utils import (
POSTGRES_TENANT_VAR,
SET_CONFIG_QUERY,
psycopg_connection,
rls_transaction,
)
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.models import (
ComplianceOverviewSummary,
ComplianceRequirementOverview,
DailySeveritySummary,
Finding,
ProviderComplianceScore,
Resource,
ResourceFindingMapping,
ResourceScanSummary,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
StateChoices,
)
logger = get_task_logger(__name__)
def backfill_resource_scan_summaries(tenant_id: str, scan_id: str):
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
@@ -355,200 +341,3 @@ def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
)
return {"status": "backfilled", "categories_count": len(category_counts)}
def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str):
"""
Backfill ScanGroupSummary for a completed scan.
Aggregates resource group counts from all findings in the scan and creates
one ScanGroupSummary row per (resource_group, severity) combination.
Args:
tenant_id: Target tenant UUID
scan_id: Scan UUID to backfill
Returns:
dict: Status indicating whether backfill was performed
"""
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
if ScanGroupSummary.objects.filter(
tenant_id=tenant_id, scan_id=scan_id
).exists():
return {"status": "already backfilled"}
if not Scan.objects.filter(
tenant_id=tenant_id,
id=scan_id,
state__in=(StateChoices.COMPLETED, StateChoices.FAILED),
).exists():
return {"status": "scan is not completed"}
resource_group_counts: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
# Get findings with their first resource UID via annotation
resource_uid_subquery = ResourceFindingMapping.objects.filter(
finding_id=OuterRef("id"), tenant_id=tenant_id
).values("resource__uid")[:1]
for finding in (
Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id)
.annotate(resource_uid=Subquery(resource_uid_subquery))
.values(
"resource_groups",
"severity",
"status",
"delta",
"muted",
"resource_uid",
)
):
aggregate_resource_group_counts(
resource_group=finding.get("resource_groups"),
severity=finding.get("severity"),
status=finding.get("status"),
delta=finding.get("delta"),
muted=finding.get("muted", False),
resource_uid=finding.get("resource_uid") or "",
cache=resource_group_counts,
group_resources_cache=group_resources_cache,
)
if not resource_group_counts:
return {"status": "no resource groups to backfill"}
# Compute group-level resource counts (same value for all severity rows in a group)
group_resource_counts = {
grp: len(uids) for grp, uids in group_resources_cache.items()
}
resource_group_summaries = [
ScanGroupSummary(
tenant_id=tenant_id,
scan_id=scan_id,
resource_group=grp,
severity=severity,
total_findings=counts["total"],
failed_findings=counts["failed"],
new_failed_findings=counts["new_failed"],
resources_count=group_resource_counts.get(grp, 0),
)
for (grp, severity), counts in resource_group_counts.items()
]
with rls_transaction(tenant_id):
ScanGroupSummary.objects.bulk_create(
resource_group_summaries, batch_size=500, ignore_conflicts=True
)
return {"status": "backfilled", "resource_groups_count": len(resource_group_counts)}
def backfill_provider_compliance_scores(tenant_id: str) -> dict:
"""
Backfill ProviderComplianceScore from latest completed scan per provider.
For each provider with completed scans, finds the most recent scan and
upserts compliance requirement statuses with FAIL-dominant aggregation.
Args:
tenant_id: Target tenant UUID
Returns:
dict: Statistics about the backfill operation
"""
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
completed_scans = Scan.all_objects.filter(
tenant_id=tenant_id,
state=StateChoices.COMPLETED,
completed_at__isnull=False,
)
if not completed_scans.exists():
return {"status": "no completed scans"}
existing_providers = set(
ProviderComplianceScore.objects.filter(tenant_id=tenant_id)
.values_list("provider_id", flat=True)
.distinct()
)
if existing_providers:
completed_scans = completed_scans.exclude(
provider_id__in=existing_providers
)
scan_info = list(
completed_scans.order_by("provider_id", "-completed_at")
.distinct("provider_id")
.values("id", "provider_id", "completed_at")
)
if not scan_info:
return {"status": "no scans to process"}
total_upserted = 0
providers_processed = 0
providers_skipped = 0
for scan in scan_info:
provider_id = scan["provider_id"]
scan_id = scan["id"]
try:
with psycopg_connection(MainRouter.default_db) as connection:
connection.autocommit = False
try:
with connection.cursor() as cursor:
cursor.execute(
SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id]
)
cursor.execute(
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
[tenant_id, str(scan_id)],
)
upserted = cursor.rowcount
connection.commit()
total_upserted += upserted
providers_processed += 1
except Exception:
connection.rollback()
raise
except Exception as e:
providers_skipped += 1
logger.exception(
"Error backfilling provider %s for tenant %s: %s",
provider_id,
tenant_id,
e,
)
# Recalculate tenant summary after all providers are backfilled
if providers_processed > 0:
with psycopg_connection(MainRouter.default_db) as connection:
connection.autocommit = False
try:
with connection.cursor() as cursor:
cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id])
# Advisory lock to prevent race conditions
cursor.execute(
"SELECT pg_advisory_xact_lock(hashtext(%s))", [tenant_id]
)
cursor.execute(
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL,
[tenant_id, tenant_id],
)
tenant_summary_count = cursor.rowcount
connection.commit()
except Exception:
connection.rollback()
raise
else:
tenant_summary_count = 0
return {
"status": "backfilled",
"providers_processed": providers_processed,
"providers_skipped": providers_skipped,
"total_upserted": total_upserted,
"tenant_summary_count": tenant_summary_count,
}
+2 -24
View File
@@ -1,19 +1,9 @@
from celery.utils.log import get_task_logger
from django.db import DatabaseError
from api.attack_paths import database as graph_database
from api.db_router import MainRouter
from api.db_utils import batch_delete, rls_transaction
from api.models import (
AttackPathsScan,
Finding,
Provider,
Resource,
Scan,
ScanSummary,
Tenant,
)
from tasks.jobs.attack_paths.db_utils import get_provider_graph_database_names
from api.models import Finding, Provider, Resource, Scan, ScanSummary, Tenant
logger = get_task_logger(__name__)
@@ -33,27 +23,16 @@ def delete_provider(tenant_id: str, pk: str):
Raises:
Provider.DoesNotExist: If no instance with the provided primary key exists.
"""
# Delete the Attack Paths' graph databases related to the provider
graph_database_names = get_provider_graph_database_names(tenant_id, pk)
try:
for graph_database_name in graph_database_names:
graph_database.drop_database(graph_database_name)
except graph_database.GraphDatabaseQueryException as gdb_error:
logger.error(f"Error deleting Provider databases: {gdb_error}")
raise
# Get all provider related data and delete them in batches
with rls_transaction(tenant_id):
instance = Provider.all_objects.get(pk=pk)
deletion_summary = {}
deletion_steps = [
("Scan Summaries", ScanSummary.all_objects.filter(scan__provider=instance)),
("Findings", Finding.all_objects.filter(scan__provider=instance)),
("Resources", Resource.all_objects.filter(provider=instance)),
("Scans", Scan.all_objects.filter(provider=instance)),
("AttackPathsScans", AttackPathsScan.all_objects.filter(provider=instance)),
]
deletion_summary = {}
for step_name, queryset in deletion_steps:
try:
_, step_summary = batch_delete(tenant_id, queryset)
@@ -69,7 +48,6 @@ def delete_provider(tenant_id: str, pk: str):
except DatabaseError as db_error:
logger.error(f"Error deleting Provider: {db_error}")
raise
return deletion_summary
-11
View File
@@ -27,7 +27,6 @@ from prowler.lib.outputs.compliance.c5.c5_gcp import GCPC5
from prowler.lib.outputs.compliance.ccc.ccc_aws import CCC_AWS
from prowler.lib.outputs.compliance.ccc.ccc_azure import CCC_Azure
from prowler.lib.outputs.compliance.ccc.ccc_gcp import CCC_GCP
from prowler.lib.outputs.compliance.cis.cis_alibabacloud import AlibabaCloudCIS
from prowler.lib.outputs.compliance.cis.cis_aws import AWSCIS
from prowler.lib.outputs.compliance.cis.cis_azure import AzureCIS
from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS
@@ -51,9 +50,6 @@ from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_azure import (
AzureMitreAttack,
)
from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_gcp import GCPMitreAttack
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_alibaba import (
ProwlerThreatScoreAlibaba,
)
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_aws import (
ProwlerThreatScoreAWS,
)
@@ -132,13 +128,6 @@ COMPLIANCE_CLASS_MAP = {
"oraclecloud": [
(lambda name: name.startswith("cis_"), OracleCloudCIS),
],
"alibabacloud": [
(lambda name: name.startswith("cis_"), AlibabaCloudCIS),
(
lambda name: name == "prowler_threatscore_alibabacloud",
ProwlerThreatScoreAlibaba,
),
],
}
+15 -16
View File
@@ -19,9 +19,6 @@ from prowler.providers.aws.aws_provider import AwsProvider
from prowler.providers.aws.lib.s3.s3 import S3
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub
from prowler.providers.common.models import Connection
from prowler.providers.aws.lib.security_hub.exceptions.exceptions import (
SecurityHubNoEnabledRegionsError,
)
logger = get_task_logger(__name__)
@@ -225,9 +222,8 @@ def get_security_hub_client_from_integration(
)
return True, security_hub
else:
# Reset regions information if connection fails and integration is not connected
# Reset regions information if connection fails
with rls_transaction(tenant_id, using=MainRouter.default_db):
integration.connected = False
integration.configuration["regions"] = {}
integration.save()
@@ -334,18 +330,15 @@ def upload_security_hub_integration(
)
if not connected:
if isinstance(
security_hub.error,
SecurityHubNoEnabledRegionsError,
logger.error(
f"Security Hub connection failed for integration {integration.id}: "
f"{security_hub.error}"
)
with rls_transaction(
tenant_id, using=MainRouter.default_db
):
logger.warning(
f"Security Hub integration {integration.id} has no enabled regions"
)
else:
logger.error(
f"Security Hub connection failed for integration {integration.id}: "
f"{security_hub.error}"
)
integration.connected = False
integration.save()
break # Skip this integration
security_hub_client = security_hub
@@ -416,16 +409,22 @@ def upload_security_hub_integration(
logger.warning(
f"Failed to archive previous findings: {str(archive_error)}"
)
except Exception as e:
logger.error(
f"Security Hub integration {integration.id} failed: {str(e)}"
)
continue
result = integration_executions == len(integrations)
if result:
logger.info(
f"All Security Hub integrations completed successfully for provider {provider_id}"
)
else:
logger.error(
f"Some Security Hub integrations failed for provider {provider_id}"
)
return result
@@ -11,41 +11,6 @@ from api.models import LighthouseProviderConfiguration, LighthouseProviderModels
logger = get_task_logger(__name__)
# OpenAI model prefixes to exclude from Lighthouse model selection.
# These models don't support text chat completions and tool calling.
EXCLUDED_OPENAI_MODEL_PREFIXES = (
"dall-e", # Image generation
"whisper", # Audio transcription
"tts-", # Text-to-speech (tts-1, tts-1-hd, etc.)
"sora", # Text-to-video (sora-2, sora-2-pro, etc.)
"text-embedding", # Embeddings
"embedding", # Embeddings (alternative naming)
"text-moderation", # Content moderation
"omni-moderation", # Content moderation
"text-davinci", # Legacy completion models
"text-curie", # Legacy completion models
"text-babbage", # Legacy completion models
"text-ada", # Legacy completion models
"davinci", # Legacy completion models
"curie", # Legacy completion models
"babbage", # Legacy completion models
"ada", # Legacy completion models
"computer-use", # Computer control agent
"gpt-image", # Image generation
"gpt-audio", # Audio models
"gpt-realtime", # Realtime voice API
)
# OpenAI model substrings to exclude (patterns that can appear anywhere in model ID).
# These patterns identify non-chat model variants.
EXCLUDED_OPENAI_MODEL_SUBSTRINGS = (
"-audio-", # Audio preview models (gpt-4o-audio-preview, etc.)
"-realtime-", # Realtime preview models (gpt-4o-realtime-preview, etc.)
"-transcribe", # Transcription models (gpt-4o-transcribe, etc.)
"-tts", # TTS models (gpt-4o-mini-tts)
"-instruct", # Legacy instruct models (gpt-3.5-turbo-instruct, etc.)
)
def _extract_error_message(e: Exception) -> str:
"""
@@ -318,41 +283,20 @@ def _fetch_openai_models(api_key: str) -> Dict[str, str]:
"""
Fetch available models from OpenAI API.
Filters out models that don't support text input/output and tool calling,
such as image generation (DALL-E), audio transcription (Whisper),
text-to-speech (TTS), embeddings, and moderation models.
Args:
api_key: OpenAI API key for authentication.
Returns:
Dict mapping model_id to model_name. For OpenAI, both are the same
as the API doesn't provide separate display names. Only includes
models that support text input, text output or tool calling.
as the API doesn't provide separate display names.
Raises:
Exception: If the API call fails.
"""
client = openai.OpenAI(api_key=api_key)
models = client.models.list()
# Filter models to only include those supporting chat completions + tool calling
filtered_models = {}
for model in getattr(models, "data", []):
model_id = model.id
# Skip if model ID starts with excluded prefixes
if model_id.startswith(EXCLUDED_OPENAI_MODEL_PREFIXES):
continue
# Skip if model ID contains excluded substrings
if any(substring in model_id for substring in EXCLUDED_OPENAI_MODEL_SUBSTRINGS):
continue
# Include model (supports chat completions + tool calling)
filtered_models[model_id] = model_id
return filtered_models
# OpenAI uses model.id for both ID and display name
return {m.id: m.id for m in getattr(models, "data", [])}
def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, str]:
-134
View File
@@ -1,134 +0,0 @@
"""
Shared SQL queries for tasks.
This module centralizes raw SQL queries used across multiple task modules
to ensure consistency and maintainability.
"""
# =============================================================================
# COMPLIANCE SCORE QUERIES
# =============================================================================
# Upsert provider compliance scores from a scan's compliance requirements.
# Uses FAIL-dominant aggregation: FAIL > MANUAL > PASS
# Parameters: [tenant_id, scan_id]
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL = """
INSERT INTO provider_compliance_scores
(id, tenant_id, provider_id, scan_id, compliance_id, requirement_id,
requirement_status, scan_completed_at)
SELECT
gen_random_uuid(),
agg.tenant_id,
agg.provider_id,
agg.scan_id,
agg.compliance_id,
agg.requirement_id,
agg.requirement_status,
agg.completed_at
FROM (
SELECT DISTINCT ON (cro.compliance_id, cro.requirement_id)
cro.tenant_id,
s.provider_id,
cro.scan_id,
cro.compliance_id,
cro.requirement_id,
(CASE
WHEN bool_or(cro.requirement_status = 'FAIL')
OVER (PARTITION BY cro.compliance_id, cro.requirement_id) THEN 'FAIL'
WHEN bool_or(cro.requirement_status = 'MANUAL')
OVER (PARTITION BY cro.compliance_id, cro.requirement_id) THEN 'MANUAL'
ELSE 'PASS'
END)::status as requirement_status,
s.completed_at
FROM compliance_requirements_overviews cro
JOIN scans s ON s.id = cro.scan_id
WHERE cro.tenant_id = %s AND cro.scan_id = %s
ORDER BY cro.compliance_id, cro.requirement_id
) agg
ON CONFLICT (tenant_id, provider_id, compliance_id, requirement_id)
DO UPDATE SET
requirement_status = EXCLUDED.requirement_status,
scan_id = EXCLUDED.scan_id,
scan_completed_at = EXCLUDED.scan_completed_at
WHERE EXCLUDED.scan_completed_at > provider_compliance_scores.scan_completed_at
"""
# Upsert tenant compliance summary for specific compliance IDs.
# Aggregates across all providers with FAIL-dominant logic at requirement level.
# Parameters: [tenant_id, tenant_id, compliance_ids_array]
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL = """
INSERT INTO tenant_compliance_summaries
(id, tenant_id, compliance_id,
requirements_passed, requirements_failed, requirements_manual,
total_requirements, updated_at)
SELECT
gen_random_uuid(),
%s as tenant_id,
compliance_id,
COUNT(*) FILTER (WHERE req_status = 'PASS') as requirements_passed,
COUNT(*) FILTER (WHERE req_status = 'FAIL') as requirements_failed,
COUNT(*) FILTER (WHERE req_status = 'MANUAL') as requirements_manual,
COUNT(*) as total_requirements,
NOW() as updated_at
FROM (
SELECT
compliance_id,
requirement_id,
CASE
WHEN bool_or(requirement_status = 'FAIL') THEN 'FAIL'
WHEN bool_or(requirement_status = 'MANUAL') THEN 'MANUAL'
ELSE 'PASS'
END as req_status
FROM provider_compliance_scores
WHERE tenant_id = %s AND compliance_id = ANY(%s)
GROUP BY compliance_id, requirement_id
) req_agg
GROUP BY compliance_id
ON CONFLICT (tenant_id, compliance_id)
DO UPDATE SET
requirements_passed = EXCLUDED.requirements_passed,
requirements_failed = EXCLUDED.requirements_failed,
requirements_manual = EXCLUDED.requirements_manual,
total_requirements = EXCLUDED.total_requirements,
updated_at = NOW()
"""
# Upsert tenant compliance summary for ALL compliance IDs in tenant.
# Used by backfill when recalculating entire tenant summary.
# Parameters: [tenant_id, tenant_id]
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL = """
INSERT INTO tenant_compliance_summaries
(id, tenant_id, compliance_id,
requirements_passed, requirements_failed, requirements_manual,
total_requirements, updated_at)
SELECT
gen_random_uuid(),
%s as tenant_id,
compliance_id,
COUNT(*) FILTER (WHERE req_status = 'PASS') as requirements_passed,
COUNT(*) FILTER (WHERE req_status = 'FAIL') as requirements_failed,
COUNT(*) FILTER (WHERE req_status = 'MANUAL') as requirements_manual,
COUNT(*) as total_requirements,
NOW() as updated_at
FROM (
SELECT
compliance_id,
requirement_id,
CASE
WHEN bool_or(requirement_status = 'FAIL') THEN 'FAIL'
WHEN bool_or(requirement_status = 'MANUAL') THEN 'MANUAL'
ELSE 'PASS'
END as req_status
FROM provider_compliance_scores
WHERE tenant_id = %s
GROUP BY compliance_id, requirement_id
) req_agg
GROUP BY compliance_id
ON CONFLICT (tenant_id, compliance_id)
DO UPDATE SET
requirements_passed = EXCLUDED.requirements_passed,
requirements_failed = EXCLUDED.requirements_failed,
requirements_manual = EXCLUDED.requirements_manual,
total_requirements = EXCLUDED.total_requirements,
updated_at = NOW()
"""
+9 -56
View File
@@ -243,28 +243,15 @@ def _safe_getattr(obj, attr: str, default: str = "N/A") -> str:
def _create_info_table_style() -> TableStyle:
"""Create a reusable table style for information/metadata tables.
ReportLab TableStyle coordinate system:
- Format: (COMMAND, (start_col, start_row), (end_col, end_row), value)
- Coordinates use (column, row) format, starting at (0, 0) for top-left cell
- Negative indices work like Python slicing: -1 means "last row/column"
- (0, 0) to (0, -1) = entire first column (all rows)
- (0, 0) to (-1, 0) = entire first row (all columns)
- (0, 0) to (-1, -1) = entire table
- Styles are applied in order; later rules override earlier ones
"""
"""Create a reusable table style for information/metadata tables."""
return TableStyle(
[
# Column 0 (labels): blue background with white text
("BACKGROUND", (0, 0), (0, -1), COLOR_BLUE),
("TEXTCOLOR", (0, 0), (0, -1), COLOR_WHITE),
("FONTNAME", (0, 0), (0, -1), "FiraCode"),
# Column 1 (values): light blue background with gray text
("BACKGROUND", (1, 0), (1, -1), COLOR_BG_BLUE),
("TEXTCOLOR", (1, 0), (1, -1), COLOR_GRAY),
("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"),
# Apply to entire table
("ALIGN", (0, 0), (-1, -1), "LEFT"),
("VALIGN", (0, 0), (-1, -1), "TOP"),
("FONTSIZE", (0, 0), (-1, -1), 11),
@@ -278,30 +265,19 @@ def _create_info_table_style() -> TableStyle:
def _create_header_table_style(header_color: colors.Color = None) -> TableStyle:
"""Create a reusable table style for tables with headers.
ReportLab TableStyle coordinate system:
- Format: (COMMAND, (start_col, start_row), (end_col, end_row), value)
- (0, 0) to (-1, 0) = entire first row (header row)
- (1, 1) to (-1, -1) = all data cells (excludes header row and first column)
- See _create_info_table_style() for full coordinate system documentation
"""
"""Create a reusable table style for tables with headers."""
if header_color is None:
header_color = COLOR_BLUE
return TableStyle(
[
# Header row (row 0): colored background with white text
("BACKGROUND", (0, 0), (-1, 0), header_color),
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
("FONTSIZE", (0, 0), (-1, 0), 10),
# Apply to entire table
("ALIGN", (0, 0), (-1, -1), "CENTER"),
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
# Data cells (excluding header): smaller font
("FONTSIZE", (1, 1), (-1, -1), 9),
# Apply to entire table
("GRID", (0, 0), (-1, -1), 1, COLOR_GRID_GRAY),
("LEFTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
("RIGHTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
@@ -312,30 +288,18 @@ def _create_header_table_style(header_color: colors.Color = None) -> TableStyle:
def _create_findings_table_style() -> TableStyle:
"""Create a reusable table style for findings tables.
ReportLab TableStyle coordinate system:
- Format: (COMMAND, (start_col, start_row), (end_col, end_row), value)
- (0, 0) to (-1, 0) = entire first row (header row)
- (0, 0) to (0, 0) = only the top-left cell
- See _create_info_table_style() for full coordinate system documentation
"""
"""Create a reusable table style for findings tables."""
return TableStyle(
[
# Header row (row 0): colored background with white text
("BACKGROUND", (0, 0), (-1, 0), COLOR_BLUE),
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
# Only top-left cell centered (for index/number column)
("ALIGN", (0, 0), (0, 0), "CENTER"),
# Apply to entire table
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
("FONTSIZE", (0, 0), (-1, -1), 9),
("GRID", (0, 0), (-1, -1), 0.1, COLOR_BORDER_GRAY),
# Remove padding only from top-left cell
("LEFTPADDING", (0, 0), (0, 0), 0),
("RIGHTPADDING", (0, 0), (0, 0), 0),
# Apply to entire table
("TOPPADDING", (0, 0), (-1, -1), PADDING_SMALL),
("BOTTOMPADDING", (0, 0), (-1, -1), PADDING_SMALL),
]
@@ -1139,15 +1103,11 @@ def generate_threatscore_report(
elements.append(Spacer(1, 0.5 * inch))
# Add compliance information table
provider_alias = provider_obj.alias or "N/A"
info_data = [
["Framework:", compliance_framework],
["ID:", compliance_id],
["Name:", Paragraph(compliance_name, normal_center)],
["Version:", compliance_version],
["Provider:", provider_type.upper()],
["Account ID:", provider_obj.uid],
["Alias:", provider_alias],
["Scan ID:", scan_id],
["Description:", Paragraph(compliance_description, normal_center)],
]
@@ -2099,15 +2059,12 @@ def generate_ens_report(
elements.append(Spacer(1, 0.5 * inch))
# Add compliance information table
provider_alias = provider_obj.alias or "N/A"
info_data = [
["Framework:", compliance_framework],
["ID:", compliance_id],
["Nombre:", Paragraph(compliance_name, normal_center)],
["Versión:", compliance_version],
["Proveedor:", provider_type.upper()],
["Account ID:", provider_obj.uid],
["Alias:", provider_alias],
["Scan ID:", scan_id],
["Descripción:", Paragraph(compliance_description, normal_center)],
]
@@ -2115,12 +2072,12 @@ def generate_ens_report(
info_table.setStyle(
TableStyle(
[
("BACKGROUND", (0, 0), (0, -1), colors.Color(0.2, 0.4, 0.6)),
("TEXTCOLOR", (0, 0), (0, -1), colors.white),
("FONTNAME", (0, 0), (0, -1), "FiraCode"),
("BACKGROUND", (1, 0), (1, -1), colors.Color(0.95, 0.97, 1.0)),
("TEXTCOLOR", (1, 0), (1, -1), colors.Color(0.2, 0.2, 0.2)),
("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"),
("BACKGROUND", (0, 0), (0, 6), colors.Color(0.2, 0.4, 0.6)),
("TEXTCOLOR", (0, 0), (0, 6), colors.white),
("FONTNAME", (0, 0), (0, 6), "FiraCode"),
("BACKGROUND", (1, 0), (1, 6), colors.Color(0.95, 0.97, 1.0)),
("TEXTCOLOR", (1, 0), (1, 6), colors.Color(0.2, 0.2, 0.2)),
("FONTNAME", (1, 0), (1, 6), "PlusJakartaSans"),
("ALIGN", (0, 0), (-1, -1), "LEFT"),
("VALIGN", (0, 0), (-1, -1), "TOP"),
("FONTSIZE", (0, 0), (-1, -1), 11),
@@ -3040,14 +2997,11 @@ def generate_nis2_report(
elements.append(Spacer(1, 0.3 * inch))
# Compliance metadata table
provider_alias = provider_obj.alias or "N/A"
metadata_data = [
["Framework:", compliance_framework],
["Name:", Paragraph(compliance_name, normal_center)],
["Version:", compliance_version or "N/A"],
["Provider:", provider_type.upper()],
["Account ID:", provider_obj.uid],
["Alias:", provider_alias],
["Scan ID:", scan_id],
["Description:", Paragraph(compliance_description, normal_center)],
]
@@ -3531,7 +3485,6 @@ def generate_compliance_reports(
"gcp",
"m365",
"kubernetes",
"alibabacloud",
]:
logger.info(
f"Provider {provider_id} ({provider_type}) is not supported for ThreatScore report"
+1 -258
View File
@@ -14,10 +14,6 @@ from config.env import env
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
from django.db import IntegrityError, OperationalError
from django.db.models import Case, Count, IntegerField, Prefetch, Q, Sum, When
from tasks.jobs.queries import (
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
)
from tasks.utils import CustomEncoder
from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
@@ -45,7 +41,6 @@ from api.models import (
ResourceTag,
Scan,
ScanCategorySummary,
ScanGroupSummary,
ScanSummary,
StateChoices,
)
@@ -128,50 +123,6 @@ def aggregate_category_counts(
cache[key]["new_failed"] += 1
def aggregate_resource_group_counts(
resource_group: str | None,
severity: str,
status: str,
delta: str | None,
muted: bool,
resource_uid: str,
cache: dict[tuple[str, str], dict[str, int]],
group_resources_cache: dict[str, set],
) -> None:
"""
Increment resource group counters in-place for a finding.
Args:
resource_group: Resource group from check metadata (e.g., "database", "compute").
severity: Severity level (e.g., "high", "medium").
status: Finding status as string ("FAIL", "PASS").
delta: Delta value as string ("new", "changed") or None.
muted: Whether the finding is muted.
resource_uid: Unique identifier for the resource to count distinct resources.
cache: Dict {(resource_group, severity): {"total", "failed", "new_failed"}} to update.
group_resources_cache: Dict {resource_group: set(resource_uids)} for group-level resource tracking.
"""
if not resource_group:
return
is_failed = status == "FAIL" and not muted
is_new_failed = is_failed and delta == "new"
key = (resource_group, severity)
if key not in cache:
cache[key] = {"total": 0, "failed": 0, "new_failed": 0}
if not muted:
cache[key]["total"] += 1
if is_failed:
cache[key]["failed"] += 1
if is_new_failed:
cache[key]["new_failed"] += 1
# Track resources at GROUP level (not per-severity) to avoid over-counting
if resource_uid and not muted:
group_resources_cache.setdefault(resource_group, set()).add(resource_uid)
def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict:
global _ATTACK_SURFACE_MAPPING_CACHE
@@ -483,8 +434,6 @@ def _process_finding_micro_batch(
scan_resource_cache: set,
mute_rules_cache: dict,
scan_categories_cache: dict[tuple[str, str], dict[str, int]],
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]],
group_resources_cache: dict[str, set],
) -> None:
"""
Process a micro-batch of findings and persist them using bulk operations.
@@ -506,8 +455,6 @@ def _process_finding_micro_batch(
scan_resource_cache: Set of tuples used to create `ResourceScanSummary` rows.
mute_rules_cache: Map of finding UID -> mute reason gathered before the scan.
scan_categories_cache: Dict tracking category counts {(category, severity): {"total", "failed", "new_failed"}}.
scan_resource_groups_cache: Dict tracking resource group counts {(resource_group, severity): {"total", "failed", "new_failed"}}.
group_resources_cache: Dict tracking unique resources per group {resource_group: set(resource_uids)}.
"""
# Accumulate objects for bulk operations
findings_to_create = []
@@ -548,8 +495,6 @@ def _process_finding_micro_batch(
with rls_transaction(tenant_id):
resource_uid = finding.resource_uid
if resource_uid not in resource_cache:
check_metadata = finding.get_metadata()
group = check_metadata.get("resourcegroup") or None
resource_instance, _ = Resource.objects.get_or_create(
tenant_id=tenant_id,
provider=provider_instance,
@@ -559,7 +504,6 @@ def _process_finding_micro_batch(
"service": finding.service_name,
"type": finding.resource_type,
"name": finding.resource_name,
"groups": [group] if group else None,
},
)
resource_cache[resource_uid] = resource_instance
@@ -580,8 +524,6 @@ def _process_finding_micro_batch(
# Track resource field changes (defer save)
updated = False
check_metadata = finding.get_metadata()
group = check_metadata.get("resourcegroup") or None
if finding.region and resource_instance.region != finding.region:
resource_instance.region = finding.region
updated = True
@@ -602,11 +544,6 @@ def _process_finding_micro_batch(
if resource_instance.partition != finding.partition:
resource_instance.partition = finding.partition
updated = True
if group and (
not resource_instance.groups or group not in resource_instance.groups
):
resource_instance.groups = (resource_instance.groups or []) + [group]
updated = True
if updated:
dirty_resources[resource_uid] = resource_instance
@@ -692,7 +629,6 @@ def _process_finding_micro_batch(
muted_reason=muted_reason,
compliance=finding.compliance,
categories=check_metadata.get("categories", []) or [],
resource_groups=check_metadata.get("resourcegroup") or None,
)
findings_to_create.append(finding_instance)
resource_denormalized_data.append((finding_instance, resource_instance))
@@ -717,18 +653,6 @@ def _process_finding_micro_batch(
cache=scan_categories_cache,
)
# Track resource groups with counts for ScanGroupSummary
aggregate_resource_group_counts(
resource_group=check_metadata.get("resourcegroup") or None,
severity=finding.severity.value,
status=status.value,
delta=delta.value if delta else None,
muted=is_muted,
resource_uid=resource_instance.uid if resource_instance else "",
cache=scan_resource_groups_cache,
group_resources_cache=group_resources_cache,
)
# Bulk operations within single transaction
with rls_transaction(tenant_id):
# Bulk create findings
@@ -786,15 +710,7 @@ def _process_finding_micro_batch(
tenant_id=tenant_id,
model=Resource,
objects=list(dirty_resources.values()),
fields=[
"metadata",
"details",
"partition",
"region",
"service",
"type",
"groups",
],
fields=["metadata", "details", "partition", "region", "service", "type"],
batch_size=1000,
)
@@ -837,8 +753,6 @@ def perform_prowler_scan(
unique_resources = set()
scan_resource_cache: set[tuple[str, str, str, str]] = set()
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
start_time = time.time()
exc = None
@@ -929,8 +843,6 @@ def perform_prowler_scan(
scan_resource_cache=scan_resource_cache,
mute_rules_cache=mute_rules_cache,
scan_categories_cache=scan_categories_cache,
scan_resource_groups_cache=scan_resource_groups_cache,
group_resources_cache=group_resources_cache,
)
# Update scan progress
@@ -1017,38 +929,6 @@ def perform_prowler_scan(
sentry_sdk.capture_exception(cat_exception)
logger.error(f"Error storing categories for scan {scan_id}: {cat_exception}")
try:
if scan_resource_groups_cache:
# Compute group-level resource counts (same value for all severity rows in a group)
group_resource_counts = {
grp: len(uids) for grp, uids in group_resources_cache.items()
}
resource_group_summaries = [
ScanGroupSummary(
tenant_id=tenant_id,
scan_id=scan_id,
resource_group=grp,
severity=severity,
total_findings=counts["total"],
failed_findings=counts["failed"],
new_failed_findings=counts["new_failed"],
resources_count=group_resource_counts.get(grp, 0),
)
for (
grp,
severity,
), counts in scan_resource_groups_cache.items()
]
with rls_transaction(tenant_id):
ScanGroupSummary.objects.bulk_create(
resource_group_summaries, batch_size=500, ignore_conflicts=True
)
except Exception as rg_exception:
sentry_sdk.capture_exception(rg_exception)
logger.error(
f"Error storing resource groups for scan {scan_id}: {rg_exception}"
)
serializer = ScanTaskSerializer(instance=scan_instance)
return serializer.data
@@ -1609,140 +1489,3 @@ def aggregate_daily_severity(tenant_id: str, scan_id: str):
"date": str(scan_date),
"severity_data": severity_data,
}
def update_provider_compliance_scores(tenant_id: str, scan_id: str):
"""
Update ProviderComplianceScore with requirement statuses from a completed scan.
Uses atomic SQL upsert with ON CONFLICT for concurrency safety. Only updates
if the new scan is more recent than existing data. Also cleans up stale
requirements that no longer exist in the new scan.
Reads from primary DB (not replica) to avoid replication lag issues since
this runs immediately after create_compliance_requirements_task.
Args:
tenant_id: Tenant that owns the scan.
scan_id: Scan UUID whose compliance data should be materialized.
Returns:
dict: Statistics about the upsert operation.
"""
with rls_transaction(tenant_id):
scan = (
Scan.all_objects.filter(
tenant_id=tenant_id,
id=scan_id,
state=StateChoices.COMPLETED,
)
.select_related("provider")
.first()
)
if not scan:
logger.warning(
f"Scan {scan_id} not found or not completed for compliance score update"
)
return {"status": "skipped", "reason": "scan not completed"}
if not scan.completed_at:
logger.warning(f"Scan {scan_id} has no completed_at timestamp")
return {"status": "skipped", "reason": "no completed_at"}
provider_id = str(scan.provider_id)
scan_completed_at = scan.completed_at
delete_stale_sql = """
DELETE FROM provider_compliance_scores pcs
WHERE pcs.tenant_id = %s
AND pcs.provider_id = %s
AND pcs.scan_completed_at < %s
AND NOT EXISTS (
SELECT 1 FROM compliance_requirements_overviews cro
WHERE cro.tenant_id = pcs.tenant_id
AND cro.scan_id = %s
AND cro.compliance_id = pcs.compliance_id
AND cro.requirement_id = pcs.requirement_id
)
RETURNING compliance_id
"""
compliance_ids_sql = """
SELECT DISTINCT compliance_id
FROM compliance_requirements_overviews
WHERE tenant_id = %s AND scan_id = %s
"""
try:
with psycopg_connection(MainRouter.default_db) as connection:
connection.autocommit = False
try:
with connection.cursor() as cursor:
cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id])
# Update requirement-level scores per provider
cursor.execute(
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL, [tenant_id, scan_id]
)
upserted_count = cursor.rowcount
cursor.execute(compliance_ids_sql, [tenant_id, scan_id])
scan_rows = cursor.fetchall()
if not isinstance(scan_rows, (list, tuple)):
scan_rows = []
scan_compliance_ids = {row[0] for row in scan_rows}
cursor.execute(
delete_stale_sql,
[tenant_id, provider_id, scan_completed_at, scan_id],
)
deleted_rows = cursor.fetchall()
if not isinstance(deleted_rows, (list, tuple)):
deleted_rows = []
deleted_ids = {row[0] for row in deleted_rows}
stale_deleted = len(deleted_ids)
impacted_compliance_ids = sorted(scan_compliance_ids | deleted_ids)
if impacted_compliance_ids:
# Advisory lock on tenant to prevent race conditions when
# multiple scans complete simultaneously for the same tenant
cursor.execute(
"SELECT pg_advisory_xact_lock(hashtext(%s))", [tenant_id]
)
# Recalculate tenant-level summary (FAIL-dominant across all providers)
cursor.execute(
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
[tenant_id, tenant_id, impacted_compliance_ids],
)
tenant_summary_count = cursor.rowcount
else:
tenant_summary_count = 0
connection.commit()
except Exception:
connection.rollback()
raise
logger.info(
f"Provider compliance scores updated for scan {scan_id}: "
f"{upserted_count} upserted, {stale_deleted} stale deleted, "
f"{tenant_summary_count} tenant summaries upserted"
)
return {
"status": "completed",
"scan_id": str(scan_id),
"provider_id": provider_id,
"upserted": upserted_count,
"stale_deleted": stale_deleted,
"tenant_summary_count": tenant_summary_count,
}
except Exception as e:
logger.error(
f"Error updating provider compliance scores for scan {scan_id}: {e}"
)
raise
+15 -152
View File
@@ -1,36 +1,18 @@
import os
from datetime import datetime, timedelta, timezone
from pathlib import Path
from shutil import rmtree
from celery import chain, group, shared_task
from celery.utils.log import get_task_logger
from django_celery_beat.models import PeriodicTask
from api.compliance import get_compliance_frameworks
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.decorators import handle_provider_deletion, set_tenant
from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices
from api.utils import initialize_prowler_provider
from api.v1.serializers import ScanTaskSerializer
from config.celery import RLSTask
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY
from prowler.lib.check.compliance_models import Compliance
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
from prowler.lib.outputs.finding import Finding as FindingOutput
from tasks.jobs.attack_paths import (
attack_paths_scan,
can_provider_run_attack_paths_scan,
)
from django_celery_beat.models import PeriodicTask
from tasks.jobs.backfill import (
backfill_compliance_summaries,
backfill_daily_severity_summaries,
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
backfill_scan_category_summaries,
backfill_scan_resource_group_summaries,
)
from tasks.jobs.connection import (
check_integration_connection,
@@ -62,65 +44,23 @@ from tasks.jobs.scan import (
aggregate_findings,
create_compliance_requirements,
perform_prowler_scan,
update_provider_compliance_scores,
)
from tasks.utils import batched, get_next_execution_datetime
from api.compliance import get_compliance_frameworks
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import rls_transaction
from api.decorators import handle_provider_deletion, set_tenant
from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices
from api.utils import initialize_prowler_provider
from api.v1.serializers import ScanTaskSerializer
from prowler.lib.check.compliance_models import Compliance
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
from prowler.lib.outputs.finding import Finding as FindingOutput
logger = get_task_logger(__name__)
def _cleanup_orphan_scheduled_scans(
tenant_id: str,
provider_id: str,
scheduler_task_id: int,
) -> int:
"""
TEMPORARY WORKAROUND: Clean up orphan AVAILABLE scans.
Detects and removes AVAILABLE scans that were never used due to an
issue during the first scheduled scan setup.
An AVAILABLE scan is considered orphan if there's also a SCHEDULED scan for
the same provider with the same scheduler_task_id. This situation indicates
that the first scan execution didn't find the AVAILABLE scan (because it
wasn't committed yet, probably) and created a new one, leaving the AVAILABLE orphaned.
Args:
tenant_id: The tenant ID.
provider_id: The provider ID.
scheduler_task_id: The PeriodicTask ID that triggers these scans.
Returns:
Number of orphan scans deleted (0 if none found).
"""
orphan_available_scans = Scan.objects.filter(
tenant_id=tenant_id,
provider_id=provider_id,
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=scheduler_task_id,
)
scheduled_scan_exists = Scan.objects.filter(
tenant_id=tenant_id,
provider_id=provider_id,
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduler_task_id=scheduler_task_id,
).exists()
if scheduled_scan_exists and orphan_available_scans.exists():
orphan_count = orphan_available_scans.count()
logger.warning(
f"[WORKAROUND] Found {orphan_count} orphan AVAILABLE scan(s) for "
f"provider {provider_id} alongside a SCHEDULED scan. Cleaning up orphans..."
)
orphan_available_scans.delete()
return orphan_count
return 0
def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str):
"""
Helper function to perform tasks after a scan is completed.
@@ -130,10 +70,9 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
scan_id (str): The ID of the scan that was performed.
provider_id (str): The primary key of the Provider instance that was scanned.
"""
chain(
create_compliance_requirements_task.si(tenant_id=tenant_id, scan_id=scan_id),
update_provider_compliance_scores_task.si(tenant_id=tenant_id, scan_id=scan_id),
).apply_async()
create_compliance_requirements_task.apply_async(
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
)
aggregate_attack_surface_task.apply_async(
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
)
@@ -158,11 +97,6 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
),
).apply_async()
if can_provider_run_attack_paths_scan(tenant_id, provider_id):
perform_attack_paths_scan_task.apply_async(
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
)
@shared_task(base=RLSTask, name="provider-connection-check")
@set_tenant
@@ -313,14 +247,6 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
return serializer.data
next_scan_datetime = get_next_execution_datetime(task_id, provider_id)
# TEMPORARY WORKAROUND: Clean up orphan scans from transaction isolation issue
_cleanup_orphan_scheduled_scans(
tenant_id=tenant_id,
provider_id=provider_id,
scheduler_task_id=periodic_task_instance.id,
)
scan_instance, _ = Scan.objects.get_or_create(
tenant_id=tenant_id,
provider_id=provider_id,
@@ -368,25 +294,6 @@ def perform_scan_summary_task(tenant_id: str, scan_id: str):
return aggregate_findings(tenant_id=tenant_id, scan_id=scan_id)
# TODO: This task must be queued at the `attack-paths` queue, don't forget to add it to the `docker-entrypoint.sh` file
@shared_task(base=RLSTask, bind=True, name="attack-paths-scan-perform", queue="scans")
def perform_attack_paths_scan_task(self, tenant_id: str, scan_id: str):
"""
Execute an Attack Paths scan for the given provider within the current tenant RLS context.
Args:
self: The task instance (automatically passed when bind=True).
tenant_id (str): The tenant identifier for RLS context.
scan_id (str): The Prowler scan identifier for obtaining the tenant and provider context.
Returns:
Any: The result from `attack_paths_scan`, including any per-scan failure details.
"""
return attack_paths_scan(
tenant_id=tenant_id, scan_id=scan_id, task_id=self.request.id
)
@shared_task(name="tenant-deletion", queue="deletion", autoretry_for=(Exception,))
def delete_tenant_task(tenant_id: str):
return delete_tenant(pk=tenant_id)
@@ -643,35 +550,6 @@ def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str):
return backfill_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="backfill-scan-resource-group-summaries", queue="backfill")
@handle_provider_deletion
def backfill_scan_resource_group_summaries_task(tenant_id: str, scan_id: str):
"""
Backfill ScanGroupSummary for a completed scan.
Aggregates unique resource groups from findings and creates a summary row.
Args:
tenant_id (str): The tenant identifier.
scan_id (str): The scan identifier.
"""
return backfill_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="backfill-provider-compliance-scores", queue="backfill")
def backfill_provider_compliance_scores_task(tenant_id: str):
"""
Backfill ProviderComplianceScore from latest completed scan per provider.
Used to populate the compliance watchlist materialized table for tenants
that had scans before the feature was deployed.
Args:
tenant_id: Target tenant UUID.
"""
return backfill_provider_compliance_scores(tenant_id=tenant_id)
@shared_task(base=RLSTask, name="scan-compliance-overviews", queue="compliance")
@handle_provider_deletion
def create_compliance_requirements_task(tenant_id: str, scan_id: str):
@@ -705,21 +583,6 @@ def aggregate_attack_surface_task(tenant_id: str, scan_id: str):
return aggregate_attack_surface(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="scan-provider-compliance-scores", queue="compliance")
def update_provider_compliance_scores_task(tenant_id: str, scan_id: str):
"""
Update provider compliance scores from a completed scan.
This task materializes compliance requirement statuses into ProviderComplianceScore
for efficient watchlist queries. Uses atomic upsert with concurrency protection.
Args:
tenant_id (str): The tenant ID for which to update scores.
scan_id (str): The ID of the scan whose data should be materialized.
"""
return update_provider_compliance_scores(tenant_id=tenant_id, scan_id=scan_id)
@shared_task(name="scan-daily-severity", queue="overview")
@handle_provider_deletion
def aggregate_daily_severity_task(tenant_id: str, scan_id: str):
@@ -1,416 +0,0 @@
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock, call, patch
import pytest
from api.models import (
AttackPathsScan,
Finding,
Provider,
Resource,
ResourceFindingMapping,
Scan,
StateChoices,
StatusChoices,
)
from prowler.lib.check.models import Severity
from tasks.jobs.attack_paths import prowler as prowler_module
from tasks.jobs.attack_paths.scan import run as attack_paths_run
@pytest.mark.django_db
class TestAttackPathsRun:
def test_run_success_flow(self, tenants_fixture, providers_fixture, scans_fixture):
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
scan = scans_fixture[0]
scan.provider = provider
scan.save()
attack_paths_scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
scan=scan,
state=StateChoices.SCHEDULED,
)
mock_session = MagicMock()
session_ctx = MagicMock()
session_ctx.__enter__.return_value = mock_session
session_ctx.__exit__.return_value = False
ingestion_result = {"organizations": "warning"}
ingestion_fn = MagicMock(return_value=ingestion_result)
with (
patch(
"tasks.jobs.attack_paths.scan.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
),
patch(
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
return_value=MagicMock(_enabled_regions=["us-east-1"]),
),
patch(
"tasks.jobs.attack_paths.scan.graph_database.get_uri",
return_value="bolt://neo4j",
),
patch(
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
return_value="db-scan-id",
) as mock_get_db_name,
patch(
"tasks.jobs.attack_paths.scan.graph_database.create_database"
) as mock_create_db,
patch(
"tasks.jobs.attack_paths.scan.graph_database.get_session",
return_value=session_ctx,
) as mock_get_session,
patch(
"tasks.jobs.attack_paths.scan.cartography_create_indexes.run"
) as mock_cartography_indexes,
patch(
"tasks.jobs.attack_paths.scan.cartography_analysis.run"
) as mock_cartography_analysis,
patch(
"tasks.jobs.attack_paths.scan.cartography_ontology.run"
) as mock_cartography_ontology,
patch(
"tasks.jobs.attack_paths.scan.prowler.create_indexes"
) as mock_prowler_indexes,
patch(
"tasks.jobs.attack_paths.scan.prowler.analysis"
) as mock_prowler_analysis,
patch(
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan",
return_value=attack_paths_scan,
) as mock_retrieve_scan,
patch(
"tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan"
) as mock_starting,
patch(
"tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress"
) as mock_update_progress,
patch(
"tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan"
) as mock_finish,
patch(
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
return_value=ingestion_fn,
) as mock_get_ingestion,
patch(
"tasks.jobs.attack_paths.scan._call_within_event_loop",
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
) as mock_event_loop,
):
result = attack_paths_run(str(tenant.id), str(scan.id), "task-123")
assert result == ingestion_result
mock_retrieve_scan.assert_called_once_with(str(tenant.id), str(scan.id))
mock_starting.assert_called_once()
config = mock_starting.call_args[0][2]
assert config.neo4j_database == "db-scan-id"
mock_create_db.assert_called_once_with("db-scan-id")
mock_get_session.assert_called_once_with("db-scan-id")
mock_cartography_indexes.assert_called_once_with(mock_session, config)
mock_prowler_indexes.assert_called_once_with(mock_session)
mock_cartography_analysis.assert_called_once_with(mock_session, config)
mock_cartography_ontology.assert_called_once_with(mock_session, config)
mock_prowler_analysis.assert_called_once_with(
mock_session,
provider,
str(scan.id),
config,
)
mock_get_ingestion.assert_called_once_with(provider.provider)
mock_event_loop.assert_called_once()
mock_update_progress.assert_any_call(attack_paths_scan, 1)
mock_update_progress.assert_any_call(attack_paths_scan, 2)
mock_update_progress.assert_any_call(attack_paths_scan, 95)
mock_finish.assert_called_once_with(
attack_paths_scan, StateChoices.COMPLETED, ingestion_result
)
mock_get_db_name.assert_called_once_with(attack_paths_scan.id)
def test_run_failure_marks_scan_failed(
self, tenants_fixture, providers_fixture, scans_fixture
):
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
scan = scans_fixture[0]
scan.provider = provider
scan.save()
attack_paths_scan = AttackPathsScan.objects.create(
tenant_id=tenant.id,
provider=provider,
scan=scan,
state=StateChoices.SCHEDULED,
)
mock_session = MagicMock()
session_ctx = MagicMock()
session_ctx.__enter__.return_value = mock_session
session_ctx.__exit__.return_value = False
ingestion_fn = MagicMock(side_effect=RuntimeError("ingestion boom"))
with (
patch(
"tasks.jobs.attack_paths.scan.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
),
patch(
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
return_value=MagicMock(_enabled_regions=["us-east-1"]),
),
patch("tasks.jobs.attack_paths.scan.graph_database.get_uri"),
patch(
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
return_value="db-scan-id",
),
patch("tasks.jobs.attack_paths.scan.graph_database.create_database"),
patch(
"tasks.jobs.attack_paths.scan.graph_database.get_session",
return_value=session_ctx,
),
patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run"),
patch("tasks.jobs.attack_paths.scan.cartography_analysis.run"),
patch("tasks.jobs.attack_paths.scan.prowler.create_indexes"),
patch("tasks.jobs.attack_paths.scan.prowler.analysis"),
patch(
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan",
return_value=attack_paths_scan,
),
patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan"),
patch(
"tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress"
),
patch(
"tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan"
) as mock_finish,
patch(
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
return_value=ingestion_fn,
),
patch(
"tasks.jobs.attack_paths.scan._call_within_event_loop",
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
),
patch(
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
return_value="Cartography failed: ingestion boom",
),
):
with pytest.raises(RuntimeError, match="ingestion boom"):
attack_paths_run(str(tenant.id), str(scan.id), "task-456")
failure_args = mock_finish.call_args[0]
assert failure_args[0] is attack_paths_scan
assert failure_args[1] == StateChoices.FAILED
assert failure_args[2] == {
"global_cartography_error": "Cartography failed: ingestion boom"
}
def test_run_returns_early_for_unsupported_provider(self, tenants_fixture):
tenant = tenants_fixture[0]
provider = Provider.objects.create(
provider=Provider.ProviderChoices.GCP,
uid="gcp-account",
alias="gcp",
tenant_id=tenant.id,
)
scan = Scan.objects.create(
name="GCP Scan",
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.AVAILABLE,
tenant_id=tenant.id,
)
with (
patch(
"tasks.jobs.attack_paths.scan.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
),
patch(
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
return_value=MagicMock(),
),
patch(
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
return_value=None,
) as mock_get_ingestion,
patch(
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan"
) as mock_retrieve,
):
mock_retrieve.return_value = None
result = attack_paths_run(str(tenant.id), str(scan.id), "task-789")
assert result == {
"global_error": "Provider gcp is not supported for Attack Paths scans"
}
mock_get_ingestion.assert_called_once_with(provider.provider)
mock_retrieve.assert_called_once_with(str(tenant.id), str(scan.id))
@pytest.mark.django_db
class TestAttackPathsProwlerHelpers:
def test_create_indexes_executes_all_statements(self):
mock_session = MagicMock()
with patch("tasks.jobs.attack_paths.prowler.run_write_query") as mock_run_write:
prowler_module.create_indexes(mock_session)
assert mock_run_write.call_count == len(prowler_module.INDEX_STATEMENTS)
mock_run_write.assert_has_calls(
[call(mock_session, stmt) for stmt in prowler_module.INDEX_STATEMENTS]
)
def test_load_findings_batches_requests(self, providers_fixture):
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
findings = [
{"id": "1", "resource_uid": "r-1"},
{"id": "2", "resource_uid": "r-2"},
]
config = SimpleNamespace(update_tag=12345)
mock_session = MagicMock()
with (
patch.object(prowler_module, "BATCH_SIZE", 1),
patch(
"tasks.jobs.attack_paths.prowler.get_root_node_label",
return_value="AWSAccount",
),
patch(
"tasks.jobs.attack_paths.prowler.get_node_uid_field",
return_value="arn",
),
):
prowler_module.load_findings(mock_session, findings, provider, config)
assert mock_session.run.call_count == 2
for call_args in mock_session.run.call_args_list:
params = call_args.args[1]
assert params["provider_uid"] == str(provider.uid)
assert params["last_updated"] == config.update_tag
assert "findings_data" in params
def test_cleanup_findings_runs_batches(self, providers_fixture):
provider = providers_fixture[0]
config = SimpleNamespace(update_tag=1024)
mock_session = MagicMock()
first_batch = MagicMock()
first_batch.single.return_value = {"deleted_findings_count": 3}
second_batch = MagicMock()
second_batch.single.return_value = {"deleted_findings_count": 0}
mock_session.run.side_effect = [first_batch, second_batch]
prowler_module.cleanup_findings(mock_session, provider, config)
assert mock_session.run.call_count == 2
params = mock_session.run.call_args.args[1]
assert params["provider_uid"] == str(provider.uid)
assert params["last_updated"] == config.update_tag
def test_get_provider_last_scan_findings_returns_latest_scan_data(
self,
tenants_fixture,
providers_fixture,
):
tenant = tenants_fixture[0]
provider = providers_fixture[0]
provider.provider = Provider.ProviderChoices.AWS
provider.save()
resource = Resource.objects.create(
tenant_id=tenant.id,
provider=provider,
uid="resource-uid",
name="Resource",
region="us-east-1",
service="ec2",
type="instance",
)
older_scan = Scan.objects.create(
name="Older",
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant_id=tenant.id,
)
old_finding = Finding.objects.create(
tenant_id=tenant.id,
uid="older-finding",
scan=older_scan,
delta=Finding.DeltaChoices.NEW,
status=StatusChoices.PASS,
status_extended="ok",
severity=Severity.low,
impact=Severity.low,
impact_extended="",
raw_result={},
check_id="check-old",
check_metadata={"checktitle": "Old"},
first_seen_at=older_scan.inserted_at,
)
ResourceFindingMapping.objects.create(
tenant_id=tenant.id,
resource=resource,
finding=old_finding,
)
latest_scan = Scan.objects.create(
name="Latest",
provider=provider,
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.COMPLETED,
tenant_id=tenant.id,
)
finding = Finding.objects.create(
tenant_id=tenant.id,
uid="finding-uid",
scan=latest_scan,
delta=Finding.DeltaChoices.NEW,
status=StatusChoices.FAIL,
status_extended="failed",
severity=Severity.high,
impact=Severity.high,
impact_extended="",
raw_result={},
check_id="check-1",
check_metadata={"checktitle": "Check title"},
first_seen_at=latest_scan.inserted_at,
)
ResourceFindingMapping.objects.create(
tenant_id=tenant.id,
resource=resource,
finding=finding,
)
latest_scan.refresh_from_db()
with patch(
"tasks.jobs.attack_paths.prowler.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
):
findings_data = prowler_module.get_provider_last_scan_findings(
provider,
str(latest_scan.id),
)
assert len(findings_data) == 1
finding_dict = findings_data[0]
assert finding_dict["id"] == str(finding.id)
assert finding_dict["resource_uid"] == resource.uid
assert finding_dict["check_title"] == "Check title"
assert finding_dict["scan_id"] == str(latest_scan.id)
@@ -1,14 +1,10 @@
from datetime import datetime, timezone
from unittest.mock import MagicMock, patch
from uuid import uuid4
import pytest
from tasks.jobs.backfill import (
backfill_compliance_summaries,
backfill_provider_compliance_scores,
backfill_resource_scan_summaries,
backfill_scan_category_summaries,
backfill_scan_resource_group_summaries,
)
from api.models import (
@@ -17,7 +13,6 @@ from api.models import (
ResourceScanSummary,
Scan,
ScanCategorySummary,
ScanGroupSummary,
StateChoices,
)
from prowler.lib.check.models import Severity
@@ -265,150 +260,3 @@ class TestBackfillScanCategorySummaries:
assert summary.total_findings == 1
assert summary.failed_findings == 1
assert summary.new_failed_findings == 1
@pytest.fixture(scope="function")
def findings_with_group_fixture(scans_fixture, resources_fixture):
scan = scans_fixture[0]
resource = resources_fixture[0]
finding = Finding.objects.create(
tenant_id=scan.tenant_id,
uid="finding_with_group",
scan=scan,
delta="new",
status=Status.FAIL,
status_extended="test status",
impact=Severity.high,
impact_extended="test impact",
severity=Severity.high,
raw_result={"status": Status.FAIL},
check_id="test_check",
check_metadata={"CheckId": "test_check"},
resource_groups="ai_ml",
first_seen_at="2024-01-02T00:00:00Z",
)
finding.add_resources([resource])
return finding
@pytest.fixture(scope="function")
def scan_resource_group_summary_fixture(scans_fixture):
scan = scans_fixture[0]
return ScanGroupSummary.objects.create(
tenant_id=scan.tenant_id,
scan=scan,
resource_group="existing-group",
severity=Severity.high,
total_findings=1,
failed_findings=0,
new_failed_findings=0,
resources_count=1,
)
@pytest.mark.django_db
class TestBackfillScanGroupSummaries:
def test_already_backfilled(self, scan_resource_group_summary_fixture):
tenant_id = scan_resource_group_summary_fixture.tenant_id
scan_id = scan_resource_group_summary_fixture.scan_id
result = backfill_scan_resource_group_summaries(str(tenant_id), str(scan_id))
assert result == {"status": "already backfilled"}
def test_not_completed_scan(self, get_not_completed_scans):
for scan in get_not_completed_scans:
result = backfill_scan_resource_group_summaries(
str(scan.tenant_id), str(scan.id)
)
assert result == {"status": "scan is not completed"}
def test_no_resource_groups_to_backfill(self, scans_fixture):
scan = scans_fixture[1] # Failed scan with no findings
result = backfill_scan_resource_group_summaries(
str(scan.tenant_id), str(scan.id)
)
assert result == {"status": "no resource groups to backfill"}
def test_successful_backfill(self, findings_with_group_fixture):
finding = findings_with_group_fixture
tenant_id = str(finding.tenant_id)
scan_id = str(finding.scan_id)
result = backfill_scan_resource_group_summaries(tenant_id, scan_id)
# 1 resource group × 1 severity = 1 row
assert result == {"status": "backfilled", "resource_groups_count": 1}
summaries = ScanGroupSummary.objects.filter(
tenant_id=tenant_id, scan_id=scan_id
)
assert summaries.count() == 1
summary = summaries.first()
assert summary.resource_group == "ai_ml"
assert summary.severity == Severity.high
assert summary.total_findings == 1
assert summary.failed_findings == 1
assert summary.new_failed_findings == 1
assert summary.resources_count == 1
@pytest.mark.django_db
class TestBackfillProviderComplianceScores:
def test_no_completed_scans(self, tenants_fixture):
tenant = tenants_fixture[2]
result = backfill_provider_compliance_scores(str(tenant.id))
assert result == {"status": "no completed scans"}
def test_no_scans_to_process(self, tenants_fixture, scans_fixture):
tenant = tenants_fixture[0]
scan = scans_fixture[0]
scan.completed_at = None
scan.save()
result = backfill_provider_compliance_scores(str(tenant.id))
assert result == {"status": "no completed scans"}
@patch("tasks.jobs.backfill.psycopg_connection")
def test_successful_backfill_executes_sql_queries(
self,
mock_psycopg_connection,
tenants_fixture,
scans_fixture,
settings,
):
"""Test successful backfill executes SQL queries and returns correct stats."""
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
tenant = tenants_fixture[0]
scan = scans_fixture[0]
# Set completed_at to make the scan eligible for backfill
scan.completed_at = datetime.now(timezone.utc)
scan.save()
connection = MagicMock()
cursor = MagicMock()
cursor_context = MagicMock()
cursor_context.__enter__.return_value = cursor
cursor_context.__exit__.return_value = False
connection.cursor.return_value = cursor_context
connection.__enter__.return_value = connection
connection.__exit__.return_value = False
connection.autocommit = True
context_manager = MagicMock()
context_manager.__enter__.return_value = connection
context_manager.__exit__.return_value = False
mock_psycopg_connection.return_value = context_manager
cursor.rowcount = 5
result = backfill_provider_compliance_scores(str(tenant.id))
assert result["status"] == "backfilled"
assert result["providers_processed"] == 1
assert result["providers_skipped"] == 0
assert result["total_upserted"] == 5
assert result["tenant_summary_count"] == 5
+1 -1
View File
@@ -28,7 +28,7 @@ class TestScheduleProviderScan:
"tenant_id": str(provider_instance.tenant_id),
"provider_id": str(provider_instance.id),
},
countdown=5,
countdown=1,
)
task_name = f"scan-perform-scheduled-{provider_instance.id}"
+30 -98
View File
@@ -1,60 +1,27 @@
from unittest.mock import call, patch
import pytest
from django.core.exceptions import ObjectDoesNotExist
from tasks.jobs.deletion import delete_provider, delete_tenant
from api.models import Provider, Tenant
from tasks.jobs.deletion import delete_provider, delete_tenant
@pytest.mark.django_db
class TestDeleteProvider:
def test_delete_provider_success(self, providers_fixture):
with patch(
"tasks.jobs.deletion.get_provider_graph_database_names"
) as mock_get_provider_graph_database_names, patch(
"tasks.jobs.deletion.graph_database.drop_database"
) as mock_drop_database:
graph_db_names = ["graph-db-1", "graph-db-2"]
mock_get_provider_graph_database_names.return_value = graph_db_names
instance = providers_fixture[0]
tenant_id = str(instance.tenant_id)
result = delete_provider(tenant_id, instance.id)
instance = providers_fixture[0]
tenant_id = str(instance.tenant_id)
result = delete_provider(tenant_id, instance.id)
assert result
with pytest.raises(ObjectDoesNotExist):
Provider.objects.get(pk=instance.id)
mock_get_provider_graph_database_names.assert_called_once_with(
tenant_id, instance.id
)
mock_drop_database.assert_has_calls(
[call(graph_db_name) for graph_db_name in graph_db_names]
)
assert result
with pytest.raises(ObjectDoesNotExist):
Provider.objects.get(pk=instance.id)
def test_delete_provider_does_not_exist(self, tenants_fixture):
with patch(
"tasks.jobs.deletion.get_provider_graph_database_names"
) as mock_get_provider_graph_database_names, patch(
"tasks.jobs.deletion.graph_database.drop_database"
) as mock_drop_database:
graph_db_names = ["graph-db-1"]
mock_get_provider_graph_database_names.return_value = graph_db_names
tenant_id = str(tenants_fixture[0].id)
non_existent_pk = "babf6796-cfcc-4fd3-9dcf-88d012247645"
tenant_id = str(tenants_fixture[0].id)
non_existent_pk = "babf6796-cfcc-4fd3-9dcf-88d012247645"
with pytest.raises(ObjectDoesNotExist):
delete_provider(tenant_id, non_existent_pk)
mock_get_provider_graph_database_names.assert_called_once_with(
tenant_id, non_existent_pk
)
mock_drop_database.assert_has_calls(
[call(graph_db_name) for graph_db_name in graph_db_names]
)
with pytest.raises(ObjectDoesNotExist):
delete_provider(tenant_id, non_existent_pk)
@pytest.mark.django_db
@@ -63,68 +30,33 @@ class TestDeleteTenant:
"""
Test successful deletion of a tenant and its related data.
"""
with patch(
"tasks.jobs.deletion.get_provider_graph_database_names"
) as mock_get_provider_graph_database_names, patch(
"tasks.jobs.deletion.graph_database.drop_database"
) as mock_drop_database:
tenant = tenants_fixture[0]
providers = list(Provider.objects.filter(tenant_id=tenant.id))
tenant = tenants_fixture[0]
providers = Provider.objects.filter(tenant_id=tenant.id)
graph_db_names_per_provider = [
[f"graph-db-{provider.id}"] for provider in providers
]
mock_get_provider_graph_database_names.side_effect = (
graph_db_names_per_provider
)
# Ensure the tenant and related providers exist before deletion
assert Tenant.objects.filter(id=tenant.id).exists()
assert providers.exists()
# Ensure the tenant and related providers exist before deletion
assert Tenant.objects.filter(id=tenant.id).exists()
assert providers
# Call the function and validate the result
deletion_summary = delete_tenant(tenant.id)
# Call the function and validate the result
deletion_summary = delete_tenant(tenant.id)
assert deletion_summary is not None
assert not Tenant.objects.filter(id=tenant.id).exists()
assert not Provider.objects.filter(tenant_id=tenant.id).exists()
expected_calls = [
call(provider.tenant_id, provider.id) for provider in providers
]
mock_get_provider_graph_database_names.assert_has_calls(
expected_calls, any_order=True
)
assert mock_get_provider_graph_database_names.call_count == len(
expected_calls
)
expected_drop_calls = [
call(graph_db_name[0]) for graph_db_name in graph_db_names_per_provider
]
mock_drop_database.assert_has_calls(expected_drop_calls, any_order=True)
assert mock_drop_database.call_count == len(expected_drop_calls)
assert deletion_summary is not None
assert not Tenant.objects.filter(id=tenant.id).exists()
assert not Provider.objects.filter(tenant_id=tenant.id).exists()
def test_delete_tenant_with_no_providers(self, tenants_fixture):
"""
Test deletion of a tenant with no related providers.
"""
with patch(
"tasks.jobs.deletion.get_provider_graph_database_names"
) as mock_get_provider_graph_database_names, patch(
"tasks.jobs.deletion.graph_database.drop_database"
) as mock_drop_database:
tenant = tenants_fixture[1] # Assume this tenant has no providers
providers = Provider.objects.filter(tenant_id=tenant.id)
tenant = tenants_fixture[1] # Assume this tenant has no providers
providers = Provider.objects.filter(tenant_id=tenant.id)
# Ensure the tenant exists but has no related providers
assert Tenant.objects.filter(id=tenant.id).exists()
assert not providers.exists()
# Ensure the tenant exists but has no related providers
assert Tenant.objects.filter(id=tenant.id).exists()
assert not providers.exists()
# Call the function and validate the result
deletion_summary = delete_tenant(tenant.id)
# Call the function and validate the result
deletion_summary = delete_tenant(tenant.id)
assert deletion_summary == {} # No providers, so empty summary
assert not Tenant.objects.filter(id=tenant.id).exists()
mock_get_provider_graph_database_names.assert_not_called()
mock_drop_database.assert_not_called()
assert deletion_summary == {} # No providers, so empty summary
assert not Tenant.objects.filter(id=tenant.id).exists()
@@ -1199,6 +1199,9 @@ class TestSecurityHubIntegrationUploads:
)
assert result is False
# Integration should be marked as disconnected
integration.save.assert_called_once()
assert integration.connected is False
@patch("tasks.jobs.integrations.ASFF")
@patch("tasks.jobs.integrations.FindingOutput")
-137
View File
@@ -24,7 +24,6 @@ from tasks.jobs.scan import (
aggregate_findings,
create_compliance_requirements,
perform_prowler_scan,
update_provider_compliance_scores,
)
from tasks.utils import CustomEncoder
@@ -1380,8 +1379,6 @@ class TestProcessFindingMicroBatch:
scan_resource_cache: set[tuple[str, str, str, str]] = set()
mute_rules_cache = {}
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
with (
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1400,8 +1397,6 @@ class TestProcessFindingMicroBatch:
scan_resource_cache,
mute_rules_cache,
scan_categories_cache,
scan_resource_groups_cache,
group_resources_cache,
)
created_finding = Finding.objects.get(uid=finding.uid)
@@ -1495,8 +1490,6 @@ class TestProcessFindingMicroBatch:
scan_resource_cache: set[tuple[str, str, str, str]] = set()
mute_rules_cache = {finding.uid: "Muted via rule"}
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
with (
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1515,8 +1508,6 @@ class TestProcessFindingMicroBatch:
scan_resource_cache,
mute_rules_cache,
scan_categories_cache,
scan_resource_groups_cache,
group_resources_cache,
)
existing_resource.refresh_from_db()
@@ -1625,8 +1616,6 @@ class TestProcessFindingMicroBatch:
scan_resource_cache: set[tuple[str, str, str, str]] = set()
mute_rules_cache = {}
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
with (
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1646,8 +1635,6 @@ class TestProcessFindingMicroBatch:
scan_resource_cache,
mute_rules_cache,
scan_categories_cache,
scan_resource_groups_cache,
group_resources_cache,
)
# Verify the long UID finding was NOT created
@@ -1725,8 +1712,6 @@ class TestProcessFindingMicroBatch:
scan_resource_cache: set[tuple[str, str, str, str]] = set()
mute_rules_cache = {}
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
group_resources_cache: dict[str, set] = {}
with (
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
@@ -1745,8 +1730,6 @@ class TestProcessFindingMicroBatch:
scan_resource_cache,
mute_rules_cache,
scan_categories_cache,
scan_resource_groups_cache,
group_resources_cache,
)
# finding1: PASS, severity=low, categories=["gen-ai", "security"]
@@ -4039,123 +4022,3 @@ class TestAggregateCategoryCounts:
assert len(cache) == 3
for cat in ["security", "compliance", "data-protection"]:
assert cache[(cat, "low")] == {"total": 1, "failed": 1, "new_failed": 1}
@pytest.mark.django_db
class TestUpdateProviderComplianceScores:
@patch("tasks.jobs.scan.psycopg_connection")
def test_update_provider_compliance_scores_basic(
self,
mock_psycopg_connection,
tenants_fixture,
scans_fixture,
settings,
):
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
tenant = tenants_fixture[0]
scan = scans_fixture[0]
tenant_id = str(tenant.id)
scan_id = str(scan.id)
scan.state = StateChoices.COMPLETED
scan.completed_at = datetime.now(timezone.utc)
scan.save()
connection = MagicMock()
cursor = MagicMock()
cursor_context = MagicMock()
cursor_context.__enter__.return_value = cursor
cursor_context.__exit__.return_value = False
connection.cursor.return_value = cursor_context
connection.__enter__.return_value = connection
connection.__exit__.return_value = False
connection.autocommit = True
context_manager = MagicMock()
context_manager.__enter__.return_value = connection
context_manager.__exit__.return_value = False
mock_psycopg_connection.return_value = context_manager
cursor.rowcount = 2
result = update_provider_compliance_scores(tenant_id, scan_id)
assert result["status"] == "completed"
assert result["upserted"] == 2
assert cursor.execute.call_count >= 3
connection.commit.assert_called_once()
def test_update_provider_compliance_scores_skips_incomplete_scan(
self, tenants_fixture, scans_fixture
):
tenant = tenants_fixture[0]
scan = scans_fixture[1]
tenant_id = str(tenant.id)
scan_id = str(scan.id)
result = update_provider_compliance_scores(tenant_id, scan_id)
assert result["status"] == "skipped"
assert result["reason"] == "scan not completed"
def test_update_provider_compliance_scores_skips_no_completed_at(
self, tenants_fixture, scans_fixture
):
tenant = tenants_fixture[0]
scan = scans_fixture[0]
tenant_id = str(tenant.id)
scan_id = str(scan.id)
scan.state = StateChoices.COMPLETED
scan.completed_at = None
scan.save()
result = update_provider_compliance_scores(tenant_id, scan_id)
assert result["status"] == "skipped"
assert result["reason"] == "no completed_at"
@patch("tasks.jobs.scan.psycopg_connection")
def test_update_provider_compliance_scores_executes_sql_queries(
self,
mock_psycopg_connection,
tenants_fixture,
providers_fixture,
scans_fixture,
settings,
):
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
tenant = tenants_fixture[0]
scan = scans_fixture[0]
tenant_id = str(tenant.id)
scan_id = str(scan.id)
scan.state = StateChoices.COMPLETED
scan.completed_at = datetime.now(timezone.utc)
scan.save()
connection = MagicMock()
cursor = MagicMock()
cursor_context = MagicMock()
cursor_context.__enter__.return_value = cursor
cursor_context.__exit__.return_value = False
connection.cursor.return_value = cursor_context
connection.__enter__.return_value = connection
connection.__exit__.return_value = False
context_manager = MagicMock()
context_manager.__enter__.return_value = connection
context_manager.__exit__.return_value = False
mock_psycopg_connection.return_value = context_manager
cursor.rowcount = 1
cursor.fetchall.side_effect = [[("aws_cis_2.0",)], []]
result = update_provider_compliance_scores(tenant_id, scan_id)
assert result["status"] == "completed"
calls = [str(c) for c in cursor.execute.call_args_list]
assert any("provider_compliance_scores" in c for c in calls)
assert any("tenant_compliance_summaries" in c for c in calls)
assert any("pg_advisory_xact_lock" in c for c in calls)
+9 -431
View File
@@ -1,37 +1,29 @@
import uuid
from contextlib import contextmanager
from unittest.mock import MagicMock, patch
import openai
import pytest
from botocore.exceptions import ClientError
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from api.models import (
Integration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
Scan,
StateChoices,
)
from tasks.jobs.lighthouse_providers import (
_create_bedrock_client,
_extract_bedrock_credentials,
)
from tasks.tasks import (
_cleanup_orphan_scheduled_scans,
_perform_scan_complete_tasks,
check_integrations_task,
check_lighthouse_provider_connection_task,
generate_outputs_task,
perform_attack_paths_scan_task,
refresh_lighthouse_provider_models_task,
s3_integration_task,
security_hub_integration_task,
)
from api.models import (
Integration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
)
@pytest.mark.django_db
class TestExtractBedrockCredentials:
@@ -734,39 +726,26 @@ class TestGenerateOutputs:
class TestScanCompleteTasks:
@patch("tasks.tasks.aggregate_attack_surface_task.apply_async")
@patch("tasks.tasks.chain")
@patch("tasks.tasks.create_compliance_requirements_task.si")
@patch("tasks.tasks.update_provider_compliance_scores_task.si")
@patch("tasks.tasks.create_compliance_requirements_task.apply_async")
@patch("tasks.tasks.perform_scan_summary_task.si")
@patch("tasks.tasks.generate_outputs_task.si")
@patch("tasks.tasks.generate_compliance_reports_task.si")
@patch("tasks.tasks.check_integrations_task.si")
@patch("tasks.tasks.perform_attack_paths_scan_task.apply_async")
@patch("tasks.tasks.can_provider_run_attack_paths_scan", return_value=False)
def test_scan_complete_tasks(
self,
mock_can_run_attack_paths,
mock_attack_paths_task,
mock_check_integrations_task,
mock_compliance_reports_task,
mock_outputs_task,
mock_scan_summary_task,
mock_update_compliance_scores_task,
mock_compliance_requirements_task,
mock_chain,
mock_attack_surface_task,
):
"""Test that scan complete tasks are properly orchestrated with optimized reports."""
_perform_scan_complete_tasks("tenant-id", "scan-id", "provider-id")
# Verify compliance requirements task is called via chain
# Verify compliance requirements task is called
mock_compliance_requirements_task.assert_called_once_with(
tenant_id="tenant-id", scan_id="scan-id"
)
# Verify update provider compliance scores task is called via chain
mock_update_compliance_scores_task.assert_called_once_with(
tenant_id="tenant-id", scan_id="scan-id"
kwargs={"tenant_id": "tenant-id", "scan_id": "scan-id"},
)
# Verify attack surface task is called
@@ -801,67 +780,6 @@ class TestScanCompleteTasks:
scan_id="scan-id",
)
# Attack Paths task should be skipped when provider cannot run it
mock_attack_paths_task.assert_not_called()
class TestAttackPathsTasks:
@staticmethod
@contextmanager
def _override_task_request(task, **attrs):
request = task.request
sentinel = object()
previous = {key: getattr(request, key, sentinel) for key in attrs}
for key, value in attrs.items():
setattr(request, key, value)
try:
yield
finally:
for key, prev in previous.items():
if prev is sentinel:
if hasattr(request, key):
delattr(request, key)
else:
setattr(request, key, prev)
def test_perform_attack_paths_scan_task_calls_runner(self):
with (
patch("tasks.tasks.attack_paths_scan") as mock_attack_paths_scan,
self._override_task_request(
perform_attack_paths_scan_task, id="celery-task-id"
),
):
mock_attack_paths_scan.return_value = {"status": "ok"}
result = perform_attack_paths_scan_task.run(
tenant_id="tenant-id", scan_id="scan-id"
)
mock_attack_paths_scan.assert_called_once_with(
tenant_id="tenant-id", scan_id="scan-id", task_id="celery-task-id"
)
assert result == {"status": "ok"}
def test_perform_attack_paths_scan_task_propagates_exception(self):
with (
patch(
"tasks.tasks.attack_paths_scan",
side_effect=RuntimeError("Exception to propagate"),
) as mock_attack_paths_scan,
self._override_task_request(
perform_attack_paths_scan_task, id="celery-task-error"
),
):
with pytest.raises(RuntimeError, match="Exception to propagate"):
perform_attack_paths_scan_task.run(
tenant_id="tenant-id", scan_id="scan-id"
)
mock_attack_paths_scan.assert_called_once_with(
tenant_id="tenant-id", scan_id="scan-id", task_id="celery-task-error"
)
@pytest.mark.django_db
class TestCheckIntegrationsTask:
@@ -1797,343 +1715,3 @@ class TestRefreshLighthouseProviderModelsTask:
assert result["deleted"] == 0
assert "error" in result
assert result["error"] is not None
@pytest.mark.django_db
class TestCleanupOrphanScheduledScans:
"""Unit tests for _cleanup_orphan_scheduled_scans helper function."""
def _create_periodic_task(self, provider_id, tenant_id):
"""Helper to create a PeriodicTask for testing."""
interval, _ = IntervalSchedule.objects.get_or_create(every=24, period="hours")
return PeriodicTask.objects.create(
name=f"scan-perform-scheduled-{provider_id}",
task="scan-perform-scheduled",
interval=interval,
kwargs=f'{{"tenant_id": "{tenant_id}", "provider_id": "{provider_id}"}}',
enabled=True,
)
def test_cleanup_deletes_orphan_when_both_available_and_scheduled_exist(
self, tenants_fixture, providers_fixture
):
"""Test that AVAILABLE scan is deleted when SCHEDULED also exists."""
tenant = tenants_fixture[0]
provider = providers_fixture[0]
periodic_task = self._create_periodic_task(provider.id, tenant.id)
# Create orphan AVAILABLE scan
orphan_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=periodic_task.id,
)
# Create SCHEDULED scan (next execution)
scheduled_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduler_task_id=periodic_task.id,
)
# Execute cleanup
deleted_count = _cleanup_orphan_scheduled_scans(
tenant_id=str(tenant.id),
provider_id=str(provider.id),
scheduler_task_id=periodic_task.id,
)
# Verify orphan was deleted
assert deleted_count == 1
assert not Scan.objects.filter(id=orphan_scan.id).exists()
assert Scan.objects.filter(id=scheduled_scan.id).exists()
def test_cleanup_does_not_delete_when_only_available_exists(
self, tenants_fixture, providers_fixture
):
"""Test that AVAILABLE scan is NOT deleted when no SCHEDULED exists."""
tenant = tenants_fixture[0]
provider = providers_fixture[0]
periodic_task = self._create_periodic_task(provider.id, tenant.id)
# Create only AVAILABLE scan (normal first scan scenario)
available_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=periodic_task.id,
)
# Execute cleanup
deleted_count = _cleanup_orphan_scheduled_scans(
tenant_id=str(tenant.id),
provider_id=str(provider.id),
scheduler_task_id=periodic_task.id,
)
# Verify nothing was deleted
assert deleted_count == 0
assert Scan.objects.filter(id=available_scan.id).exists()
def test_cleanup_does_not_delete_when_only_scheduled_exists(
self, tenants_fixture, providers_fixture
):
"""Test that nothing is deleted when only SCHEDULED exists."""
tenant = tenants_fixture[0]
provider = providers_fixture[0]
periodic_task = self._create_periodic_task(provider.id, tenant.id)
# Create only SCHEDULED scan (normal subsequent scan scenario)
scheduled_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduler_task_id=periodic_task.id,
)
# Execute cleanup
deleted_count = _cleanup_orphan_scheduled_scans(
tenant_id=str(tenant.id),
provider_id=str(provider.id),
scheduler_task_id=periodic_task.id,
)
# Verify nothing was deleted
assert deleted_count == 0
assert Scan.objects.filter(id=scheduled_scan.id).exists()
def test_cleanup_returns_zero_when_no_scans_exist(
self, tenants_fixture, providers_fixture
):
"""Test that cleanup returns 0 when no scans exist."""
tenant = tenants_fixture[0]
provider = providers_fixture[0]
periodic_task = self._create_periodic_task(provider.id, tenant.id)
# Execute cleanup with no scans
deleted_count = _cleanup_orphan_scheduled_scans(
tenant_id=str(tenant.id),
provider_id=str(provider.id),
scheduler_task_id=periodic_task.id,
)
assert deleted_count == 0
def test_cleanup_deletes_multiple_orphan_available_scans(
self, tenants_fixture, providers_fixture
):
"""Test that multiple AVAILABLE orphan scans are all deleted."""
tenant = tenants_fixture[0]
provider = providers_fixture[0]
periodic_task = self._create_periodic_task(provider.id, tenant.id)
# Create multiple orphan AVAILABLE scans
orphan_scan_1 = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=periodic_task.id,
)
orphan_scan_2 = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=periodic_task.id,
)
# Create SCHEDULED scan
scheduled_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduler_task_id=periodic_task.id,
)
# Execute cleanup
deleted_count = _cleanup_orphan_scheduled_scans(
tenant_id=str(tenant.id),
provider_id=str(provider.id),
scheduler_task_id=periodic_task.id,
)
# Verify all orphans were deleted
assert deleted_count == 2
assert not Scan.objects.filter(id=orphan_scan_1.id).exists()
assert not Scan.objects.filter(id=orphan_scan_2.id).exists()
assert Scan.objects.filter(id=scheduled_scan.id).exists()
def test_cleanup_does_not_affect_different_provider(
self, tenants_fixture, providers_fixture
):
"""Test that cleanup only affects scans for the specified provider."""
tenant = tenants_fixture[0]
provider1 = providers_fixture[0]
provider2 = providers_fixture[1]
periodic_task1 = self._create_periodic_task(provider1.id, tenant.id)
periodic_task2 = self._create_periodic_task(provider2.id, tenant.id)
# Create orphan scenario for provider1
orphan_scan_p1 = Scan.objects.create(
tenant_id=tenant.id,
provider=provider1,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=periodic_task1.id,
)
scheduled_scan_p1 = Scan.objects.create(
tenant_id=tenant.id,
provider=provider1,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduler_task_id=periodic_task1.id,
)
# Create AVAILABLE scan for provider2 (should not be affected)
available_scan_p2 = Scan.objects.create(
tenant_id=tenant.id,
provider=provider2,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=periodic_task2.id,
)
# Execute cleanup for provider1 only
deleted_count = _cleanup_orphan_scheduled_scans(
tenant_id=str(tenant.id),
provider_id=str(provider1.id),
scheduler_task_id=periodic_task1.id,
)
# Verify only provider1's orphan was deleted
assert deleted_count == 1
assert not Scan.objects.filter(id=orphan_scan_p1.id).exists()
assert Scan.objects.filter(id=scheduled_scan_p1.id).exists()
assert Scan.objects.filter(id=available_scan_p2.id).exists()
def test_cleanup_does_not_affect_manual_scans(
self, tenants_fixture, providers_fixture
):
"""Test that cleanup only affects SCHEDULED trigger scans, not MANUAL."""
tenant = tenants_fixture[0]
provider = providers_fixture[0]
periodic_task = self._create_periodic_task(provider.id, tenant.id)
# Create orphan AVAILABLE scheduled scan
orphan_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=periodic_task.id,
)
# Create SCHEDULED scan
scheduled_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduler_task_id=periodic_task.id,
)
# Create AVAILABLE manual scan (should not be affected)
manual_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Manual scan",
trigger=Scan.TriggerChoices.MANUAL,
state=StateChoices.AVAILABLE,
)
# Execute cleanup
deleted_count = _cleanup_orphan_scheduled_scans(
tenant_id=str(tenant.id),
provider_id=str(provider.id),
scheduler_task_id=periodic_task.id,
)
# Verify only scheduled orphan was deleted
assert deleted_count == 1
assert not Scan.objects.filter(id=orphan_scan.id).exists()
assert Scan.objects.filter(id=scheduled_scan.id).exists()
assert Scan.objects.filter(id=manual_scan.id).exists()
def test_cleanup_does_not_affect_different_scheduler_task(
self, tenants_fixture, providers_fixture
):
"""Test that cleanup only affects scans with the specified scheduler_task_id."""
tenant = tenants_fixture[0]
provider = providers_fixture[0]
periodic_task1 = self._create_periodic_task(provider.id, tenant.id)
# Create another periodic task
interval, _ = IntervalSchedule.objects.get_or_create(every=24, period="hours")
periodic_task2 = PeriodicTask.objects.create(
name=f"scan-perform-scheduled-other-{provider.id}",
task="scan-perform-scheduled",
interval=interval,
kwargs=f'{{"tenant_id": "{tenant.id}", "provider_id": "{provider.id}"}}',
enabled=True,
)
# Create orphan scenario for periodic_task1
orphan_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=periodic_task1.id,
)
scheduled_scan = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.SCHEDULED,
scheduler_task_id=periodic_task1.id,
)
# Create AVAILABLE scan for periodic_task2 (should not be affected)
available_scan_other_task = Scan.objects.create(
tenant_id=tenant.id,
provider=provider,
name="Daily scheduled scan",
trigger=Scan.TriggerChoices.SCHEDULED,
state=StateChoices.AVAILABLE,
scheduler_task_id=periodic_task2.id,
)
# Execute cleanup for periodic_task1 only
deleted_count = _cleanup_orphan_scheduled_scans(
tenant_id=str(tenant.id),
provider_id=str(provider.id),
scheduler_task_id=periodic_task1.id,
)
# Verify only periodic_task1's orphan was deleted
assert deleted_count == 1
assert not Scan.objects.filter(id=orphan_scan.id).exists()
assert Scan.objects.filter(id=scheduled_scan.id).exists()
assert Scan.objects.filter(id=available_scan_other_task.id).exists()
@@ -1,24 +0,0 @@
import warnings
from dashboard.common_methods import get_section_containers_cis
warnings.filterwarnings("ignore")
def get_table(data):
aux = data[
[
"REQUIREMENTS_ID",
"REQUIREMENTS_DESCRIPTION",
"REQUIREMENTS_ATTRIBUTES_SECTION",
"CHECKID",
"STATUS",
"REGION",
"ACCOUNTID",
"RESOURCEID",
]
].copy()
return get_section_containers_cis(
aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION"
)
-25
View File
@@ -1,25 +0,0 @@
import warnings
from dashboard.common_methods import get_section_containers_cis
warnings.filterwarnings("ignore")
def get_table(data):
aux = data[
[
"REQUIREMENTS_ID",
"REQUIREMENTS_DESCRIPTION",
"REQUIREMENTS_ATTRIBUTES_SECTION",
"CHECKID",
"STATUS",
"REGION",
"ACCOUNTID",
"RESOURCEID",
]
].copy()
return get_section_containers_cis(
aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION"
)

Some files were not shown because too many files have changed in this diff Show More