mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-06-11 05:46:05 +00:00
Compare commits
272 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 49636274c7 | |||
| 8a38b1cc0e | |||
| d46f649ff1 | |||
| 34787659b9 | |||
| 3d6aa6c650 | |||
| ee93ad6cbc | |||
| 7f4c02c738 | |||
| d386730770 | |||
| 5784592437 | |||
| 35f263dea6 | |||
| a1637ec46b | |||
| 6c6a6c55cf | |||
| 31b53f091b | |||
| f7a16fff99 | |||
| cb5c9ea1c5 | |||
| cb367da97d | |||
| be2a58dc82 | |||
| 29133f2d7e | |||
| babf18ffea | |||
| b6a34d2220 | |||
| 77dc79df32 | |||
| 91e3c01f51 | |||
| 6cb0edf3e1 | |||
| 7dfafb9337 | |||
| dce05295ef | |||
| 03d4c19ed5 | |||
| 963ece9a0b | |||
| a32eff6946 | |||
| 3bb326133a | |||
| 799826758e | |||
| 1208005a94 | |||
| ecdece9f1e | |||
| 9c2c555628 | |||
| ca2f3ccc1c | |||
| 9ffa0043ab | |||
| e76ecfdd4d | |||
| f11f71bc42 | |||
| 607cfd61ef | |||
| 9c76dafaa4 | |||
| 7b839d9f9e | |||
| f39a82fdf4 | |||
| d1a7eed5fa | |||
| 5be4ec511f | |||
| a0166aede7 | |||
| 1a2a2ea3cc | |||
| e61d1401b9 | |||
| a2789b7fc6 | |||
| 34217492d0 | |||
| ed50ed1e6d | |||
| 186977f81c | |||
| c33f20ad72 | |||
| d0b0c66ef0 | |||
| e849959fd5 | |||
| 7c090a6a07 | |||
| bc4484f269 | |||
| 7601142e42 | |||
| f47310bceb | |||
| 032499c29a | |||
| d7af97b30a | |||
| aa24034ca7 | |||
| ec4eb70539 | |||
| 76a8610121 | |||
| d5e2c930a9 | |||
| 2c4f866e42 | |||
| 31845df1a7 | |||
| d8c1273a57 | |||
| 3317c0a5e0 | |||
| 847645543a | |||
| 76aa65cb61 | |||
| 484a1d1fef | |||
| c8bc0576ea | |||
| 76cda6d777 | |||
| 28978f6db6 | |||
| d4bc6d7531 | |||
| 1bf49747ad | |||
| 2cde4c939d | |||
| 9844379d30 | |||
| 211b1b67f9 | |||
| 864b2099c3 | |||
| 270266c906 | |||
| c8fab497fd | |||
| b0eea61468 | |||
| 463fc32fca | |||
| 17f5633a8d | |||
| 48274f1d54 | |||
| 9719f9ee86 | |||
| d38be934a3 | |||
| 0472eb74d2 | |||
| e5b86da6e5 | |||
| 429c591819 | |||
| 87c0747174 | |||
| 62a8540169 | |||
| 9ee77c2b97 | |||
| 5f2cb614ad | |||
| 6c01151d78 | |||
| 05466cff22 | |||
| a57b6d78bf | |||
| d3eb30c066 | |||
| 7f2fa275c6 | |||
| 42ae5b6e3e | |||
| 7c1bcfc781 | |||
| 68684b107a | |||
| d04716ea95 | |||
| 8d8b7aad15 | |||
| f3ba70dd6b | |||
| 27492cbd42 | |||
| 795220e290 | |||
| 64ab8e64b0 | |||
| a0f9df07bd | |||
| 3d16c62f30 | |||
| fa2deef241 | |||
| 211639d849 | |||
| 25c90f9f63 | |||
| bbdb230bb2 | |||
| 6e2ba66a5a | |||
| 3332e5b891 | |||
| 44d791dfe9 | |||
| 73375ee289 | |||
| 503b56188b | |||
| 7c9dd8fe89 | |||
| f407a24022 | |||
| 8f5c43744f | |||
| 8d78831d29 | |||
| 858446c740 | |||
| e9ca8bfda6 | |||
| 5cd446c446 | |||
| 319f5b6c38 | |||
| 64c9dd4947 | |||
| 8b2dea52fa | |||
| da567138fa | |||
| 5b59986ae7 | |||
| df8d82345d | |||
| 3e4458c8f3 | |||
| e12e0dc1aa | |||
| beb2daa30d | |||
| 14b60b8bee | |||
| cab9b008d1 | |||
| ced0b8def4 | |||
| f31e230537 | |||
| c6cc82c527 | |||
| 5cc3cdc466 | |||
| b7f83da012 | |||
| 4169611a6a | |||
| 9ad2e1ef98 | |||
| 78ce4d8d9b | |||
| 49585ac6c7 | |||
| 0c3c6aea0e | |||
| 144d59de45 | |||
| e3027190de | |||
| 9f4b5e01cf | |||
| 8acdf8e65b | |||
| 35c727c7e4 | |||
| 18fa788268 | |||
| b6e04f507c | |||
| 85c90cac31 | |||
| 4ed27e1aaa | |||
| 53b5030f00 | |||
| 627d6da699 | |||
| 352f136a0f | |||
| ab4d7e0c19 | |||
| 47532cf498 | |||
| afb8701450 | |||
| 942177ae59 | |||
| 750182cd6d | |||
| 9bfa1e740c | |||
| e58e939f55 | |||
| d7f0b5b190 | |||
| a37aea84e7 | |||
| 8d1d041092 | |||
| 6f018183cd | |||
| 8ce56b5ed6 | |||
| ad5095595c | |||
| 3fbe157d10 | |||
| 83d04753ef | |||
| de8e2219c2 | |||
| 2850c40dd5 | |||
| e213afd4e1 | |||
| deada62d66 | |||
| b8d9860a2f | |||
| be759216c4 | |||
| ca9211b5ed | |||
| 3cf7f7845e | |||
| 81e046ecf6 | |||
| 0d363e6100 | |||
| 0719e31b58 | |||
| 19ceb7db88 | |||
| 43875b6ae7 | |||
| 641dc78c3a | |||
| 57b9a2ea10 | |||
| 19e9a9965b | |||
| 3eb2595f6d | |||
| d776356d16 | |||
| 5118d0ecb4 | |||
| df8e465366 | |||
| f4a78d64f1 | |||
| e5cd25e60c | |||
| 7d963751aa | |||
| fa4371bbf6 | |||
| ff6fbcbf48 | |||
| 9bf3702d71 | |||
| ec32be2f1d | |||
| d93c7dcc4d | |||
| 4abead2787 | |||
| d1d03ba421 | |||
| bd47fe2072 | |||
| b395f52a00 | |||
| d14bf31844 | |||
| fcea8dba12 | |||
| 83dac0c59f | |||
| 0bdd1c3f35 | |||
| c6b4b9c94f | |||
| 1c241bb53c | |||
| d15dd53708 | |||
| 15eac061fc | |||
| 597364fb09 | |||
| 13ec7c13b9 | |||
| 89b3b5a81f | |||
| c58ca136f0 | |||
| 594188f7ed | |||
| b9bfdc1a5a | |||
| c83374d4ed | |||
| c1e1fb00c6 | |||
| cbc621cb43 | |||
| 433853493b | |||
| 5aa112d438 | |||
| 1b2c73d2e3 | |||
| 90e3fabc33 | |||
| d4b90abd10 | |||
| 251fc6d4e3 | |||
| dd85da703e | |||
| b549c8dbad | |||
| 79ac7cf6d4 | |||
| d292c6e58a | |||
| 8f361e7e8d | |||
| 3eb278cb9f | |||
| 2f7eec8bca | |||
| 00063c57de | |||
| 2341b5bc7d | |||
| 4015beff20 | |||
| ab475bafc3 | |||
| b4ce01afd4 | |||
| 2b4b23c719 | |||
| 4398b00801 | |||
| e0cf8bffd4 | |||
| 6761f0ffd0 | |||
| 51bbaeb403 | |||
| 6158c16108 | |||
| 0c2c5ea265 | |||
| 3b56166c34 | |||
| b5151a8ee5 | |||
| 0495267351 | |||
| eefe045c18 | |||
| d7d1b22c45 | |||
| 439dbe679b | |||
| 0e9ba4b116 | |||
| 89295f7e7d | |||
| 7cf7758851 | |||
| 06142094cd | |||
| 93f1c02f44 | |||
| e2f30e0987 | |||
| c80710adfc | |||
| 1410fe2ff1 | |||
| 284910d402 | |||
| 04f795bd49 | |||
| 8b5e00163e | |||
| 57d7f77c81 | |||
| 16b1052ff1 | |||
| 978e2c82af | |||
| 0c3ba0b737 | |||
| 4addfcc848 | |||
| 8588cc03f4 | |||
| 7507fea24b |
@@ -15,6 +15,13 @@ AUTH_SECRET="N/c6mnaS5+SWq81+819OrzQZlmx1Vxtp/orjttJSmw8="
|
||||
# Google Tag Manager ID
|
||||
NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID=""
|
||||
|
||||
#### MCP Server ####
|
||||
PROWLER_MCP_VERSION=stable
|
||||
# For UI and MCP running on docker:
|
||||
PROWLER_MCP_SERVER_URL=http://mcp-server:8000/mcp
|
||||
# For UI running on host, MCP in docker:
|
||||
# PROWLER_MCP_SERVER_URL=http://localhost:8000/mcp
|
||||
|
||||
#### Code Review Configuration ####
|
||||
# Enable Claude Code standards validation on pre-push hook
|
||||
# Set to 'true' to validate changes against AGENTS.md standards via Claude Code
|
||||
@@ -41,6 +48,26 @@ POSTGRES_DB=prowler_db
|
||||
# POSTGRES_REPLICA_MAX_ATTEMPTS=3
|
||||
# POSTGRES_REPLICA_RETRY_BASE_DELAY=0.5
|
||||
|
||||
# Neo4j auth
|
||||
NEO4J_HOST=neo4j
|
||||
NEO4J_PORT=7687
|
||||
NEO4J_USER=neo4j
|
||||
NEO4J_PASSWORD=neo4j_password
|
||||
# Neo4j settings
|
||||
NEO4J_DBMS_MAX__DATABASES=1000
|
||||
NEO4J_SERVER_MEMORY_PAGECACHE_SIZE=1G
|
||||
NEO4J_SERVER_MEMORY_HEAP_INITIAL__SIZE=1G
|
||||
NEO4J_SERVER_MEMORY_HEAP_MAX__SIZE=1G
|
||||
NEO4J_POC_EXPORT_FILE_ENABLED=true
|
||||
NEO4J_APOC_IMPORT_FILE_ENABLED=true
|
||||
NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG=true
|
||||
NEO4J_PLUGINS=["apoc"]
|
||||
NEO4J_DBMS_SECURITY_PROCEDURES_ALLOWLIST=apoc.*
|
||||
NEO4J_DBMS_SECURITY_PROCEDURES_UNRESTRICTED=apoc.*
|
||||
NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS=0.0.0.0:7687
|
||||
# Neo4j Prowler settings
|
||||
ATTACK_PATHS_FINDINGS_BATCH_SIZE=1000
|
||||
|
||||
# Celery-Prowler task settings
|
||||
TASK_RETRY_DELAY_SECONDS=0.1
|
||||
TASK_RETRY_ATTEMPTS=5
|
||||
@@ -110,9 +137,8 @@ SENTRY_ENVIRONMENT=local
|
||||
SENTRY_RELEASE=local
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.12.2
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.16.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -29,7 +29,7 @@ runs:
|
||||
run: |
|
||||
BRANCH_NAME="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
|
||||
echo "Using branch: $BRANCH_NAME"
|
||||
sed -i "s|@master|@$BRANCH_NAME|g" pyproject.toml
|
||||
sed -i "s|\(git+https://github.com/prowler-cloud/prowler[^@]*\)@master|\1@$BRANCH_NAME|g" pyproject.toml
|
||||
|
||||
- name: Install poetry
|
||||
shell: bash
|
||||
|
||||
@@ -47,6 +47,16 @@ provider/oci:
|
||||
- any-glob-to-any-file: "prowler/providers/oraclecloud/**"
|
||||
- any-glob-to-any-file: "tests/providers/oraclecloud/**"
|
||||
|
||||
provider/alibabacloud:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: "prowler/providers/alibabacloud/**"
|
||||
- any-glob-to-any-file: "tests/providers/alibabacloud/**"
|
||||
|
||||
provider/cloudflare:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: "prowler/providers/cloudflare/**"
|
||||
- any-glob-to-any-file: "tests/providers/cloudflare/**"
|
||||
|
||||
github_actions:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: ".github/workflows/*"
|
||||
@@ -62,13 +72,21 @@ mutelist:
|
||||
- any-glob-to-any-file: "prowler/providers/azure/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/gcp/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/kubernetes/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/m365/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/mongodbatlas/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/oraclecloud/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/alibabacloud/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "prowler/providers/cloudflare/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/aws/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/azure/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/gcp/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/kubernetes/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/m365/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/mongodbatlas/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/oraclecloud/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/alibabacloud/lib/mutelist/**"
|
||||
- any-glob-to-any-file: "tests/providers/cloudflare/lib/mutelist/**"
|
||||
|
||||
integration/s3:
|
||||
- changed-files:
|
||||
|
||||
@@ -14,14 +14,26 @@ Please add a detailed description of how to review this PR.
|
||||
|
||||
### Checklist
|
||||
|
||||
- Are there new checks included in this PR? Yes / No
|
||||
- If so, do we need to update permissions for the provider? Please review this carefully.
|
||||
<details>
|
||||
|
||||
<summary><b>Community Checklist</b></summary>
|
||||
|
||||
- [ ] This feature/issue is listed in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
|
||||
- [ ] Is it assigned to me, if not, request it via the issue/feature in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](goto.prowler.com/slack)
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
- [ ] Review if the code is being covered by tests.
|
||||
- [ ] Review if code is being documented following this specification https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings
|
||||
- [ ] Review if backport is needed.
|
||||
- [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable.
|
||||
|
||||
#### SDK/CLI
|
||||
- Are there new checks included in this PR? Yes / No
|
||||
- If so, do we need to update permissions for the provider? Please review this carefully.
|
||||
|
||||
#### UI
|
||||
- [ ] All issue/task requirements work as expected on the UI
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px)
|
||||
@@ -30,6 +42,11 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable.
|
||||
|
||||
#### API
|
||||
- [ ] All issue/task requirements work as expected on the API
|
||||
- [ ] Endpoint response output (if applicable)
|
||||
- [ ] EXPLAIN ANALYZE output for new/modified queries or indexes (if applicable)
|
||||
- [ ] Performance test results (if applicable)
|
||||
- [ ] Any other relevant evidence of the implementation (if applicable)
|
||||
- [ ] Verify if API specs need to be regenerated.
|
||||
- [ ] Check if version updates are required (e.g., specs, Poetry, etc.).
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable.
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
name: 'API: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
current_api_version: ${{ steps.get_api_version.outputs.current_api_version }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Get current API version
|
||||
id: get_api_version
|
||||
run: |
|
||||
CURRENT_API_VERSION=$(grep -oP '^version = "\K[^"]+' api/pyproject.toml)
|
||||
echo "current_api_version=${CURRENT_API_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "Current API version: $CURRENT_API_VERSION"
|
||||
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next API minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}"
|
||||
|
||||
# API version follows Prowler minor + 1
|
||||
# For Prowler 5.17.0 -> API 1.18.0
|
||||
# For next master (Prowler 5.18.0) -> API 1.19.0
|
||||
NEXT_API_VERSION=1.$((MINOR_VERSION + 2)).0
|
||||
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_API_VERSION=${NEXT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0"
|
||||
echo "Current API version: $CURRENT_API_VERSION"
|
||||
echo "Next API minor version (for master): $NEXT_API_VERSION"
|
||||
|
||||
- name: Bump API versions in files for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_VERSION}\"|" api/src/backend/api/v1/views.py
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next API minor version to master
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}'
|
||||
branch: api-version-bump-to-v${{ env.NEXT_API_VERSION }}
|
||||
title: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler API version to v${{ env.NEXT_API_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Calculate first API patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}"
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
# API version follows Prowler minor + 1
|
||||
# For Prowler 5.17.0 release -> version branch v5.17 should have API 1.18.1
|
||||
FIRST_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).1
|
||||
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "FIRST_API_PATCH_VERSION=${FIRST_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0"
|
||||
echo "First API patch version (for ${VERSION_BRANCH}): $FIRST_API_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump API versions in files for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${FIRST_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${FIRST_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${FIRST_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for first API patch version to version branch
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}'
|
||||
branch: api-version-bump-to-v${{ env.FIRST_API_PATCH_VERSION }}
|
||||
title: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler API version to v${{ env.FIRST_API_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next API patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }}
|
||||
CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}"
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
# Extract current API patch to increment it
|
||||
if [[ $CURRENT_API_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
API_PATCH=${BASH_REMATCH[3]}
|
||||
|
||||
# API version follows Prowler minor + 1
|
||||
# Keep same API minor (based on Prowler minor), increment patch
|
||||
NEXT_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).$((API_PATCH + 1))
|
||||
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_API_PATCH_VERSION=${NEXT_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}"
|
||||
echo "Current API version: $CURRENT_API_VERSION"
|
||||
echo "Next API patch version: $NEXT_API_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
else
|
||||
echo "::error::Invalid API version format: $CURRENT_API_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Bump API versions in files for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next API patch version to version branch
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}'
|
||||
branch: api-version-bump-to-v${{ env.NEXT_API_PATCH_VERSION }}
|
||||
title: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler API version to v${{ env.NEXT_API_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -33,11 +33,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
@@ -46,6 +46,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -42,15 +42,15 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/api-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
message-ts: ${{ steps.slack-notification.outputs.ts }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Notify container push started
|
||||
id: slack-notification
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build and push API container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
@@ -120,18 +120,18 @@ jobs:
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
needs: [setup, container-build-push]
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Create and push manifests for push event
|
||||
if: github.event_name == 'push'
|
||||
@@ -170,7 +170,7 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Determine overall outcome
|
||||
id: outcome
|
||||
@@ -198,8 +198,8 @@ jobs:
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
if: github.event_name == 'push'
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
@@ -207,7 +207,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Trigger API deployment
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -20,6 +20,7 @@ env:
|
||||
|
||||
jobs:
|
||||
api-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
@@ -27,11 +28,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: api/Dockerfile
|
||||
|
||||
@@ -43,6 +44,7 @@ jobs:
|
||||
ignore: DL3013
|
||||
|
||||
api-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -61,21 +63,22 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: api/**
|
||||
files_ignore: |
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build container for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -90,7 +93,7 @@ jobs:
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Scan container with Trivy for ${{ matrix.arch }}
|
||||
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
|
||||
@@ -33,11 +33,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
@@ -46,6 +46,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -60,9 +61,8 @@ jobs:
|
||||
|
||||
- name: Safety
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
# 76352, 76353, 77323 come from SDK, but they cannot upgrade it yet. It does not affect API
|
||||
# TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X
|
||||
run: poetry run safety check --ignore 70612,66963,74429,76352,76353,77323,77744,77745
|
||||
run: poetry run safety check --ignore 79023,79027
|
||||
# TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0
|
||||
|
||||
- name: Vulture
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -73,11 +73,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
@@ -86,6 +86,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -100,7 +101,7 @@ jobs:
|
||||
|
||||
- name: Upload coverage reports to Codecov
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
name: 'Docs: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
current_docs_version: ${{ steps.get_docs_version.outputs.current_docs_version }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Get current documentation version
|
||||
id: get_docs_version
|
||||
run: |
|
||||
CURRENT_DOCS_VERSION=$(grep -oP 'PROWLER_UI_VERSION="\K[^"]+' docs/getting-started/installation/prowler-app.mdx)
|
||||
echo "current_docs_version=${CURRENT_DOCS_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "Current documentation version: $CURRENT_DOCS_VERSION"
|
||||
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}"
|
||||
|
||||
NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current documentation version: $CURRENT_DOCS_VERSION"
|
||||
echo "Current release version: $PROWLER_VERSION"
|
||||
echo "Next minor version: $NEXT_MINOR_VERSION"
|
||||
|
||||
- name: Bump versions in documentation for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to master
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
- All `*.mdx` files with `<VersionBadge>` components
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Calculate first patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}"
|
||||
|
||||
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "First patch version: $FIRST_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump versions in documentation for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to version branch
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}-branch
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }}
|
||||
CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}"
|
||||
|
||||
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current documentation version: $CURRENT_DOCS_VERSION"
|
||||
echo "Current release version: $PROWLER_VERSION"
|
||||
echo "Next patch version: $NEXT_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump versions in documentation for patch version
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to version branch
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -23,11 +23,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Scan for secrets with TruffleHog
|
||||
uses: trufflesecurity/trufflehog@b84c3d14d189e16da175e2c27fa8136603783ffc # v3.90.12
|
||||
uses: trufflesecurity/trufflehog@ef6e76c3c4023279497fab4721ffa071a722fd05 # v3.92.4
|
||||
with:
|
||||
extra_args: '--results=verified,unknown'
|
||||
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
message-ts: ${{ steps.slack-notification.outputs.ts }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Notify container push started
|
||||
id: slack-notification
|
||||
@@ -91,7 +91,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
@@ -100,7 +100,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build and push MCP container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
@@ -126,18 +126,18 @@ jobs:
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
needs: [setup, container-build-push]
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Create and push manifests for push event
|
||||
if: github.event_name == 'push'
|
||||
@@ -176,7 +176,7 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Determine overall outcome
|
||||
id: outcome
|
||||
@@ -204,8 +204,8 @@ jobs:
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
if: github.event_name == 'push'
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
@@ -213,7 +213,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Trigger MCP deployment
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -20,6 +20,7 @@ env:
|
||||
|
||||
jobs:
|
||||
mcp-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
@@ -27,11 +28,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: mcp_server/Dockerfile
|
||||
|
||||
@@ -42,6 +43,7 @@ jobs:
|
||||
dockerfile: mcp_server/Dockerfile
|
||||
|
||||
mcp-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -60,11 +62,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for MCP changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: mcp_server/**
|
||||
files_ignore: |
|
||||
@@ -73,7 +75,7 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build MCP container for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -88,7 +90,7 @@ jobs:
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Scan MCP container with Trivy for ${{ matrix.arch }}
|
||||
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
name: "MCP: PyPI Release"
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- "published"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
||||
PYTHON_VERSION: "3.12"
|
||||
WORKING_DIRECTORY: ./mcp_server
|
||||
|
||||
jobs:
|
||||
validate-release:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
prowler_version: ${{ steps.parse-version.outputs.version }}
|
||||
major_version: ${{ steps.parse-version.outputs.major }}
|
||||
|
||||
steps:
|
||||
- name: Parse and validate version
|
||||
id: parse-version
|
||||
run: |
|
||||
PROWLER_VERSION="${{ env.RELEASE_TAG }}"
|
||||
echo "version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Extract major version
|
||||
MAJOR_VERSION="${PROWLER_VERSION%%.*}"
|
||||
echo "major=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Validate major version (only Prowler 3, 4, 5 supported)
|
||||
case ${MAJOR_VERSION} in
|
||||
3|4|5)
|
||||
echo "✓ Releasing Prowler MCP for tag ${PROWLER_VERSION}"
|
||||
;;
|
||||
*)
|
||||
echo "::error::Unsupported Prowler major version: ${MAJOR_VERSION}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
publish-prowler-mcp:
|
||||
needs: validate-release
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
environment:
|
||||
name: pypi-prowler-mcp
|
||||
url: https://pypi.org/project/prowler-mcp/
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Build prowler-mcp package
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: uv build
|
||||
|
||||
- name: Publish prowler-mcp package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
with:
|
||||
packages-dir: ${{ env.WORKING_DIRECTORY }}/dist/
|
||||
print-hash: true
|
||||
@@ -29,27 +29,29 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
ui/**
|
||||
prowler/**
|
||||
mcp_server/**
|
||||
poetry.lock
|
||||
pyproject.toml
|
||||
|
||||
- name: Check for folder changes and changelog presence
|
||||
id: check-folders
|
||||
run: |
|
||||
missing_changelogs=""
|
||||
|
||||
# Check api folder
|
||||
if [[ "${{ steps.changed-files.outputs.any_changed }}" == "true" ]]; then
|
||||
# Check monitored folders
|
||||
for folder in $MONITORED_FOLDERS; do
|
||||
# Get files changed in this folder
|
||||
changed_in_folder=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "^${folder}/" || true)
|
||||
@@ -64,6 +66,22 @@ jobs:
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Check root-level dependency files (poetry.lock, pyproject.toml)
|
||||
# These are associated with the prowler folder changelog
|
||||
root_deps_changed=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -E "^(poetry\.lock|pyproject\.toml)$" || true)
|
||||
if [ -n "$root_deps_changed" ]; then
|
||||
echo "Detected changes in root dependency files: $root_deps_changed"
|
||||
# Check if prowler/CHANGELOG.md was already updated (might have been caught above)
|
||||
prowler_changelog_updated=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "^prowler/CHANGELOG.md$" || true)
|
||||
if [ -z "$prowler_changelog_updated" ]; then
|
||||
# Only add if prowler wasn't already flagged
|
||||
if ! echo "$missing_changelogs" | grep -q "prowler"; then
|
||||
echo "No changelog update found for root dependency changes"
|
||||
missing_changelogs="${missing_changelogs}- \`prowler\` (root dependency files changed)"$'\n'
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
{
|
||||
|
||||
@@ -25,14 +25,14 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout PR head
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: '**'
|
||||
|
||||
|
||||
@@ -13,7 +13,10 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
trigger-cloud-pull-request:
|
||||
if: github.event.pull_request.merged == true && github.repository == 'prowler-cloud/prowler'
|
||||
if: |
|
||||
github.event.pull_request.merged == true &&
|
||||
github.repository == 'prowler-cloud/prowler' &&
|
||||
!contains(github.event.pull_request.labels.*.name, 'skip-sync')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
@@ -26,7 +29,7 @@ jobs:
|
||||
echo "SHORT_SHA=${SHORT_SHA::7}" >> $GITHUB_ENV
|
||||
|
||||
- name: Trigger Cloud repository pull request
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -27,13 +27,13 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
@@ -344,7 +344,7 @@ jobs:
|
||||
|
||||
- name: Create PR for API dependency update
|
||||
if: ${{ env.PATCH_VERSION == '0' }}
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
commit-message: 'chore(api): update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}'
|
||||
@@ -374,7 +374,7 @@ jobs:
|
||||
no-changelog
|
||||
|
||||
- name: Create draft release
|
||||
uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2.4.1
|
||||
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0
|
||||
with:
|
||||
tag_name: ${{ env.PROWLER_VERSION }}
|
||||
name: Prowler ${{ env.PROWLER_VERSION }}
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next minor version
|
||||
run: |
|
||||
@@ -86,13 +86,12 @@ jobs:
|
||||
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_MINOR_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_MINOR_VERSION}\"|" prowler/config/config.py
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next minor version to master
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
@@ -100,7 +99,7 @@ jobs:
|
||||
commit-message: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
branch: version-bump-to-v${{ env.NEXT_MINOR_VERSION }}
|
||||
title: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
labels: no-changelog
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
@@ -111,7 +110,7 @@ jobs:
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
@@ -135,13 +134,12 @@ jobs:
|
||||
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${FIRST_PATCH_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${FIRST_PATCH_VERSION}\"|" prowler/config/config.py
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for first patch version to version branch
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
@@ -149,7 +147,7 @@ jobs:
|
||||
commit-message: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
branch: version-bump-to-v${{ env.FIRST_PATCH_VERSION }}
|
||||
title: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
labels: no-changelog
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
@@ -169,7 +167,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next patch version
|
||||
run: |
|
||||
@@ -193,13 +191,12 @@ jobs:
|
||||
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_PATCH_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_PATCH_VERSION}\"|" prowler/config/config.py
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next patch version to version branch
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
@@ -207,7 +204,7 @@ jobs:
|
||||
commit-message: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
branch: version-bump-to-v${{ env.NEXT_PATCH_VERSION }}
|
||||
title: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
labels: no-changelog
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
|
||||
@@ -31,11 +31,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: ./**
|
||||
files_ignore: |
|
||||
@@ -47,6 +47,7 @@ jobs:
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
@@ -55,6 +56,7 @@ jobs:
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Install Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -62,7 +64,7 @@ jobs:
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'poetry'
|
||||
@@ -79,11 +81,11 @@ jobs:
|
||||
|
||||
- name: Lint with flake8
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api
|
||||
run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api,skills
|
||||
|
||||
- name: Check format with black
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run black --exclude api ui --check .
|
||||
run: poetry run black --exclude "api|ui|skills" --check .
|
||||
|
||||
- name: Lint with pylint
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -49,15 +49,15 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/sdk-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -61,10 +61,10 @@ jobs:
|
||||
stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
message-ts: ${{ steps.slack-notification.outputs.ts }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Notify container push started
|
||||
id: slack-notification
|
||||
@@ -151,7 +151,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
@@ -169,7 +169,7 @@ jobs:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build and push SDK container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
@@ -188,18 +188,18 @@ jobs:
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
needs: [setup, container-build-push]
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
@@ -208,7 +208,7 @@ jobs:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Create and push manifests for push event
|
||||
if: github.event_name == 'push'
|
||||
@@ -252,7 +252,7 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Determine overall outcome
|
||||
id: outcome
|
||||
@@ -280,8 +280,8 @@ jobs:
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
dispatch-v3-deployment:
|
||||
if: needs.setup.outputs.prowler_version_major == '3'
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && needs.setup.outputs.prowler_version_major == '3' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
@@ -294,7 +294,7 @@ jobs:
|
||||
|
||||
- name: Dispatch v3 deployment (latest)
|
||||
if: github.event_name == 'push'
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
|
||||
@@ -303,7 +303,7 @@ jobs:
|
||||
|
||||
- name: Dispatch v3 deployment (release)
|
||||
if: github.event_name == 'release'
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
|
||||
|
||||
@@ -27,11 +27,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: Dockerfile
|
||||
|
||||
@@ -62,11 +62,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: ./**
|
||||
files_ignore: |
|
||||
@@ -78,6 +78,7 @@ jobs:
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
@@ -86,10 +87,11 @@ jobs:
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build SDK container for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -104,7 +106,7 @@ jobs:
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Scan SDK container with Trivy for ${{ matrix.arch }}
|
||||
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
|
||||
@@ -59,13 +59,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Install Poetry
|
||||
run: pipx install poetry==2.1.1
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'poetry'
|
||||
@@ -91,13 +91,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Install Poetry
|
||||
run: pipx install poetry==2.1.1
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'poetry'
|
||||
|
||||
@@ -25,12 +25,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: 'master'
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
run: pip install boto3
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
|
||||
with:
|
||||
aws-region: ${{ env.AWS_REGION }}
|
||||
role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }}
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
|
||||
- name: Create pull request
|
||||
id: create-pr
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
author: 'prowler-bot <179230569+prowler-bot@users.noreply.github.com>'
|
||||
|
||||
@@ -24,13 +24,15 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: ./**
|
||||
files:
|
||||
./**
|
||||
.github/workflows/sdk-security.yml
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
@@ -40,6 +42,7 @@ jobs:
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
@@ -48,6 +51,7 @@ jobs:
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Install Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -55,7 +59,7 @@ jobs:
|
||||
|
||||
- name: Set up Python 3.12
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
cache: 'poetry'
|
||||
@@ -70,7 +74,7 @@ jobs:
|
||||
|
||||
- name: Security scan with Safety
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: poetry run safety check --ignore 70612 -r pyproject.toml
|
||||
run: poetry run safety check -r pyproject.toml
|
||||
|
||||
- name: Dead code detection with Vulture
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -31,11 +31,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: ./**
|
||||
files_ignore: |
|
||||
@@ -47,6 +47,7 @@ jobs:
|
||||
ui/**
|
||||
dashboard/**
|
||||
mcp_server/**
|
||||
skills/**
|
||||
README.md
|
||||
mkdocs.yml
|
||||
.backportrc.json
|
||||
@@ -55,6 +56,7 @@ jobs:
|
||||
examples/**
|
||||
.gitignore
|
||||
contrib/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Install Poetry
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -62,7 +64,7 @@ jobs:
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'poetry'
|
||||
@@ -75,7 +77,7 @@ jobs:
|
||||
- name: Check if AWS files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-aws
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/aws/**
|
||||
@@ -189,7 +191,7 @@ jobs:
|
||||
|
||||
- name: Upload AWS coverage to Codecov
|
||||
if: steps.changed-aws.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -200,7 +202,7 @@ jobs:
|
||||
- name: Check if Azure files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-azure
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/azure/**
|
||||
@@ -213,7 +215,7 @@ jobs:
|
||||
|
||||
- name: Upload Azure coverage to Codecov
|
||||
if: steps.changed-azure.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -224,7 +226,7 @@ jobs:
|
||||
- name: Check if GCP files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-gcp
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/gcp/**
|
||||
@@ -237,7 +239,7 @@ jobs:
|
||||
|
||||
- name: Upload GCP coverage to Codecov
|
||||
if: steps.changed-gcp.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -248,7 +250,7 @@ jobs:
|
||||
- name: Check if Kubernetes files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-kubernetes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/kubernetes/**
|
||||
@@ -261,7 +263,7 @@ jobs:
|
||||
|
||||
- name: Upload Kubernetes coverage to Codecov
|
||||
if: steps.changed-kubernetes.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -272,7 +274,7 @@ jobs:
|
||||
- name: Check if GitHub files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-github
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/github/**
|
||||
@@ -285,7 +287,7 @@ jobs:
|
||||
|
||||
- name: Upload GitHub coverage to Codecov
|
||||
if: steps.changed-github.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -296,7 +298,7 @@ jobs:
|
||||
- name: Check if NHN files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-nhn
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/nhn/**
|
||||
@@ -309,7 +311,7 @@ jobs:
|
||||
|
||||
- name: Upload NHN coverage to Codecov
|
||||
if: steps.changed-nhn.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -320,7 +322,7 @@ jobs:
|
||||
- name: Check if M365 files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-m365
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/m365/**
|
||||
@@ -333,7 +335,7 @@ jobs:
|
||||
|
||||
- name: Upload M365 coverage to Codecov
|
||||
if: steps.changed-m365.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -344,7 +346,7 @@ jobs:
|
||||
- name: Check if IaC files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-iac
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/iac/**
|
||||
@@ -357,7 +359,7 @@ jobs:
|
||||
|
||||
- name: Upload IaC coverage to Codecov
|
||||
if: steps.changed-iac.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -368,7 +370,7 @@ jobs:
|
||||
- name: Check if MongoDB Atlas files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-mongodbatlas
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/mongodbatlas/**
|
||||
@@ -381,7 +383,7 @@ jobs:
|
||||
|
||||
- name: Upload MongoDB Atlas coverage to Codecov
|
||||
if: steps.changed-mongodbatlas.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -392,7 +394,7 @@ jobs:
|
||||
- name: Check if OCI files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-oraclecloud
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/oraclecloud/**
|
||||
@@ -405,7 +407,7 @@ jobs:
|
||||
|
||||
- name: Upload OCI coverage to Codecov
|
||||
if: steps.changed-oraclecloud.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -416,7 +418,7 @@ jobs:
|
||||
- name: Check if Lib files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-lib
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/lib/**
|
||||
@@ -429,7 +431,7 @@ jobs:
|
||||
|
||||
- name: Upload Lib coverage to Codecov
|
||||
if: steps.changed-lib.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -440,7 +442,7 @@ jobs:
|
||||
- name: Check if Config files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-config
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
./prowler/config/**
|
||||
@@ -453,7 +455,7 @@ jobs:
|
||||
|
||||
- name: Upload Config coverage to Codecov
|
||||
if: steps.changed-config.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
name: 'UI: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
steps:
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
|
||||
echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current version: $PROWLER_VERSION"
|
||||
echo "Next minor version: $NEXT_MINOR_VERSION"
|
||||
|
||||
- name: Bump UI version in .env for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next minor version to master
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
branch: ui-version-bump-to-v${{ env.NEXT_MINOR_VERSION }}
|
||||
title: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler UI version to v${{ env.NEXT_MINOR_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Calculate first patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "First patch version: $FIRST_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump UI version in .env for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for first patch version to version branch
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
branch: ui-version-bump-to-v${{ env.FIRST_PATCH_VERSION }}
|
||||
title: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler UI version to v${{ env.FIRST_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }}
|
||||
|
||||
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current version: $PROWLER_VERSION"
|
||||
echo "Next patch version: $NEXT_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump UI version in .env for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next patch version to version branch
|
||||
uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
branch: ui-version-bump-to-v${{ env.NEXT_PATCH_VERSION }}
|
||||
title: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler UI version to v${{ env.NEXT_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -45,15 +45,15 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/ui-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
message-ts: ${{ steps.slack-notification.outputs.ts }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Notify container push started
|
||||
id: slack-notification
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
@@ -104,7 +104,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build and push UI container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
@@ -125,18 +125,18 @@ jobs:
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
needs: [setup, container-build-push]
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
if: always() && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Create and push manifests for push event
|
||||
if: github.event_name == 'push'
|
||||
@@ -175,7 +175,7 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Determine overall outcome
|
||||
id: outcome
|
||||
@@ -203,8 +203,8 @@ jobs:
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
if: github.event_name == 'push'
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
@@ -212,7 +212,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Trigger UI deployment
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -20,6 +20,7 @@ env:
|
||||
|
||||
jobs:
|
||||
ui-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
@@ -27,11 +28,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: ui/Dockerfile
|
||||
|
||||
@@ -43,6 +44,7 @@ jobs:
|
||||
ignore: DL3018
|
||||
|
||||
ui-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -61,20 +63,21 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for UI changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: ui/**
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Build UI container for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -92,7 +95,7 @@ jobs:
|
||||
NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX
|
||||
|
||||
- name: Scan UI container with Trivy for ${{ matrix.arch }}
|
||||
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
|
||||
@@ -54,7 +54,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
- name: Create k8s Kind Cluster
|
||||
uses: helm/kind-action@v1
|
||||
with:
|
||||
@@ -114,9 +114,9 @@ jobs:
|
||||
echo "All database fixtures loaded successfully!"
|
||||
'
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
|
||||
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
|
||||
with:
|
||||
node-version: '20.x'
|
||||
node-version: '24.13.0'
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
@@ -125,21 +125,25 @@ jobs:
|
||||
- name: Get pnpm store directory
|
||||
shell: bash
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
|
||||
- name: Setup pnpm cache
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
- name: Setup pnpm and Next.js cache
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
with:
|
||||
path: ${{ env.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
path: |
|
||||
${{ env.STORE_PATH }}
|
||||
./ui/node_modules
|
||||
./ui/.next/cache
|
||||
key: ${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}-${{ hashFiles('ui/**/*.ts', 'ui/**/*.tsx', 'ui/**/*.js', 'ui/**/*.jsx') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pnpm-store-
|
||||
${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}-
|
||||
${{ runner.os }}-pnpm-nextjs-
|
||||
- name: Install UI dependencies
|
||||
working-directory: ./ui
|
||||
run: pnpm install --frozen-lockfile
|
||||
run: pnpm install --frozen-lockfile --prefer-offline
|
||||
- name: Build UI application
|
||||
working-directory: ./ui
|
||||
run: pnpm run build
|
||||
- name: Cache Playwright browsers
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
id: playwright-cache
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
@@ -154,7 +158,7 @@ jobs:
|
||||
working-directory: ./ui
|
||||
run: pnpm run test:e2e
|
||||
- name: Upload test reports
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
||||
if: failure()
|
||||
with:
|
||||
name: playwright-report
|
||||
|
||||
@@ -16,7 +16,7 @@ concurrency:
|
||||
|
||||
env:
|
||||
UI_WORKING_DIR: ./ui
|
||||
NODE_VERSION: '20.x'
|
||||
NODE_VERSION: '24.13.0'
|
||||
|
||||
jobs:
|
||||
ui-tests:
|
||||
@@ -30,11 +30,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for UI changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1
|
||||
with:
|
||||
files: |
|
||||
ui/**
|
||||
@@ -42,10 +42,11 @@ jobs:
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
|
||||
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
|
||||
@@ -61,18 +62,22 @@ jobs:
|
||||
shell: bash
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
|
||||
|
||||
- name: Setup pnpm cache
|
||||
- name: Setup pnpm and Next.js cache
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
with:
|
||||
path: ${{ env.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
path: |
|
||||
${{ env.STORE_PATH }}
|
||||
${{ env.UI_WORKING_DIR }}/node_modules
|
||||
${{ env.UI_WORKING_DIR }}/.next/cache
|
||||
key: ${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}-${{ hashFiles('ui/**/*.ts', 'ui/**/*.tsx', 'ui/**/*.js', 'ui/**/*.jsx') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pnpm-store-
|
||||
${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}-
|
||||
${{ runner.os }}-pnpm-nextjs-
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: pnpm install --frozen-lockfile
|
||||
run: pnpm install --frozen-lockfile --prefer-offline
|
||||
|
||||
- name: Run healthcheck
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
+12
-1
@@ -82,6 +82,9 @@ continue.json
|
||||
.continuerc
|
||||
.continuerc.json
|
||||
|
||||
# AI Coding Assistants - OpenCode
|
||||
opencode.json
|
||||
|
||||
# AI Coding Assistants - GitHub Copilot
|
||||
.copilot/
|
||||
.github/copilot/
|
||||
@@ -147,8 +150,16 @@ node_modules
|
||||
# Persistent data
|
||||
_data/
|
||||
|
||||
# Claude
|
||||
# AI Instructions (generated by skills/setup.sh from AGENTS.md)
|
||||
CLAUDE.md
|
||||
GEMINI.md
|
||||
.github/copilot-instructions.md
|
||||
|
||||
# Compliance report
|
||||
*.pdf
|
||||
|
||||
# AI Skills symlinks (generated by skills/setup.sh)
|
||||
.claude/skills
|
||||
.codex/skills
|
||||
.github/skills
|
||||
.gemini/skills
|
||||
|
||||
@@ -34,6 +34,7 @@ repos:
|
||||
rev: v2.3.1
|
||||
hooks:
|
||||
- id: autoflake
|
||||
exclude: ^skills/
|
||||
args:
|
||||
[
|
||||
"--in-place",
|
||||
@@ -41,22 +42,24 @@ repos:
|
||||
"--remove-unused-variable",
|
||||
]
|
||||
|
||||
- repo: https://github.com/timothycrosley/isort
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.13.2
|
||||
hooks:
|
||||
- id: isort
|
||||
exclude: ^skills/
|
||||
args: ["--profile", "black"]
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 24.4.2
|
||||
hooks:
|
||||
- id: black
|
||||
exclude: ^skills/
|
||||
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
rev: 7.0.0
|
||||
hooks:
|
||||
- id: flake8
|
||||
exclude: contrib
|
||||
exclude: (contrib|^skills/)
|
||||
args: ["--ignore=E266,W503,E203,E501,W605"]
|
||||
|
||||
- repo: https://github.com/python-poetry/poetry
|
||||
@@ -109,7 +112,7 @@ repos:
|
||||
- id: bandit
|
||||
name: bandit
|
||||
description: "Bandit is a tool for finding common security issues in Python code"
|
||||
entry: bash -c 'bandit -q -lll -x '*_test.py,./contrib/,./.venv/' -r .'
|
||||
entry: bash -c 'bandit -q -lll -x '*_test.py,./contrib/,./.venv/,./skills/' -r .'
|
||||
language: system
|
||||
files: '.*\.py'
|
||||
|
||||
@@ -117,13 +120,14 @@ repos:
|
||||
name: safety
|
||||
description: "Safety is a tool that checks your installed dependencies for known security vulnerabilities"
|
||||
# TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X
|
||||
entry: bash -c 'safety check --ignore 70612,66963,74429,76352,76353,77744,77745'
|
||||
# TODO: 79023 & 79027 knack ReDoS until `azure-cli-core` (via `cartography`) allows `knack` >=0.13.0
|
||||
entry: bash -c 'safety check --ignore 70612,66963,74429,76352,76353,77744,77745,79023,79027'
|
||||
language: system
|
||||
|
||||
- id: vulture
|
||||
name: vulture
|
||||
description: "Vulture finds unused code in Python programs."
|
||||
entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/" --min-confidence 100 .'
|
||||
entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/,skills/" --min-confidence 100 .'
|
||||
language: system
|
||||
files: '.*\.py'
|
||||
|
||||
|
||||
@@ -2,109 +2,149 @@
|
||||
|
||||
## How to Use This Guide
|
||||
|
||||
- Start here for cross-project norms, Prowler is a monorepo with several components. Every component should have an `AGENTS.md` file that contains the guidelines for the agents in that component. The file is located beside the code you are touching (e.g. `api/AGENTS.md`, `ui/AGENTS.md`, `prowler/AGENTS.md`).
|
||||
- Follow the stricter rule when guidance conflicts; component docs override this file for their scope.
|
||||
- Keep instructions synchronized. When you add new workflows or scripts, update both, the relevant component `AGENTS.md` and this file if they apply broadly.
|
||||
- Start here for cross-project norms. Prowler is a monorepo with several components.
|
||||
- Each component has an `AGENTS.md` file with specific guidelines (e.g., `api/AGENTS.md`, `ui/AGENTS.md`).
|
||||
- Component docs override this file when guidance conflicts.
|
||||
|
||||
## Available Skills
|
||||
|
||||
Use these skills for detailed patterns on-demand:
|
||||
|
||||
### Generic Skills (Any Project)
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) |
|
||||
| `react-19` | No useMemo/useCallback, React Compiler | [SKILL.md](skills/react-19/SKILL.md) |
|
||||
| `nextjs-15` | App Router, Server Actions, streaming | [SKILL.md](skills/nextjs-15/SKILL.md) |
|
||||
| `tailwind-4` | cn() utility, no var() in className | [SKILL.md](skills/tailwind-4/SKILL.md) |
|
||||
| `playwright` | Page Object Model, MCP workflow, selectors | [SKILL.md](skills/playwright/SKILL.md) |
|
||||
| `pytest` | Fixtures, mocking, markers, parametrize | [SKILL.md](skills/pytest/SKILL.md) |
|
||||
| `django-drf` | ViewSets, Serializers, Filters | [SKILL.md](skills/django-drf/SKILL.md) |
|
||||
| `jsonapi` | Strict JSON:API v1.1 spec compliance | [SKILL.md](skills/jsonapi/SKILL.md) |
|
||||
| `zod-4` | New API (z.email(), z.uuid()) | [SKILL.md](skills/zod-4/SKILL.md) |
|
||||
| `zustand-5` | Persist, selectors, slices | [SKILL.md](skills/zustand-5/SKILL.md) |
|
||||
| `ai-sdk-5` | UIMessage, streaming, LangChain | [SKILL.md](skills/ai-sdk-5/SKILL.md) |
|
||||
|
||||
### Prowler-Specific Skills
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `prowler` | Project overview, component navigation | [SKILL.md](skills/prowler/SKILL.md) |
|
||||
| `prowler-api` | Django + RLS + JSON:API patterns | [SKILL.md](skills/prowler-api/SKILL.md) |
|
||||
| `prowler-ui` | Next.js + shadcn conventions | [SKILL.md](skills/prowler-ui/SKILL.md) |
|
||||
| `prowler-sdk-check` | Create new security checks | [SKILL.md](skills/prowler-sdk-check/SKILL.md) |
|
||||
| `prowler-mcp` | MCP server tools and models | [SKILL.md](skills/prowler-mcp/SKILL.md) |
|
||||
| `prowler-test-sdk` | SDK testing (pytest + moto) | [SKILL.md](skills/prowler-test-sdk/SKILL.md) |
|
||||
| `prowler-test-api` | API testing (pytest-django + RLS) | [SKILL.md](skills/prowler-test-api/SKILL.md) |
|
||||
| `prowler-test-ui` | E2E testing (Playwright) | [SKILL.md](skills/prowler-test-ui/SKILL.md) |
|
||||
| `prowler-compliance` | Compliance framework structure | [SKILL.md](skills/prowler-compliance/SKILL.md) |
|
||||
| `prowler-compliance-review` | Review compliance framework PRs | [SKILL.md](skills/prowler-compliance-review/SKILL.md) |
|
||||
| `prowler-provider` | Add new cloud providers | [SKILL.md](skills/prowler-provider/SKILL.md) |
|
||||
| `prowler-changelog` | Changelog entries (keepachangelog.com) | [SKILL.md](skills/prowler-changelog/SKILL.md) |
|
||||
| `prowler-ci` | CI checks and PR gates (GitHub Actions) | [SKILL.md](skills/prowler-ci/SKILL.md) |
|
||||
| `prowler-commit` | Professional commits (conventional-commits) | [SKILL.md](skills/prowler-commit/SKILL.md) |
|
||||
| `prowler-pr` | Pull request conventions | [SKILL.md](skills/prowler-pr/SKILL.md) |
|
||||
| `prowler-docs` | Documentation style guide | [SKILL.md](skills/prowler-docs/SKILL.md) |
|
||||
| `skill-creator` | Create new AI agent skills | [SKILL.md](skills/skill-creator/SKILL.md) |
|
||||
|
||||
### Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
| Action | Skill |
|
||||
|--------|-------|
|
||||
| Add changelog entry for a PR or feature | `prowler-changelog` |
|
||||
| Adding DRF pagination or permissions | `django-drf` |
|
||||
| Adding new providers | `prowler-provider` |
|
||||
| Adding services to existing providers | `prowler-provider` |
|
||||
| After creating/modifying a skill | `skill-sync` |
|
||||
| App Router / Server Actions | `nextjs-15` |
|
||||
| Building AI chat features | `ai-sdk-5` |
|
||||
| Committing changes | `prowler-commit` |
|
||||
| Create PR that requires changelog entry | `prowler-changelog` |
|
||||
| Create a PR with gh pr create | `prowler-pr` |
|
||||
| Creating API endpoints | `jsonapi` |
|
||||
| Creating ViewSets, serializers, or filters in api/ | `django-drf` |
|
||||
| Creating Zod schemas | `zod-4` |
|
||||
| Creating a git commit | `prowler-commit` |
|
||||
| Creating new checks | `prowler-sdk-check` |
|
||||
| Creating new skills | `skill-creator` |
|
||||
| Creating/modifying Prowler UI components | `prowler-ui` |
|
||||
| Creating/modifying models, views, serializers | `prowler-api` |
|
||||
| Creating/updating compliance frameworks | `prowler-compliance` |
|
||||
| Debug why a GitHub Actions job is failing | `prowler-ci` |
|
||||
| Fill .github/pull_request_template.md (Context/Description/Steps to review/Checklist) | `prowler-pr` |
|
||||
| General Prowler development questions | `prowler` |
|
||||
| Implementing JSON:API endpoints | `django-drf` |
|
||||
| Inspect PR CI checks and gates (.github/workflows/*) | `prowler-ci` |
|
||||
| Inspect PR CI workflows (.github/workflows/*): conventional-commit, pr-check-changelog, pr-conflict-checker, labeler | `prowler-pr` |
|
||||
| Mapping checks to compliance controls | `prowler-compliance` |
|
||||
| Mocking AWS with moto in tests | `prowler-test-sdk` |
|
||||
| Modifying API responses | `jsonapi` |
|
||||
| Regenerate AGENTS.md Auto-invoke tables (sync.sh) | `skill-sync` |
|
||||
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Testing RLS tenant isolation | `prowler-test-api` |
|
||||
| Troubleshoot why a skill is missing from AGENTS.md auto-invoke | `skill-sync` |
|
||||
| Understand CODEOWNERS/labeler-based automation | `prowler-ci` |
|
||||
| Understand PR title conventional-commit validation | `prowler-ci` |
|
||||
| Understand changelog gate and no-changelog label behavior | `prowler-ci` |
|
||||
| Understand review ownership with CODEOWNERS | `prowler-pr` |
|
||||
| Update CHANGELOG.md in any component | `prowler-changelog` |
|
||||
| Updating existing checks and metadata | `prowler-sdk-check` |
|
||||
| Using Zustand stores | `zustand-5` |
|
||||
| Working on MCP server tools | `prowler-mcp` |
|
||||
| Working on Prowler UI structure (actions/adapters/types/hooks) | `prowler-ui` |
|
||||
| Working with Prowler UI test helpers/pages | `prowler-test-ui` |
|
||||
| Working with Tailwind classes | `tailwind-4` |
|
||||
| Writing Playwright E2E tests | `playwright` |
|
||||
| Writing Prowler API tests | `prowler-test-api` |
|
||||
| Writing Prowler SDK tests | `prowler-test-sdk` |
|
||||
| Writing Prowler UI E2E tests | `prowler-test-ui` |
|
||||
| Writing Python tests with pytest | `pytest` |
|
||||
| Writing React components | `react-19` |
|
||||
| Writing TypeScript types/interfaces | `typescript` |
|
||||
| Writing documentation | `prowler-docs` |
|
||||
|
||||
---
|
||||
|
||||
## Project Overview
|
||||
|
||||
Prowler is an open-source cloud security assessment tool that supports multiple cloud providers (AWS, Azure, GCP, Kubernetes, GitHub, M365, etc.). The project consists in a monorepo with the following main components:
|
||||
Prowler is an open-source cloud security assessment tool supporting AWS, Azure, GCP, Kubernetes, GitHub, M365, and more.
|
||||
|
||||
- **Prowler SDK**: Python SDK, includes the Prowler CLI, providers, services, checks, compliances, config, etc. (`prowler/`)
|
||||
- **Prowler API**: Django-based REST API backend (`api/`)
|
||||
- **Prowler UI**: Next.js frontend application (`ui/`)
|
||||
- **Prowler MCP Server**: Model Context Protocol server that gives access to the entire Prowler ecosystem for LLMs (`mcp_server/`)
|
||||
- **Prowler Dashboard**: Prowler CLI feature that allows to visualize the results of the scans in a simple dashboard (`dashboard/`)
|
||||
| Component | Location | Tech Stack |
|
||||
|-----------|----------|------------|
|
||||
| SDK | `prowler/` | Python 3.9+, Poetry |
|
||||
| API | `api/` | Django 5.1, DRF, Celery |
|
||||
| UI | `ui/` | Next.js 15, React 19, Tailwind 4 |
|
||||
| MCP Server | `mcp_server/` | FastMCP, Python 3.12+ |
|
||||
| Dashboard | `dashboard/` | Dash, Plotly |
|
||||
|
||||
### Project Structure (Key Folders & Files)
|
||||
|
||||
- `prowler/`: Main source code for Prowler SDK (CLI, providers, services, checks, compliances, config, etc.)
|
||||
- `api/`: Django-based REST API backend components
|
||||
- `ui/`: Next.js frontend application
|
||||
- `mcp_server/`: Model Context Protocol server that gives access to the entire Prowler ecosystem for LLMs
|
||||
- `dashboard/`: Prowler CLI feature that allows to visualize the results of the scans in a simple dashboard
|
||||
- `docs/`: Documentation
|
||||
- `examples/`: Example output formats for providers and scripts
|
||||
- `permissions/`: Permission-related files and policies
|
||||
- `contrib/`: Community-contributed scripts or modules
|
||||
- `tests/`: Prowler SDK test suite
|
||||
- `docker-compose.yml`: Docker compose file to run the Prowler App (API + UI) production environment
|
||||
- `docker-compose-dev.yml`: Docker compose file to run the Prowler App (API + UI) development environment
|
||||
- `pyproject.toml`: Poetry Prowler SDK project file
|
||||
- `.pre-commit-config.yaml`: Pre-commit hooks configuration
|
||||
- `Makefile`: Makefile to run the project
|
||||
- `LICENSE`: License file
|
||||
- `README.md`: README file
|
||||
- `CONTRIBUTING.md`: Contributing guide
|
||||
---
|
||||
|
||||
## Python Development
|
||||
|
||||
Most of the code is written in Python, so the main files in the root are focused on Python code.
|
||||
|
||||
### Poetry Dev Environment
|
||||
|
||||
For developing in Python we recommend using `poetry` to manage the dependencies. The minimal version is `2.1.1`. So it is recommended to run all commands using `poetry run ...`.
|
||||
|
||||
To install the core dependencies to develop it is needed to run `poetry install --with dev`.
|
||||
|
||||
### Pre-commit hooks
|
||||
|
||||
The project has pre-commit hooks to lint and format the code. They are installed by running `poetry run pre-commit install`.
|
||||
|
||||
When commiting a change, the hooks will be run automatically. Some of them are:
|
||||
|
||||
- Code formatting (black, isort)
|
||||
- Linting (flake8, pylint)
|
||||
- Security checks (bandit, safety, trufflehog)
|
||||
- YAML/JSON validation
|
||||
- Poetry lock file validation
|
||||
|
||||
|
||||
### Linting and Formatting
|
||||
|
||||
We use the following tools to lint and format the code:
|
||||
|
||||
- `flake8`: for linting the code
|
||||
- `black`: for formatting the code
|
||||
- `pylint`: for linting the code
|
||||
|
||||
You can run all using the `make` command:
|
||||
```bash
|
||||
# Setup
|
||||
poetry install --with dev
|
||||
prek install # install git hooks (requires prek: pipx install prek)
|
||||
|
||||
# Code quality
|
||||
poetry run make lint
|
||||
poetry run make format
|
||||
prek run --all-files
|
||||
```
|
||||
|
||||
Or they will be run automatically when you commit your changes using pre-commit hooks.
|
||||
---
|
||||
|
||||
## Commit & Pull Request Guidelines
|
||||
|
||||
For the commit messages and pull requests name follow the conventional-commit style.
|
||||
Follow conventional-commit style: `<type>[scope]: <description>`
|
||||
|
||||
Befire creating a pull request, complete the checklist in `.github/pull_request_template.md`. Summaries should explain deployment impact, highlight review steps, and note changelog or permission updates. Run all relevant tests and linters before requesting review and link screenshots for UI or dashboard changes.
|
||||
**Types:** `feat`, `fix`, `docs`, `chore`, `perf`, `refactor`, `style`, `test`
|
||||
|
||||
### Conventional Commit Style
|
||||
|
||||
The Conventional Commits specification is a lightweight convention on top of commit messages. It provides an easy set of rules for creating an explicit commit history; which makes it easier to write automated tools on top of.
|
||||
|
||||
The commit message should be structured as follows:
|
||||
|
||||
```
|
||||
<type>[optional scope]: <description>
|
||||
<BLANK LINE>
|
||||
[optional body]
|
||||
<BLANK LINE>
|
||||
[optional footer(s)]
|
||||
```
|
||||
|
||||
Any line of the commit message cannot be longer 100 characters! This allows the message to be easier to read on GitHub as well as in various git tools
|
||||
|
||||
#### Commit Types
|
||||
|
||||
- **feat**: code change introuce new functionality to the application
|
||||
- **fix**: code change that solve a bug in the codebase
|
||||
- **docs**: documentation only changes
|
||||
- **chore**: changes related to the build process or auxiliary tools and libraries, that do not affect the application's functionality
|
||||
- **perf**: code change that improves performance
|
||||
- **refactor**: code change that neither fixes a bug nor adds a feature
|
||||
- **style**: changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
|
||||
- **test**: adding missing tests or correcting existing tests
|
||||
Before creating a PR:
|
||||
1. Complete checklist in `.github/pull_request_template.md`
|
||||
2. Run all relevant tests and linters
|
||||
3. Link screenshots for UI changes
|
||||
|
||||
@@ -47,12 +47,12 @@ help: ## Show this help.
|
||||
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
|
||||
|
||||
##@ Build no cache
|
||||
build-no-cache-dev:
|
||||
docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat
|
||||
build-no-cache-dev:
|
||||
docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat mcp-server
|
||||
|
||||
##@ Development Environment
|
||||
run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, and workers
|
||||
docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat
|
||||
run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, MCP, and workers
|
||||
docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat mcp-server
|
||||
|
||||
##@ Development Environment
|
||||
build-and-run-api-dev: build-no-cache-dev run-api-dev
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<b><i>Prowler</b> is the Open Cloud Security platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
|
||||
</p>
|
||||
<p align="center">
|
||||
<b>Learn more at <a href="https://prowler.com">prowler.com</i></b>
|
||||
<b>Secure ANY cloud at AI Speed at <a href="https://prowler.com">prowler.com</i></b>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -23,6 +23,7 @@
|
||||
<a href="https://hub.docker.com/r/toniblyx/prowler"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/toniblyx/prowler"></a>
|
||||
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height=19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://github.com/prowler-cloud/prowler/releases"><img alt="Version" src="https://img.shields.io/github/v/release/prowler-cloud/prowler"></a>
|
||||
@@ -35,28 +36,32 @@
|
||||
</p>
|
||||
<hr>
|
||||
<p align="center">
|
||||
<img align="center" src="/docs/img/prowler-cli-quick.gif" width="100%" height="100%">
|
||||
<img align="center" src="/docs/img/prowler-cloud.gif" width="100%" height="100%">
|
||||
</p>
|
||||
|
||||
# Description
|
||||
|
||||
**Prowler** is an open-source security tool designed to assess and enforce security best practices across AWS, Azure, Google Cloud, and Kubernetes. It supports tasks such as security audits, incident response, continuous monitoring, system hardening, forensic readiness, and remediation processes.
|
||||
**Prowler** is the world’s most widely used _open-source cloud security platform_ that automates security and compliance across **any cloud environment**. With hundreds of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size.
|
||||
|
||||
Prowler includes hundreds of built-in controls to ensure compliance with standards and frameworks, including:
|
||||
|
||||
- **Industry Standards:** CIS, NIST 800, NIST CSF, and CISA
|
||||
- **Regulatory Compliance and Governance:** RBI, FedRAMP, and PCI-DSS
|
||||
- **Prowler ThreatScore:** Weighted risk prioritization scoring that helps you focus on the most critical security findings first
|
||||
- **Industry Standards:** CIS, NIST 800, NIST CSF, CISA, and MITRE ATT&CK
|
||||
- **Regulatory Compliance and Governance:** RBI, FedRAMP, PCI-DSS, and NIS2
|
||||
- **Frameworks for Sensitive Data and Privacy:** GDPR, HIPAA, and FFIEC
|
||||
- **Frameworks for Organizational Governance and Quality Control:** SOC2 and GXP
|
||||
- **AWS-Specific Frameworks:** AWS Foundational Technical Review (FTR) and AWS Well-Architected Framework (Security Pillar)
|
||||
- **National Security Standards:** ENS (Spanish National Security Scheme)
|
||||
- **Frameworks for Organizational Governance and Quality Control:** SOC2, GXP, and ISO 27001
|
||||
- **Cloud-Specific Frameworks:** AWS Foundational Technical Review (FTR), AWS Well-Architected Framework, and BSI C5
|
||||
- **National Security Standards:** ENS (Spanish National Security Scheme) and KISA ISMS-P (Korean)
|
||||
- **Custom Security Frameworks:** Tailored to your needs
|
||||
|
||||
## Prowler App
|
||||
## Prowler App / Prowler Cloud
|
||||
|
||||
Prowler App is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments.
|
||||
Prowler App / [Prowler Cloud](https://cloud.prowler.com/) is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments.
|
||||
|
||||

|
||||

|
||||

|
||||
|
||||
|
||||
>For more details, refer to the [Prowler App Documentation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-app-installation)
|
||||
|
||||
@@ -75,6 +80,23 @@ prowler dashboard
|
||||
```
|
||||

|
||||
|
||||
|
||||
## Attack Paths
|
||||
|
||||
Attack Paths automatically extends every completed AWS scan with a Neo4j graph that combines Cartography's cloud inventory with Prowler findings. The feature runs in the API worker after each scan and therefore requires:
|
||||
|
||||
- An accessible Neo4j instance (the Docker Compose files already ships a `neo4j` service).
|
||||
- The following environment variables so Django and Celery can connect:
|
||||
|
||||
| Variable | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `NEO4J_HOST` | Hostname used by the API containers. | `neo4j` |
|
||||
| `NEO4J_PORT` | Bolt port exposed by Neo4j. | `7687` |
|
||||
| `NEO4J_USER` / `NEO4J_PASSWORD` | Credentials with rights to create per-tenant databases. | `neo4j` / `neo4j_password` |
|
||||
|
||||
Every AWS provider scan will enqueue an Attack Paths ingestion job automatically. Other cloud providers will be added in future iterations.
|
||||
|
||||
|
||||
# Prowler at a Glance
|
||||
> [!Tip]
|
||||
> For the most accurate and up-to-date information about checks, services, frameworks, and categories, visit [**Prowler Hub**](https://hub.prowler.com).
|
||||
@@ -82,16 +104,16 @@ prowler dashboard
|
||||
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|
||||
|---|---|---|---|---|---|---|
|
||||
| AWS | 576 | 82 | 39 | 10 | Official | UI, API, CLI |
|
||||
| GCP | 79 | 13 | 13 | 3 | Official | UI, API, CLI |
|
||||
| Azure | 162 | 19 | 13 | 4 | Official | UI, API, CLI |
|
||||
| Kubernetes | 83 | 7 | 5 | 7 | Official | UI, API, CLI |
|
||||
| GitHub | 17 | 2 | 1 | 0 | Official | Stable | UI, API, CLI |
|
||||
| AWS | 584 | 85 | 40 | 17 | Official | UI, API, CLI |
|
||||
| GCP | 89 | 17 | 14 | 5 | Official | UI, API, CLI |
|
||||
| Azure | 169 | 22 | 15 | 8 | Official | UI, API, CLI |
|
||||
| Kubernetes | 84 | 7 | 6 | 9 | Official | UI, API, CLI |
|
||||
| GitHub | 20 | 2 | 1 | 2 | Official | UI, API, CLI |
|
||||
| M365 | 70 | 7 | 3 | 2 | Official | UI, API, CLI |
|
||||
| OCI | 51 | 13 | 1 | 10 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 61 | 9 | 1 | 9 | Official | CLI |
|
||||
| OCI | 52 | 15 | 1 | 12 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 63 | 10 | 1 | 9 | Official | CLI |
|
||||
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 4 | 0 | 3 | Official | UI, API, CLI |
|
||||
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
|
||||
@@ -143,9 +165,9 @@ If your workstation's architecture is incompatible, you can resolve this by:
|
||||
### Common Issues with Docker Pull Installation
|
||||
|
||||
> [!Note]
|
||||
If you want to use AWS role assumption (e.g., with the "Connect assuming IAM Role" option), you may need to mount your local `.aws` directory into the container as a volume (e.g., `- "${HOME}/.aws:/home/prowler/.aws:ro"`). There are several ways to configure credentials for Docker containers. See the [Troubleshooting](./docs/troubleshooting.md) section for more details and examples.
|
||||
If you want to use AWS role assumption (e.g., with the "Connect assuming IAM Role" option), you may need to mount your local `.aws` directory into the container as a volume (e.g., `- "${HOME}/.aws:/home/prowler/.aws:ro"`). There are several ways to configure credentials for Docker containers. See the [Troubleshooting](./docs/troubleshooting.mdx) section for more details and examples.
|
||||
|
||||
You can find more information in the [Troubleshooting](./docs/troubleshooting.md) section.
|
||||
You can find more information in the [Troubleshooting](./docs/troubleshooting.mdx) section.
|
||||
|
||||
|
||||
### From GitHub
|
||||
@@ -272,11 +294,12 @@ python prowler-cli.py -v
|
||||
# ✏️ High level architecture
|
||||
|
||||
## Prowler App
|
||||
**Prowler App** is composed of three key components:
|
||||
**Prowler App** is composed of four key components:
|
||||
|
||||
- **Prowler UI**: A web-based interface, built with Next.js, providing a user-friendly experience for executing Prowler scans and visualizing results.
|
||||
- **Prowler API**: A backend service, developed with Django REST Framework, responsible for running Prowler scans and storing the generated results.
|
||||
- **Prowler SDK**: A Python SDK designed to extend the functionality of the Prowler CLI for advanced capabilities.
|
||||
- **Prowler MCP Server**: A Model Context Protocol server that provides AI tools for Lighthouse, the AI-powered security assistant. This is a critical dependency for Lighthouse functionality.
|
||||
|
||||

|
||||
|
||||
@@ -304,6 +327,45 @@ And many more environments.
|
||||
|
||||

|
||||
|
||||
# 🤖 AI Skills for Development
|
||||
|
||||
Prowler includes a comprehensive set of **AI Skills** that help AI coding assistants understand Prowler's codebase patterns and conventions.
|
||||
|
||||
## What are AI Skills?
|
||||
|
||||
Skills are structured instructions that give AI assistants the context they need to write code that follows Prowler's standards. They include:
|
||||
|
||||
- **Coding patterns** for each component (SDK, API, UI, MCP Server)
|
||||
- **Testing conventions** (pytest, Playwright)
|
||||
- **Architecture guidelines** (Clean Architecture, RLS patterns)
|
||||
- **Framework-specific rules** (React 19, Next.js 15, Django DRF, Tailwind 4)
|
||||
|
||||
## Available Skills
|
||||
|
||||
| Category | Skills |
|
||||
|----------|--------|
|
||||
| **Generic** | `typescript`, `react-19`, `nextjs-15`, `tailwind-4`, `playwright`, `pytest`, `django-drf`, `zod-4`, `zustand-5`, `ai-sdk-5` |
|
||||
| **Prowler** | `prowler`, `prowler-api`, `prowler-ui`, `prowler-mcp`, `prowler-sdk-check`, `prowler-test-ui`, `prowler-test-api`, `prowler-test-sdk`, `prowler-compliance`, `prowler-provider`, `prowler-pr`, `prowler-docs` |
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
./skills/setup.sh
|
||||
```
|
||||
|
||||
This configures skills for AI coding assistants that follow the [agentskills.io](https://agentskills.io) standard:
|
||||
|
||||
| Tool | Configuration |
|
||||
|------|---------------|
|
||||
| **Claude Code** | `.claude/skills/` (symlink) |
|
||||
| **OpenCode** | `.claude/skills/` (symlink) |
|
||||
| **Codex (OpenAI)** | `.codex/skills/` (symlink) |
|
||||
| **GitHub Copilot** | `.github/skills/` (symlink) |
|
||||
| **Gemini CLI** | `.gemini/skills/` (symlink) |
|
||||
|
||||
> **Note:** Restart your AI coding assistant after running setup to load the skills.
|
||||
> Gemini CLI requires `experimental.skills` enabled in settings.
|
||||
|
||||
# 📖 Documentation
|
||||
|
||||
For installation instructions, usage details, tutorials, and the Developer Guide, visit https://docs.prowler.com/
|
||||
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
# Prowler API - AI Agent Ruleset
|
||||
|
||||
> **Skills Reference**: For detailed patterns, use these skills:
|
||||
> - [`prowler-api`](../skills/prowler-api/SKILL.md) - Models, Serializers, Views, RLS patterns
|
||||
> - [`prowler-test-api`](../skills/prowler-test-api/SKILL.md) - Testing patterns (pytest-django)
|
||||
> - [`django-drf`](../skills/django-drf/SKILL.md) - Generic DRF patterns
|
||||
> - [`jsonapi`](../skills/jsonapi/SKILL.md) - Strict JSON:API v1.1 spec compliance
|
||||
> - [`pytest`](../skills/pytest/SKILL.md) - Generic pytest patterns
|
||||
|
||||
### Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
| Action | Skill |
|
||||
|--------|-------|
|
||||
| Add changelog entry for a PR or feature | `prowler-changelog` |
|
||||
| Adding DRF pagination or permissions | `django-drf` |
|
||||
| Committing changes | `prowler-commit` |
|
||||
| Create PR that requires changelog entry | `prowler-changelog` |
|
||||
| Creating API endpoints | `jsonapi` |
|
||||
| Creating ViewSets, serializers, or filters in api/ | `django-drf` |
|
||||
| Creating a git commit | `prowler-commit` |
|
||||
| Creating/modifying models, views, serializers | `prowler-api` |
|
||||
| Implementing JSON:API endpoints | `django-drf` |
|
||||
| Modifying API responses | `jsonapi` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Testing RLS tenant isolation | `prowler-test-api` |
|
||||
| Update CHANGELOG.md in any component | `prowler-changelog` |
|
||||
| Writing Prowler API tests | `prowler-test-api` |
|
||||
| Writing Python tests with pytest | `pytest` |
|
||||
|
||||
---
|
||||
|
||||
## CRITICAL RULES - NON-NEGOTIABLE
|
||||
|
||||
### Models
|
||||
- ALWAYS: UUIDv4 PKs, `inserted_at`/`updated_at` timestamps, `JSONAPIMeta` class
|
||||
- ALWAYS: Inherit from `RowLevelSecurityProtectedModel` for tenant-scoped data
|
||||
- NEVER: Auto-increment integer PKs, models without tenant isolation
|
||||
|
||||
### Serializers
|
||||
- ALWAYS: Separate serializers for Create/Update operations
|
||||
- ALWAYS: Inherit from `RLSSerializer` for tenant-scoped models
|
||||
- NEVER: Write logic in serializers (use services/utils)
|
||||
|
||||
### Views
|
||||
- ALWAYS: Inherit from `BaseRLSViewSet` for tenant-scoped resources
|
||||
- ALWAYS: Define `filterset_class`, use `@extend_schema` for OpenAPI
|
||||
- NEVER: Raw SQL queries, business logic in views
|
||||
|
||||
### Row-Level Security (RLS)
|
||||
- ALWAYS: Use `rls_transaction(tenant_id)` context manager
|
||||
- NEVER: Query across tenants, trust client-provided tenant_id
|
||||
|
||||
### Celery Tasks
|
||||
- ALWAYS: `@shared_task` with `name`, `queue`, `RLSTask` base class
|
||||
- NEVER: Long-running ops in views, request context in tasks
|
||||
|
||||
---
|
||||
|
||||
## DECISION TREES
|
||||
|
||||
### Serializer Selection
|
||||
```
|
||||
Read → <Model>Serializer
|
||||
Create → <Model>CreateSerializer
|
||||
Update → <Model>UpdateSerializer
|
||||
Nested read → <Model>IncludeSerializer
|
||||
```
|
||||
|
||||
### Task vs View
|
||||
```
|
||||
< 100ms → View
|
||||
> 100ms or external API → Celery task
|
||||
Needs retry → Celery task
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## TECH STACK
|
||||
|
||||
Django 5.1.x | DRF 3.15.x | djangorestframework-jsonapi 7.x | Celery 5.4.x | PostgreSQL 16 | pytest 8.x
|
||||
|
||||
---
|
||||
|
||||
## PROJECT STRUCTURE
|
||||
|
||||
```
|
||||
api/src/backend/
|
||||
├── api/ # Main Django app
|
||||
│ ├── v1/ # API version 1 (views, serializers, urls)
|
||||
│ ├── models.py # Django models
|
||||
│ ├── filters.py # FilterSet classes
|
||||
│ ├── base_views.py # Base ViewSet classes
|
||||
│ ├── rls.py # Row-Level Security
|
||||
│ └── tests/ # Unit tests
|
||||
├── config/ # Django configuration
|
||||
└── tasks/ # Celery tasks
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## COMMANDS
|
||||
|
||||
```bash
|
||||
# Development
|
||||
poetry run python src/backend/manage.py runserver
|
||||
poetry run celery -A config.celery worker -l INFO
|
||||
|
||||
# Database
|
||||
poetry run python src/backend/manage.py makemigrations
|
||||
poetry run python src/backend/manage.py migrate
|
||||
|
||||
# Testing & Linting
|
||||
poetry run pytest -x --tb=short
|
||||
poetry run make lint
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## QA CHECKLIST
|
||||
|
||||
- [ ] `poetry run pytest` passes
|
||||
- [ ] `poetry run make lint` passes
|
||||
- [ ] Migrations created if models changed
|
||||
- [ ] New endpoints have `@extend_schema` decorators
|
||||
- [ ] RLS properly applied for tenant data
|
||||
- [ ] Tests cover success and error cases
|
||||
|
||||
---
|
||||
|
||||
## NAMING CONVENTIONS
|
||||
|
||||
| Entity | Pattern | Example |
|
||||
|--------|---------|---------|
|
||||
| Serializer (read) | `<Model>Serializer` | `ProviderSerializer` |
|
||||
| Serializer (create) | `<Model>CreateSerializer` | `ProviderCreateSerializer` |
|
||||
| Serializer (update) | `<Model>UpdateSerializer` | `ProviderUpdateSerializer` |
|
||||
| Filter | `<Model>Filter` | `ProviderFilter` |
|
||||
| ViewSet | `<Model>ViewSet` | `ProviderViewSet` |
|
||||
| Task | `<action>_<entity>_task` | `sync_provider_resources_task` |
|
||||
|
||||
---
|
||||
|
||||
## API CONVENTIONS (JSON:API)
|
||||
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"type": "providers",
|
||||
"id": "uuid",
|
||||
"attributes": { "name": "value" },
|
||||
"relationships": { "tenant": { "data": { "type": "tenants", "id": "uuid" } } }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- Content-Type: `application/vnd.api+json`
|
||||
- Pagination: `?page[number]=1&page[size]=20`
|
||||
- Filtering: `?filter[field]=value`, `?filter[field__in]=val1,val2`
|
||||
- Sorting: `?sort=field`, `?sort=-field`
|
||||
- Including: `?include=provider,findings`
|
||||
+183
-53
@@ -2,18 +2,101 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.16.0] (Unreleased)
|
||||
## [1.19.0] (Prowler UNRELEASED)
|
||||
|
||||
### Added
|
||||
### 🔄 Changed
|
||||
|
||||
- Lazy-load providers and compliance data to reduce API/worker startup memory and time [(#9857)](https://github.com/prowler-cloud/prowler/pull/9857)
|
||||
|
||||
---
|
||||
|
||||
## [1.18.1] (Prowler v5.17.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Improve API startup process by `manage.py` argument detection [(#9856)](https://github.com/prowler-cloud/prowler/pull/9856)
|
||||
- Deleting providers don't try to delete a `None` Neo4j database when an Attack Paths scan is scheduled [(#9858)](https://github.com/prowler-cloud/prowler/pull/9858)
|
||||
- Use replica database for reading Findings to add them to the Attack Paths graph [(#9861)](https://github.com/prowler-cloud/prowler/pull/9861)
|
||||
- Attack paths findings loading query to use streaming generator for O(batch_size) memory instead of O(total_findings) [(#9862)](https://github.com/prowler-cloud/prowler/pull/9862)
|
||||
- Lazy load Neo4j driver [(#9868)](https://github.com/prowler-cloud/prowler/pull/9868)
|
||||
- Use `Findings.all_objects` to avoid the `ActiveProviderPartitionedManager` [(#9869)](https://github.com/prowler-cloud/prowler/pull/9869)
|
||||
- Lazy load Neo4j driver for workers only [(#9872)](https://github.com/prowler-cloud/prowler/pull/9872)
|
||||
- Improve Cypher query for inserting Findings into Attack Paths scan graphs [(#9874)](https://github.com/prowler-cloud/prowler/pull/9874)
|
||||
- Clear Neo4j database cache after Attack Paths scan and each API query [(#9877)](https://github.com/prowler-cloud/prowler/pull/9877)
|
||||
- Deduplicated scheduled scans for long-running providers [(#9829)](https://github.com/prowler-cloud/prowler/pull/9829)
|
||||
|
||||
---
|
||||
|
||||
## [1.18.0] (Prowler v5.17.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `/api/v1/overviews/compliance-watchlist` endpoint to retrieve the compliance watchlist [(#9596)](https://github.com/prowler-cloud/prowler/pull/9596)
|
||||
- AlibabaCloud provider support [(#9485)](https://github.com/prowler-cloud/prowler/pull/9485)
|
||||
- `/api/v1/overviews/resource-groups` endpoint to retrieve an overview of resource groups based on finding severities [(#9694)](https://github.com/prowler-cloud/prowler/pull/9694)
|
||||
- `group` filter for `GET /findings` and `GET /findings/metadata/latest` endpoints [(#9694)](https://github.com/prowler-cloud/prowler/pull/9694)
|
||||
- `provider_id` and `provider_id__in` filter aliases for findings endpoints to enable consistent frontend parameter naming [(#9701)](https://github.com/prowler-cloud/prowler/pull/9701)
|
||||
- Attack Paths: `/api/v1/attack-paths-scans` for AWS providers backed by Neo4j [(#9805)](https://github.com/prowler-cloud/prowler/pull/9805)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- Django 5.1.15 (CVE-2025-64460, CVE-2025-13372), Werkzeug 3.1.4 (CVE-2025-66221), sqlparse 0.5.5 (PVE-2025-82038), fonttools 4.60.2 (CVE-2025-66034) [(#9730)](https://github.com/prowler-cloud/prowler/pull/9730)
|
||||
- `safety` to `3.7.0` and `filelock` to `3.20.3` due to [Safety vulnerability 82754 (CVE-2025-68146)](https://data.safetycli.com/v/82754/97c/) [(#9816)](https://github.com/prowler-cloud/prowler/pull/9816)
|
||||
- `pyasn1` to v0.6.2 to address [CVE-2026-23490](https://nvd.nist.gov/vuln/detail/CVE-2026-23490) [(#9818)](https://github.com/prowler-cloud/prowler/pull/9818)
|
||||
- `django-allauth[saml]` to v65.13.0 to address [CVE-2025-65431](https://nvd.nist.gov/vuln/detail/CVE-2025-65431) [(#9575)](https://github.com/prowler-cloud/prowler/pull/9575)
|
||||
|
||||
---
|
||||
|
||||
## [1.17.1] (Prowler v5.16.1)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Security Hub integration error when no regions [(#9635)](https://github.com/prowler-cloud/prowler/pull/9635)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Orphan scheduled scans caused by transaction isolation during provider creation [(#9633)](https://github.com/prowler-cloud/prowler/pull/9633)
|
||||
|
||||
---
|
||||
|
||||
## [1.17.0] (Prowler v5.16.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- New endpoint to retrieve and overview of the categories based on finding severities [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
|
||||
- Endpoints `GET /findings` and `GET /findings/latests` can now use the category filter [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
|
||||
- Account id, alias and provider name to PDF reporting table [(#9574)](https://github.com/prowler-cloud/prowler/pull/9574)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Endpoint `GET /overviews/attack-surfaces` no longer returns the related check IDs [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
|
||||
- OpenAI provider to only load chat-compatible models with tool calling support [(#9523)](https://github.com/prowler-cloud/prowler/pull/9523)
|
||||
- Increased execution delay for the first scheduled scan tasks to 5 seconds[(#9558)](https://github.com/prowler-cloud/prowler/pull/9558)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Made `scan_id` a required filter in the compliance overview endpoint [(#9560)](https://github.com/prowler-cloud/prowler/pull/9560)
|
||||
- Reduced unnecessary UPDATE resources operations by only saving when tag mappings change, lowering write load during scans [(#9569)](https://github.com/prowler-cloud/prowler/pull/9569)
|
||||
|
||||
---
|
||||
|
||||
## [1.16.1] (Prowler v5.15.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Race condition in scheduled scan creation by adding countdown to task [(#9516)](https://github.com/prowler-cloud/prowler/pull/9516)
|
||||
|
||||
## [1.16.0] (Prowler v5.15.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- New endpoint to retrieve an overview of the attack surfaces [(#9309)](https://github.com/prowler-cloud/prowler/pull/9309)
|
||||
- New endpoint `GET /api/v1/overviews/findings_severity/timeseries` to retrieve daily aggregated findings by severity level [(#9363)](https://github.com/prowler-cloud/prowler/pull/9363)
|
||||
- Lighthouse AI support for Amazon Bedrock API key [(#9343)](https://github.com/prowler-cloud/prowler/pull/9343)
|
||||
- Exception handler for provider deletions during scans [(#9414)](https://github.com/prowler-cloud/prowler/pull/9414)
|
||||
- Support to use admin credentials through the read replica database [(#9440)](https://github.com/prowler-cloud/prowler/pull/9440)
|
||||
- Support AlibabaCloud provider [(#9485)](https://github.com/prowler-cloud/prowler/pull/9485)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Error messages from Lighthouse celery tasks [(#9165)](https://github.com/prowler-cloud/prowler/pull/9165)
|
||||
- Restore the compliance overview endpoint's mandatory filters [(#9338)](https://github.com/prowler-cloud/prowler/pull/9338)
|
||||
@@ -22,7 +105,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.15.2] (Prowler v5.14.2)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Unique constraint violation during compliance overviews task [(#9436)](https://github.com/prowler-cloud/prowler/pull/9436)
|
||||
- Division by zero error in ENS PDF report when all requirements are manual [(#9443)](https://github.com/prowler-cloud/prowler/pull/9443)
|
||||
|
||||
@@ -30,7 +114,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.15.1] (Prowler v5.14.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Fix typo in PDF reporting [(#9345)](https://github.com/prowler-cloud/prowler/pull/9345)
|
||||
- Fix IaC provider initialization failure when mutelist processor is configured [(#9331)](https://github.com/prowler-cloud/prowler/pull/9331)
|
||||
- Match logic for ThreatScore when counting findings [(#9348)](https://github.com/prowler-cloud/prowler/pull/9348)
|
||||
@@ -39,7 +124,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.15.0] (Prowler v5.14.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- IaC (Infrastructure as Code) provider support for remote repositories [(#8751)](https://github.com/prowler-cloud/prowler/pull/8751)
|
||||
- Extend `GET /api/v1/providers` with provider-type filters and optional pagination disable to support the new Overview filters [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975)
|
||||
- New endpoint to retrieve the number of providers grouped by provider type [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975)
|
||||
@@ -58,11 +144,13 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Enhanced compliance overview endpoint with provider filtering and latest scan aggregation [(#9244)](https://github.com/prowler-cloud/prowler/pull/9244)
|
||||
- New endpoint `GET /api/v1/overview/regions` to retrieve aggregated findings data by region [(#9273)](https://github.com/prowler-cloud/prowler/pull/9273)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Optimized database write queries for scan related tasks [(#9190)](https://github.com/prowler-cloud/prowler/pull/9190)
|
||||
- Date filters are now optional for `GET /api/v1/overviews/services` endpoint; returns latest scan data by default [(#9248)](https://github.com/prowler-cloud/prowler/pull/9248)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Scans no longer fail when findings have UIDs exceeding 300 characters; such findings are now skipped with detailed logging [(#9246)](https://github.com/prowler-cloud/prowler/pull/9246)
|
||||
- Updated unique constraint for `Provider` model to exclude soft-deleted entries, resolving duplicate errors when re-deleting providers [(#9054)](https://github.com/prowler-cloud/prowler/pull/9054)
|
||||
- Removed compliance generation for providers without compliance frameworks [(#9208)](https://github.com/prowler-cloud/prowler/pull/9208)
|
||||
@@ -70,14 +158,16 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Severity overview endpoint now ignores muted findings as expected [(#9283)](https://github.com/prowler-cloud/prowler/pull/9283)
|
||||
- Fixed discrepancy between ThreatScore PDF report values and database calculations [(#9296)](https://github.com/prowler-cloud/prowler/pull/9296)
|
||||
|
||||
### Security
|
||||
### 🔐 Security
|
||||
|
||||
- Django updated to the latest 5.1 security release, 5.1.14, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/113) and [denial-of-service vulnerability](https://github.com/prowler-cloud/prowler/security/dependabot/114) [(#9176)](https://github.com/prowler-cloud/prowler/pull/9176)
|
||||
|
||||
---
|
||||
|
||||
## [1.14.1] (Prowler v5.13.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- `/api/v1/overviews/providers` collapses data by provider type so the UI receives a single aggregated record per cloud family even when multiple accounts exist [(#9053)](https://github.com/prowler-cloud/prowler/pull/9053)
|
||||
- Added retry logic to database transactions to handle Aurora read replica connection failures during scale-down events [(#9064)](https://github.com/prowler-cloud/prowler/pull/9064)
|
||||
- Security Hub integrations stop failing when they read relationships via the replica by allowing replica relations and saving updates through the primary [(#9080)](https://github.com/prowler-cloud/prowler/pull/9080)
|
||||
@@ -86,7 +176,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.14.0] (Prowler v5.13.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Default JWT keys are generated and stored if they are missing from configuration [(#8655)](https://github.com/prowler-cloud/prowler/pull/8655)
|
||||
- `compliance_name` for each compliance [(#7920)](https://github.com/prowler-cloud/prowler/pull/7920)
|
||||
- Support C5 compliance framework for the AWS provider [(#8830)](https://github.com/prowler-cloud/prowler/pull/8830)
|
||||
@@ -99,35 +190,41 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Support Common Cloud Controls for AWS, Azure and GCP [(#8000)](https://github.com/prowler-cloud/prowler/pull/8000)
|
||||
- Add `provider_id__in` filter support to findings and findings severity overview endpoints [(#8951)](https://github.com/prowler-cloud/prowler/pull/8951)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Now the MANAGE_ACCOUNT permission is required to modify or read user permissions instead of MANAGE_USERS [(#8281)](https://github.com/prowler-cloud/prowler/pull/8281)
|
||||
- Now at least one user with MANAGE_ACCOUNT permission is required in the tenant [(#8729)](https://github.com/prowler-cloud/prowler/pull/8729)
|
||||
|
||||
### Security
|
||||
### 🔐 Security
|
||||
|
||||
- Django updated to the latest 5.1 security release, 5.1.13, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/104) and [directory traversals](https://github.com/prowler-cloud/prowler/security/dependabot/103) [(#8842)](https://github.com/prowler-cloud/prowler/pull/8842)
|
||||
|
||||
---
|
||||
|
||||
## [1.13.2] (Prowler v5.12.3)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- 500 error when deleting user [(#8731)](https://github.com/prowler-cloud/prowler/pull/8731)
|
||||
|
||||
---
|
||||
|
||||
## [1.13.1] (Prowler v5.12.2)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Renamed compliance overview task queue to `compliance` [(#8755)](https://github.com/prowler-cloud/prowler/pull/8755)
|
||||
|
||||
### Security
|
||||
### 🔐 Security
|
||||
|
||||
- Django updated to the latest 5.1 security release, 5.1.12, due to [problems](https://www.djangoproject.com/weblog/2025/sep/03/security-releases/) with potential SQL injection in FilteredRelation column aliases [(#8693)](https://github.com/prowler-cloud/prowler/pull/8693)
|
||||
|
||||
---
|
||||
|
||||
## [1.13.0] (Prowler v5.12.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Integration with JIRA, enabling sending findings to a JIRA project [(#8622)](https://github.com/prowler-cloud/prowler/pull/8622), [(#8637)](https://github.com/prowler-cloud/prowler/pull/8637)
|
||||
- `GET /overviews/findings_severity` now supports `filter[status]` and `filter[status__in]` to aggregate by specific statuses (`FAIL`, `PASS`)[(#8186)](https://github.com/prowler-cloud/prowler/pull/8186)
|
||||
- Throttling options for `/api/v1/tokens` using the `DJANGO_THROTTLE_TOKEN_OBTAIN` environment variable [(#8647)](https://github.com/prowler-cloud/prowler/pull/8647)
|
||||
@@ -136,101 +233,120 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.12.0] (Prowler v5.11.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Lighthouse support for OpenAI GPT-5 [(#8527)](https://github.com/prowler-cloud/prowler/pull/8527)
|
||||
- Integration with Amazon Security Hub, enabling sending findings to Security Hub [(#8365)](https://github.com/prowler-cloud/prowler/pull/8365)
|
||||
- Generate ASFF output for AWS providers with SecurityHub integration enabled [(#8569)](https://github.com/prowler-cloud/prowler/pull/8569)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- GitHub provider always scans user instead of organization when using provider UID [(#8587)](https://github.com/prowler-cloud/prowler/pull/8587)
|
||||
|
||||
---
|
||||
|
||||
## [1.11.0] (Prowler v5.10.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Github provider support [(#8271)](https://github.com/prowler-cloud/prowler/pull/8271)
|
||||
- Integration with Amazon S3, enabling storage and retrieval of scan data via S3 buckets [(#8056)](https://github.com/prowler-cloud/prowler/pull/8056)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Avoid sending errors to Sentry in M365 provider when user authentication fails [(#8420)](https://github.com/prowler-cloud/prowler/pull/8420)
|
||||
|
||||
---
|
||||
|
||||
## [1.10.2] (Prowler v5.9.2)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Optimized queries for resources views [(#8336)](https://github.com/prowler-cloud/prowler/pull/8336)
|
||||
|
||||
---
|
||||
|
||||
## [v1.10.1] (Prowler v5.9.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Calculate failed findings during scans to prevent heavy database queries [(#8322)](https://github.com/prowler-cloud/prowler/pull/8322)
|
||||
|
||||
---
|
||||
|
||||
## [v1.10.0] (Prowler v5.9.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- SSO with SAML support [(#8175)](https://github.com/prowler-cloud/prowler/pull/8175)
|
||||
- `GET /resources/metadata`, `GET /resources/metadata/latest` and `GET /resources/latest` to expose resource metadata and latest scan results [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- `/processors` endpoints to post-process findings. Currently, only the Mutelist processor is supported to allow to mute findings.
|
||||
- Optimized the underlying queries for resources endpoints [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112)
|
||||
- Optimized include parameters for resources view [(#8229)](https://github.com/prowler-cloud/prowler/pull/8229)
|
||||
- Optimized overview background tasks [(#8300)](https://github.com/prowler-cloud/prowler/pull/8300)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Search filter for findings and resources [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112)
|
||||
- RBAC is now applied to `GET /overviews/providers` [(#8277)](https://github.com/prowler-cloud/prowler/pull/8277)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- `POST /schedules/daily` returns a `409 CONFLICT` if already created [(#8258)](https://github.com/prowler-cloud/prowler/pull/8258)
|
||||
|
||||
### Security
|
||||
### 🔐 Security
|
||||
|
||||
- Enhanced password validation to enforce 12+ character passwords with special characters, uppercase, lowercase, and numbers [(#8225)](https://github.com/prowler-cloud/prowler/pull/8225)
|
||||
|
||||
---
|
||||
|
||||
## [v1.9.1] (Prowler v5.8.1)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Custom exception for provider connection errors during scans [(#8234)](https://github.com/prowler-cloud/prowler/pull/8234)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Summary and overview tasks now use a dedicated queue and no longer propagate errors to compliance tasks [(#8214)](https://github.com/prowler-cloud/prowler/pull/8214)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Scan with no resources will not trigger legacy code for findings metadata [(#8183)](https://github.com/prowler-cloud/prowler/pull/8183)
|
||||
- Invitation email comparison case-insensitive [(#8206)](https://github.com/prowler-cloud/prowler/pull/8206)
|
||||
|
||||
### Removed
|
||||
### ❌ Removed
|
||||
|
||||
- Validation of the provider's secret type during updates [(#8197)](https://github.com/prowler-cloud/prowler/pull/8197)
|
||||
|
||||
---
|
||||
|
||||
## [v1.9.0] (Prowler v5.8.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Support GCP Service Account key [(#7824)](https://github.com/prowler-cloud/prowler/pull/7824)
|
||||
- `GET /compliance-overviews` endpoints to retrieve compliance metadata and specific requirements statuses [(#7877)](https://github.com/prowler-cloud/prowler/pull/7877)
|
||||
- Lighthouse configuration support [(#7848)](https://github.com/prowler-cloud/prowler/pull/7848)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Reworked `GET /compliance-overviews` to return proper requirement metrics [(#7877)](https://github.com/prowler-cloud/prowler/pull/7877)
|
||||
- Optional `user` and `password` for M365 provider [(#7992)](https://github.com/prowler-cloud/prowler/pull/7992)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Scheduled scans are no longer deleted when their daily schedule run is disabled [(#8082)](https://github.com/prowler-cloud/prowler/pull/8082)
|
||||
|
||||
---
|
||||
|
||||
## [v1.8.5] (Prowler v5.7.5)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Normalize provider UID to ensure safe and unique export directory paths [(#8007)](https://github.com/prowler-cloud/prowler/pull/8007).
|
||||
- Blank resource types in `/metadata` endpoints [(#8027)](https://github.com/prowler-cloud/prowler/pull/8027)
|
||||
|
||||
@@ -238,20 +354,24 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.8.4] (Prowler v5.7.4)
|
||||
|
||||
### Removed
|
||||
### ❌ Removed
|
||||
|
||||
- Reverted RLS transaction handling and DB custom backend [(#7994)](https://github.com/prowler-cloud/prowler/pull/7994)
|
||||
|
||||
---
|
||||
|
||||
## [v1.8.3] (Prowler v5.7.3)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Database backend to handle already closed connections [(#7935)](https://github.com/prowler-cloud/prowler/pull/7935)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Renamed field encrypted_password to password for M365 provider [(#7784)](https://github.com/prowler-cloud/prowler/pull/7784)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Transaction persistence with RLS operations [(#7916)](https://github.com/prowler-cloud/prowler/pull/7916)
|
||||
- Reverted the change `get_with_retry` to use the original `get` method for retrieving tasks [(#7932)](https://github.com/prowler-cloud/prowler/pull/7932)
|
||||
|
||||
@@ -259,7 +379,8 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.8.2] (Prowler v5.7.2)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Task lookup to use task_kwargs instead of task_args for scan report resolution [(#7830)](https://github.com/prowler-cloud/prowler/pull/7830)
|
||||
- Kubernetes UID validation to allow valid context names [(#7871)](https://github.com/prowler-cloud/prowler/pull/7871)
|
||||
- Connection status verification before launching a scan [(#7831)](https://github.com/prowler-cloud/prowler/pull/7831)
|
||||
@@ -270,14 +391,16 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.8.1] (Prowler v5.7.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Added database index to improve performance on finding lookup [(#7800)](https://github.com/prowler-cloud/prowler/pull/7800)
|
||||
|
||||
---
|
||||
|
||||
## [v1.8.0] (Prowler v5.7.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Huge improvements to `/findings/metadata` and resource related filters for findings [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690)
|
||||
- Improvements to `/overviews` endpoints [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690)
|
||||
- Queue to perform backfill background tasks [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690)
|
||||
@@ -288,7 +411,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.7.0] (Prowler v5.6.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- M365 as a new provider [(#7563)](https://github.com/prowler-cloud/prowler/pull/7563)
|
||||
- `compliance/` folder and ZIP‐export functionality for all compliance reports [(#7653)](https://github.com/prowler-cloud/prowler/pull/7653)
|
||||
@@ -298,7 +421,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.6.0] (Prowler v5.5.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Support for developing new integrations [(#7167)](https://github.com/prowler-cloud/prowler/pull/7167)
|
||||
- HTTP Security Headers [(#7289)](https://github.com/prowler-cloud/prowler/pull/7289)
|
||||
@@ -310,14 +433,16 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.5.4] (Prowler v5.4.4)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Bug with periodic tasks when trying to delete a provider [(#7466)](https://github.com/prowler-cloud/prowler/pull/7466)
|
||||
|
||||
---
|
||||
|
||||
## [v1.5.3] (Prowler v5.4.3)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Duplicated scheduled scans handling [(#7401)](https://github.com/prowler-cloud/prowler/pull/7401)
|
||||
- Environment variable to configure the deletion task batch size [(#7423)](https://github.com/prowler-cloud/prowler/pull/7423)
|
||||
|
||||
@@ -325,14 +450,16 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.5.2] (Prowler v5.4.2)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Refactored deletion logic and implemented retry mechanism for deletion tasks [(#7349)](https://github.com/prowler-cloud/prowler/pull/7349)
|
||||
|
||||
---
|
||||
|
||||
## [v1.5.1] (Prowler v5.4.1)
|
||||
|
||||
### Fixed
|
||||
### 🐞 Fixed
|
||||
|
||||
- Handle response in case local files are missing [(#7183)](https://github.com/prowler-cloud/prowler/pull/7183)
|
||||
- Race condition when deleting export files after the S3 upload [(#7172)](https://github.com/prowler-cloud/prowler/pull/7172)
|
||||
- Handle exception when a provider has no secret in test connection [(#7283)](https://github.com/prowler-cloud/prowler/pull/7283)
|
||||
@@ -341,19 +468,22 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [v1.5.0] (Prowler v5.4.0)
|
||||
|
||||
### Added
|
||||
### 🚀 Added
|
||||
|
||||
- Social login integration with Google and GitHub [(#6906)](https://github.com/prowler-cloud/prowler/pull/6906)
|
||||
- API scan report system, now all scans launched from the API will generate a compressed file with the report in OCSF, CSV and HTML formats [(#6878)](https://github.com/prowler-cloud/prowler/pull/6878)
|
||||
- Configurable Sentry integration [(#6874)](https://github.com/prowler-cloud/prowler/pull/6874)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Optimized `GET /findings` endpoint to improve response time and size [(#7019)](https://github.com/prowler-cloud/prowler/pull/7019)
|
||||
|
||||
---
|
||||
|
||||
## [v1.4.0] (Prowler v5.3.0)
|
||||
|
||||
### Changed
|
||||
### 🔄 Changed
|
||||
|
||||
- Daily scheduled scan instances are now created beforehand with `SCHEDULED` state [(#6700)](https://github.com/prowler-cloud/prowler/pull/6700)
|
||||
- Findings endpoints now require at least one date filter [(#6800)](https://github.com/prowler-cloud/prowler/pull/6800)
|
||||
- Findings metadata endpoint received a performance improvement [(#6863)](https://github.com/prowler-cloud/prowler/pull/6863)
|
||||
|
||||
@@ -32,7 +32,7 @@ start_prod_server() {
|
||||
|
||||
start_worker() {
|
||||
echo "Starting the worker..."
|
||||
poetry run python -m celery -A config.celery worker -l "${DJANGO_LOGGING_LEVEL:-info}" -Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance -E --max-tasks-per-child 1
|
||||
poetry run python -m celery -A config.celery worker -l "${DJANGO_LOGGING_LEVEL:-info}" -Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans -E --max-tasks-per-child 1
|
||||
}
|
||||
|
||||
start_worker_beat() {
|
||||
|
||||
Generated
+2452
-492
File diff suppressed because it is too large
Load Diff
+12
-6
@@ -7,8 +7,8 @@ authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}]
|
||||
dependencies = [
|
||||
"celery[pytest] (>=5.4.0,<6.0.0)",
|
||||
"dj-rest-auth[with_social,jwt] (==7.0.1)",
|
||||
"django (==5.1.14)",
|
||||
"django-allauth[saml] (>=65.8.0,<66.0.0)",
|
||||
"django (==5.1.15)",
|
||||
"django-allauth[saml] (>=65.13.0,<66.0.0)",
|
||||
"django-celery-beat (>=2.7.0,<3.0.0)",
|
||||
"django-celery-results (>=2.5.1,<3.0.0)",
|
||||
"django-cors-headers==4.4.0",
|
||||
@@ -36,7 +36,12 @@ dependencies = [
|
||||
"drf-simple-apikey (==2.2.1)",
|
||||
"matplotlib (>=3.10.6,<4.0.0)",
|
||||
"reportlab (>=4.4.4,<5.0.0)",
|
||||
"gevent (>=25.9.1,<26.0.0)"
|
||||
"neo4j (<6.0.0)",
|
||||
"cartography @ git+https://github.com/prowler-cloud/cartography@master",
|
||||
"gevent (>=25.9.1,<26.0.0)",
|
||||
"werkzeug (>=3.1.4)",
|
||||
"sqlparse (>=0.5.4)",
|
||||
"fonttools (>=4.60.2)"
|
||||
]
|
||||
description = "Prowler's API (Django/DRF)"
|
||||
license = "Apache-2.0"
|
||||
@@ -44,7 +49,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.16.0"
|
||||
version = "1.19.0"
|
||||
|
||||
[project.scripts]
|
||||
celery = "src.backend.config.settings.celery"
|
||||
@@ -65,6 +70,7 @@ pytest-env = "1.1.3"
|
||||
pytest-randomly = "3.15.0"
|
||||
pytest-xdist = "3.6.1"
|
||||
ruff = "0.5.0"
|
||||
safety = "3.2.9"
|
||||
tqdm = "4.67.1"
|
||||
safety = "3.7.0"
|
||||
filelock = "3.20.3"
|
||||
vulture = "2.14"
|
||||
tqdm = "4.67.1"
|
||||
|
||||
+37
-16
@@ -30,17 +30,48 @@ class ApiConfig(AppConfig):
|
||||
def ready(self):
|
||||
from api import schema_extensions # noqa: F401
|
||||
from api import signals # noqa: F401
|
||||
from api.compliance import load_prowler_compliance
|
||||
from api.attack_paths import database as graph_database
|
||||
|
||||
# Generate required cryptographic keys if not present, but only if:
|
||||
# `"manage.py" not in sys.argv`: If an external server (e.g., Gunicorn) is running the app
|
||||
# `"manage.py" not in sys.argv[0]`: If an external server (e.g., Gunicorn) is running the app
|
||||
# `os.environ.get("RUN_MAIN")`: If it's not a Django command or using `runserver`,
|
||||
# only the main process will do it
|
||||
if "manage.py" not in sys.argv or os.environ.get("RUN_MAIN"):
|
||||
if (len(sys.argv) >= 1 and "manage.py" not in sys.argv[0]) or os.environ.get(
|
||||
"RUN_MAIN"
|
||||
):
|
||||
self._ensure_crypto_keys()
|
||||
|
||||
load_prowler_compliance()
|
||||
self._initialize_attack_surface_mapping()
|
||||
# Commands that don't need Neo4j
|
||||
SKIP_NEO4J_DJANGO_COMMANDS = [
|
||||
"makemigrations",
|
||||
"migrate",
|
||||
"pgpartition",
|
||||
"check",
|
||||
"help",
|
||||
"showmigrations",
|
||||
"check_and_fix_socialaccount_sites_migration",
|
||||
]
|
||||
|
||||
# Skip Neo4j initialization during tests, some Django commands, and Celery
|
||||
if getattr(settings, "TESTING", False) or (
|
||||
len(sys.argv) > 1
|
||||
and (
|
||||
(
|
||||
"manage.py" in sys.argv[0]
|
||||
and sys.argv[1] in SKIP_NEO4J_DJANGO_COMMANDS
|
||||
)
|
||||
or "celery" in sys.argv[0]
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
"Skipping Neo4j initialization because tests, some Django commands or Celery"
|
||||
)
|
||||
|
||||
else:
|
||||
graph_database.init_driver()
|
||||
|
||||
# Neo4j driver is initialized at API startup (see api.attack_paths.database)
|
||||
# It remains lazy for Celery workers and selected Django commands
|
||||
|
||||
def _ensure_crypto_keys(self):
|
||||
"""
|
||||
@@ -55,7 +86,7 @@ class ApiConfig(AppConfig):
|
||||
global _keys_initialized
|
||||
|
||||
# Skip key generation if running tests
|
||||
if hasattr(settings, "TESTING") and settings.TESTING:
|
||||
if getattr(settings, "TESTING", False):
|
||||
return
|
||||
|
||||
# Skip if already initialized in this process
|
||||
@@ -168,13 +199,3 @@ class ApiConfig(AppConfig):
|
||||
f"Error generating JWT keys: {e}. Please set '{SIGNING_KEY_ENV}' and '{VERIFYING_KEY_ENV}' manually."
|
||||
)
|
||||
raise e
|
||||
|
||||
def _initialize_attack_surface_mapping(self):
|
||||
from tasks.jobs.scan import ( # noqa: F401
|
||||
_get_attack_surface_mapping_from_provider,
|
||||
)
|
||||
|
||||
from api.models import Provider # noqa: F401
|
||||
|
||||
for provider_type, _label in Provider.ProviderChoices.choices:
|
||||
_get_attack_surface_mapping_from_provider(provider_type)
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
from api.attack_paths.query_definitions import (
|
||||
AttackPathsQueryDefinition,
|
||||
AttackPathsQueryParameterDefinition,
|
||||
get_queries_for_provider,
|
||||
get_query_by_id,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AttackPathsQueryDefinition",
|
||||
"AttackPathsQueryParameterDefinition",
|
||||
"get_queries_for_provider",
|
||||
"get_query_by_id",
|
||||
]
|
||||
@@ -0,0 +1,161 @@
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator
|
||||
from uuid import UUID
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from django.conf import settings
|
||||
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
|
||||
# Without this Celery goes crazy with Neo4j logging
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = 3
|
||||
|
||||
# Module-level process-wide driver singleton
|
||||
_driver: neo4j.Driver | None = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
# Base Neo4j functions
|
||||
|
||||
|
||||
def get_uri() -> str:
|
||||
host = settings.DATABASES["neo4j"]["HOST"]
|
||||
port = settings.DATABASES["neo4j"]["PORT"]
|
||||
return f"bolt://{host}:{port}"
|
||||
|
||||
|
||||
def init_driver() -> neo4j.Driver:
|
||||
global _driver
|
||||
if _driver is not None:
|
||||
return _driver
|
||||
|
||||
with _lock:
|
||||
if _driver is None:
|
||||
uri = get_uri()
|
||||
config = settings.DATABASES["neo4j"]
|
||||
|
||||
_driver = neo4j.GraphDatabase.driver(
|
||||
uri,
|
||||
auth=(config["USER"], config["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=7200,
|
||||
connection_acquisition_timeout=120,
|
||||
max_connection_pool_size=50,
|
||||
)
|
||||
_driver.verify_connectivity()
|
||||
|
||||
# Register cleanup handler (only runs once since we're inside the _driver is None block)
|
||||
atexit.register(close_driver)
|
||||
|
||||
return _driver
|
||||
|
||||
|
||||
def get_driver() -> neo4j.Driver:
|
||||
return init_driver()
|
||||
|
||||
|
||||
def close_driver() -> None: # TODO: Use it
|
||||
global _driver
|
||||
with _lock:
|
||||
if _driver is not None:
|
||||
try:
|
||||
_driver.close()
|
||||
|
||||
finally:
|
||||
_driver = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_session(database: str | None = None) -> Iterator[RetryableSession]:
|
||||
session_wrapper: RetryableSession | None = None
|
||||
|
||||
try:
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: get_driver().session(database=database),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
raise GraphDatabaseQueryException(message=exc.message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
|
||||
def create_database(database: str) -> None:
|
||||
query = "CREATE DATABASE $database IF NOT EXISTS"
|
||||
parameters = {"database": database}
|
||||
|
||||
with get_session() as session:
|
||||
session.run(query, parameters)
|
||||
|
||||
|
||||
def drop_database(database: str) -> None:
|
||||
query = f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA"
|
||||
|
||||
with get_session() as session:
|
||||
session.run(query)
|
||||
|
||||
|
||||
def drop_subgraph(database: str, root_node_label: str, root_node_id: str) -> int:
|
||||
query = """
|
||||
MATCH (a:__ROOT_NODE_LABEL__ {id: $root_node_id})
|
||||
CALL apoc.path.subgraphNodes(a, {})
|
||||
YIELD node
|
||||
DETACH DELETE node
|
||||
RETURN COUNT(node) AS deleted_nodes_count
|
||||
""".replace("__ROOT_NODE_LABEL__", root_node_label)
|
||||
parameters = {"root_node_id": root_node_id}
|
||||
|
||||
with get_session(database) as session:
|
||||
result = session.run(query, parameters)
|
||||
|
||||
try:
|
||||
return result.single()["deleted_nodes_count"]
|
||||
|
||||
except neo4j.exceptions.ResultConsumedError:
|
||||
return 0 # As there are no nodes to delete, the result is empty
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
query = "CALL db.clearQueryCaches()"
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
session.run(query)
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
logging.warning(f"Failed to clear query cache for database `{database}`: {exc}")
|
||||
|
||||
|
||||
# Neo4j functions related to Prowler + Cartography
|
||||
DATABASE_NAME_TEMPLATE = "db-{attack_paths_scan_id}"
|
||||
|
||||
|
||||
def get_database_name(attack_paths_scan_id: UUID) -> str:
|
||||
attack_paths_scan_id_str = str(attack_paths_scan_id).lower()
|
||||
return DATABASE_NAME_TEMPLATE.format(attack_paths_scan_id=attack_paths_scan_id_str)
|
||||
|
||||
|
||||
# Exceptions
|
||||
|
||||
|
||||
class GraphDatabaseQueryException(Exception):
|
||||
def __init__(self, message: str, code: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.code = code
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.code:
|
||||
return f"{self.code}: {self.message}"
|
||||
|
||||
return self.message
|
||||
@@ -0,0 +1,514 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
# Dataclases for handling API's Attack Path query definitions and their parameters
|
||||
@dataclass
|
||||
class AttackPathsQueryParameterDefinition:
|
||||
"""
|
||||
Metadata describing a parameter that must be provided to an Attack Paths query.
|
||||
"""
|
||||
|
||||
name: str
|
||||
label: str
|
||||
data_type: str = "string"
|
||||
cast: type = str
|
||||
description: str | None = None
|
||||
placeholder: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttackPathsQueryDefinition:
|
||||
"""
|
||||
Immutable representation of an Attack Path query.
|
||||
"""
|
||||
|
||||
id: str
|
||||
name: str
|
||||
description: str
|
||||
provider: str
|
||||
cypher: str
|
||||
parameters: list[AttackPathsQueryParameterDefinition] = field(default_factory=list)
|
||||
|
||||
|
||||
# Accessor functions for API's Attack Paths query definitions
|
||||
def get_queries_for_provider(provider: str) -> list[AttackPathsQueryDefinition]:
|
||||
return _QUERY_DEFINITIONS.get(provider, [])
|
||||
|
||||
|
||||
def get_query_by_id(query_id: str) -> AttackPathsQueryDefinition | None:
|
||||
return _QUERIES_BY_ID.get(query_id)
|
||||
|
||||
|
||||
# API's Attack Paths query definitions
|
||||
_QUERY_DEFINITIONS: dict[str, list[AttackPathsQueryDefinition]] = {
|
||||
"aws": [
|
||||
# Custom query for detecting internet-exposed EC2 instances with sensitive S3 access
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-internet-exposed-ec2-sensitive-s3-access",
|
||||
name="Identify internet-exposed EC2 instances with sensitive S3 access",
|
||||
description="Detect EC2 instances with SSH exposed to the internet that can assume higher-privileged roles to read tagged sensitive S3 buckets despite bucket-level public access blocks.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
|
||||
YIELD node AS internet
|
||||
|
||||
MATCH path_s3 = (aws:AWSAccount {id: $provider_uid})--(s3:S3Bucket)--(t:AWSTag)
|
||||
WHERE toLower(t.key) = toLower($tag_key) AND toLower(t.value) = toLower($tag_value)
|
||||
|
||||
MATCH path_ec2 = (aws)--(ec2:EC2Instance)--(sg:EC2SecurityGroup)--(ipi:IpPermissionInbound)
|
||||
WHERE ec2.exposed_internet = true
|
||||
AND ipi.toport = 22
|
||||
|
||||
MATCH path_role = (r:AWSRole)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
|
||||
WHERE ANY(x IN stmt.resource WHERE x CONTAINS s3.name)
|
||||
AND ANY(x IN stmt.action WHERE toLower(x) =~ 's3:(listbucket|getobject).*')
|
||||
|
||||
MATCH path_assume_role = (ec2)-[p:STS_ASSUMEROLE_ALLOW*1..9]-(r:AWSRole)
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, ec2)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path_s3) + nodes(path_ec2) + nodes(path_role) + nodes(path_assume_role) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path_s3, path_ec2, path_role, path_assume_role, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[
|
||||
AttackPathsQueryParameterDefinition(
|
||||
name="tag_key",
|
||||
label="Tag key",
|
||||
description="Tag key to filter the S3 bucket, e.g. DataClassification.",
|
||||
placeholder="DataClassification",
|
||||
),
|
||||
AttackPathsQueryParameterDefinition(
|
||||
name="tag_value",
|
||||
label="Tag value",
|
||||
description="Tag value to filter the S3 bucket, e.g. Sensitive.",
|
||||
placeholder="Sensitive",
|
||||
),
|
||||
],
|
||||
),
|
||||
# Regular Cartography Attack Paths queries
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-rds-instances",
|
||||
name="Identify provisioned RDS instances",
|
||||
description="List the selected AWS account alongside the RDS instances it owns.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})--(rds:RDSInstance)
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-rds-unencrypted-storage",
|
||||
name="Identify RDS instances without storage encryption",
|
||||
description="Find RDS instances with storage encryption disabled within the selected account.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})--(rds:RDSInstance)
|
||||
WHERE rds.storage_encrypted = false
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-s3-anonymous-access-buckets",
|
||||
name="Identify S3 buckets with anonymous access",
|
||||
description="Find S3 buckets that allow anonymous access within the selected account.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})--(s3:S3Bucket)
|
||||
WHERE s3.anonymous_access = true
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-iam-statements-allow-all-actions",
|
||||
name="Identify IAM statements that allow all actions",
|
||||
description="Find IAM policy statements that allow all actions via '*' within the selected account.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
|
||||
WHERE stmt.effect = 'Allow'
|
||||
AND any(x IN stmt.action WHERE x = '*')
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-iam-statements-allow-delete-policy",
|
||||
name="Identify IAM statements that allow iam:DeletePolicy",
|
||||
description="Find IAM policy statements that allow the iam:DeletePolicy action within the selected account.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
|
||||
WHERE stmt.effect = 'Allow'
|
||||
AND any(x IN stmt.action WHERE x = "iam:DeletePolicy")
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-iam-statements-allow-create-actions",
|
||||
name="Identify IAM statements that allow create actions",
|
||||
description="Find IAM policy statements that allow actions containing 'create' within the selected account.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)--(pol:AWSPolicy)--(stmt:AWSPolicyStatement)
|
||||
WHERE stmt.effect = "Allow"
|
||||
AND any(x IN stmt.action WHERE toLower(x) CONTAINS "create")
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-ec2-instances-internet-exposed",
|
||||
name="Identify internet-exposed EC2 instances",
|
||||
description="Find EC2 instances flagged as exposed to the internet within the selected account.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
|
||||
YIELD node AS internet
|
||||
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})--(ec2:EC2Instance)
|
||||
WHERE ec2.exposed_internet = true
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, ec2)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-security-groups-open-internet-facing",
|
||||
name="Identify internet-facing resources with open security groups",
|
||||
description="Find internet-facing resources associated with security groups that allow inbound access from '0.0.0.0/0'.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
|
||||
YIELD node AS internet
|
||||
|
||||
MATCH path_open = (aws:AWSAccount {id: $provider_uid})-[r0]-(open)
|
||||
MATCH path_sg = (open)-[r1:MEMBER_OF_EC2_SECURITY_GROUP]-(sg:EC2SecurityGroup)
|
||||
MATCH path_ip = (sg)-[r2:MEMBER_OF_EC2_SECURITY_GROUP]-(ipi:IpPermissionInbound)
|
||||
MATCH path_ipi = (ipi)-[r3]-(ir:IpRange)
|
||||
WHERE ir.range = "0.0.0.0/0"
|
||||
OPTIONAL MATCH path_dns = (dns:AWSDNSRecord)-[:DNS_POINTS_TO]->(lb)
|
||||
WHERE open.scheme = 'internet-facing'
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, open)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path_open) + nodes(path_sg) + nodes(path_ip) + nodes(path_ipi) + nodes(path_dns) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path_open, path_sg, path_ip, path_ipi, path_dns, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-classic-elb-internet-exposed",
|
||||
name="Identify internet-exposed Classic Load Balancers",
|
||||
description="Find Classic Load Balancers exposed to the internet along with their listeners.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
|
||||
YIELD node AS internet
|
||||
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})--(elb:LoadBalancer)--(listener:ELBListener)
|
||||
WHERE elb.exposed_internet = true
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, elb)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-elbv2-internet-exposed",
|
||||
name="Identify internet-exposed ELBv2 load balancers",
|
||||
description="Find ELBv2 load balancers exposed to the internet along with their listeners.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
|
||||
YIELD node AS internet
|
||||
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})--(elbv2:LoadBalancerV2)--(listener:ELBV2Listener)
|
||||
WHERE elbv2.exposed_internet = true
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, elbv2)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-public-ip-resource-lookup",
|
||||
name="Identify resources by public IP address",
|
||||
description="Given a public IP address, find the related AWS resource and its adjacent node within the selected account.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
CALL apoc.create.vNode(['Internet'], {id: 'Internet', name: 'Internet'})
|
||||
YIELD node AS internet
|
||||
|
||||
CALL () {
|
||||
MATCH path = (aws:AWSAccount {id: $provider_uid})-[r]-(x:EC2PrivateIp)-[q]-(y)
|
||||
WHERE x.public_ip = $ip
|
||||
RETURN path, x
|
||||
|
||||
UNION MATCH path = (aws:AWSAccount {id: $provider_uid})-[r]-(x:EC2Instance)-[q]-(y)
|
||||
WHERE x.publicipaddress = $ip
|
||||
RETURN path, x
|
||||
|
||||
UNION MATCH path = (aws:AWSAccount {id: $provider_uid})-[r]-(x:NetworkInterface)-[q]-(y)
|
||||
WHERE x.public_ip = $ip
|
||||
RETURN path, x
|
||||
|
||||
UNION MATCH path = (aws:AWSAccount {id: $provider_uid})-[r]-(x:ElasticIPAddress)-[q]-(y)
|
||||
WHERE x.public_ip = $ip
|
||||
RETURN path, x
|
||||
}
|
||||
|
||||
WITH path, x, internet
|
||||
|
||||
CALL apoc.create.vRelationship(internet, 'CAN_ACCESS', {}, x)
|
||||
YIELD rel AS can_access
|
||||
|
||||
UNWIND nodes(path) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr, internet, can_access
|
||||
""",
|
||||
parameters=[
|
||||
AttackPathsQueryParameterDefinition(
|
||||
name="ip",
|
||||
label="IP address",
|
||||
description="Public IP address, e.g. 192.0.2.0.",
|
||||
placeholder="192.0.2.0",
|
||||
),
|
||||
],
|
||||
),
|
||||
# Privilege Escalation Queries (based on pathfinding.cloud research): https://github.com/DataDog/pathfinding.cloud
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-iam-privesc-passrole-ec2",
|
||||
name="Privilege Escalation: iam:PassRole + ec2:RunInstances",
|
||||
description="Detect principals who can launch EC2 instances with privileged IAM roles attached. This allows gaining the permissions of the passed role by accessing the EC2 instance metadata service. This is a new-passrole escalation path (pathfinding.cloud: ec2-001).",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
// Create a single shared virtual EC2 instance node
|
||||
CALL apoc.create.vNode(['EC2Instance'], {
|
||||
id: 'potential-ec2-passrole',
|
||||
name: 'New EC2 Instance',
|
||||
description: 'Attacker-controlled EC2 with privileged role'
|
||||
})
|
||||
YIELD node AS ec2_node
|
||||
|
||||
// Create a single shared virtual escalation outcome node (styled like a finding)
|
||||
CALL apoc.create.vNode(['PrivilegeEscalation'], {
|
||||
id: 'effective-administrator-passrole-ec2',
|
||||
check_title: 'Privilege Escalation',
|
||||
name: 'Effective Administrator',
|
||||
status: 'FAIL',
|
||||
severity: 'critical'
|
||||
})
|
||||
YIELD node AS escalation_outcome
|
||||
|
||||
WITH ec2_node, escalation_outcome
|
||||
|
||||
// Find principals in the account
|
||||
MATCH path_principal = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)
|
||||
|
||||
// Find statements granting iam:PassRole
|
||||
MATCH path_passrole = (principal)--(passrole_policy:AWSPolicy)--(stmt_passrole:AWSPolicyStatement)
|
||||
WHERE stmt_passrole.effect = 'Allow'
|
||||
AND any(action IN stmt_passrole.action WHERE
|
||||
toLower(action) = 'iam:passrole'
|
||||
OR toLower(action) = 'iam:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find statements granting ec2:RunInstances
|
||||
MATCH path_ec2 = (principal)--(ec2_policy:AWSPolicy)--(stmt_ec2:AWSPolicyStatement)
|
||||
WHERE stmt_ec2.effect = 'Allow'
|
||||
AND any(action IN stmt_ec2.action WHERE
|
||||
toLower(action) = 'ec2:runinstances'
|
||||
OR toLower(action) = 'ec2:*'
|
||||
OR action = '*'
|
||||
)
|
||||
|
||||
// Find roles that trust EC2 service (can be passed to EC2)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)
|
||||
WHERE target_role.arn CONTAINS $provider_uid
|
||||
// Check if principal can pass this role
|
||||
AND any(resource IN stmt_passrole.resource WHERE
|
||||
resource = '*'
|
||||
OR target_role.arn CONTAINS resource
|
||||
OR resource CONTAINS target_role.name
|
||||
)
|
||||
|
||||
// Check if target role has elevated permissions (optional, for severity assessment)
|
||||
OPTIONAL MATCH (target_role)--(role_policy:AWSPolicy)--(role_stmt:AWSPolicyStatement)
|
||||
WHERE role_stmt.effect = 'Allow'
|
||||
AND (
|
||||
any(action IN role_stmt.action WHERE action = '*')
|
||||
OR any(action IN role_stmt.action WHERE toLower(action) = 'iam:*')
|
||||
)
|
||||
|
||||
CALL apoc.create.vRelationship(principal, 'CAN_LAUNCH', {
|
||||
via: 'ec2:RunInstances + iam:PassRole'
|
||||
}, ec2_node)
|
||||
YIELD rel AS launch_rel
|
||||
|
||||
CALL apoc.create.vRelationship(ec2_node, 'ASSUMES_ROLE', {}, target_role)
|
||||
YIELD rel AS assumes_rel
|
||||
|
||||
CALL apoc.create.vRelationship(target_role, 'GRANTS_ACCESS', {
|
||||
reference: 'https://pathfinding.cloud/paths/ec2-001'
|
||||
}, escalation_outcome)
|
||||
YIELD rel AS grants_rel
|
||||
|
||||
UNWIND nodes(path_principal) + nodes(path_passrole) + nodes(path_ec2) + nodes(path_target) as n
|
||||
OPTIONAL MATCH (n)-[pfr]-(pf:ProwlerFinding)
|
||||
WHERE pf.status = 'FAIL'
|
||||
|
||||
RETURN path_principal, path_passrole, path_ec2, path_target,
|
||||
ec2_node, escalation_outcome, launch_rel, assumes_rel, grants_rel,
|
||||
collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
AttackPathsQueryDefinition(
|
||||
id="aws-glue-privesc-passrole-dev-endpoint",
|
||||
name="Privilege Escalation: Glue Dev Endpoint with PassRole",
|
||||
description="Detect principals that can escalate privileges by passing a role to a Glue development endpoint. The attacker creates a dev endpoint with an arbitrary role attached, then accesses those credentials through the endpoint.",
|
||||
provider="aws",
|
||||
cypher="""
|
||||
CALL apoc.create.vNode(['PrivilegeEscalation'], {
|
||||
id: 'effective-administrator-glue',
|
||||
check_title: 'Privilege Escalation',
|
||||
name: 'Effective Administrator (Glue)',
|
||||
status: 'FAIL',
|
||||
severity: 'critical'
|
||||
})
|
||||
YIELD node AS escalation_outcome
|
||||
|
||||
WITH escalation_outcome
|
||||
|
||||
// Find principals in the account
|
||||
MATCH path_principal = (aws:AWSAccount {id: $provider_uid})--(principal:AWSPrincipal)
|
||||
|
||||
// Principal can assume roles (up to 2 hops)
|
||||
OPTIONAL MATCH path_assume = (principal)-[:STS_ASSUMEROLE_ALLOW*0..2]->(acting_as:AWSRole)
|
||||
WITH escalation_outcome, principal, path_principal, path_assume,
|
||||
CASE WHEN path_assume IS NULL THEN principal ELSE acting_as END AS effective_principal
|
||||
|
||||
// Find iam:PassRole permission
|
||||
MATCH path_passrole = (effective_principal)--(passrole_policy:AWSPolicy)--(passrole_stmt:AWSPolicyStatement)
|
||||
WHERE passrole_stmt.effect = 'Allow'
|
||||
AND any(action IN passrole_stmt.action WHERE toLower(action) = 'iam:passrole' OR action = '*')
|
||||
|
||||
// Find Glue CreateDevEndpoint permission
|
||||
MATCH (effective_principal)--(glue_policy:AWSPolicy)--(glue_stmt:AWSPolicyStatement)
|
||||
WHERE glue_stmt.effect = 'Allow'
|
||||
AND any(action IN glue_stmt.action WHERE toLower(action) = 'glue:createdevendpoint' OR action = '*' OR toLower(action) = 'glue:*')
|
||||
|
||||
// Find target role with elevated permissions
|
||||
MATCH (aws)--(target_role:AWSRole)--(target_policy:AWSPolicy)--(target_stmt:AWSPolicyStatement)
|
||||
WHERE target_stmt.effect = 'Allow'
|
||||
AND (
|
||||
any(action IN target_stmt.action WHERE action = '*')
|
||||
OR any(action IN target_stmt.action WHERE toLower(action) = 'iam:*')
|
||||
)
|
||||
|
||||
// Deduplicate before creating virtual nodes
|
||||
WITH DISTINCT escalation_outcome, aws, principal, effective_principal, target_role
|
||||
|
||||
// Create virtual Glue endpoint node (one per unique principal->target pair)
|
||||
CALL apoc.create.vNode(['GlueDevEndpoint'], {
|
||||
name: 'New Dev Endpoint',
|
||||
description: 'Glue endpoint with target role attached',
|
||||
id: effective_principal.arn + '->' + target_role.arn
|
||||
})
|
||||
YIELD node AS glue_endpoint
|
||||
|
||||
CALL apoc.create.vRelationship(effective_principal, 'CREATES_ENDPOINT', {
|
||||
permissions: ['iam:PassRole', 'glue:CreateDevEndpoint'],
|
||||
technique: 'new-passrole'
|
||||
}, glue_endpoint)
|
||||
YIELD rel AS create_rel
|
||||
|
||||
CALL apoc.create.vRelationship(glue_endpoint, 'RUNS_AS', {}, target_role)
|
||||
YIELD rel AS runs_rel
|
||||
|
||||
CALL apoc.create.vRelationship(target_role, 'GRANTS_ACCESS', {
|
||||
reference: 'https://pathfinding.cloud/paths/glue-001'
|
||||
}, escalation_outcome)
|
||||
YIELD rel AS grants_rel
|
||||
|
||||
// Re-match paths for visualization
|
||||
MATCH path_principal = (aws)--(principal)
|
||||
MATCH path_target = (aws)--(target_role)
|
||||
|
||||
RETURN path_principal, path_target,
|
||||
glue_endpoint, escalation_outcome, create_rel, runs_rel, grants_rel
|
||||
""",
|
||||
parameters=[],
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
_QUERIES_BY_ID: dict[str, AttackPathsQueryDefinition] = {
|
||||
definition.id: definition
|
||||
for definitions in _QUERY_DEFINITIONS.values()
|
||||
for definition in definitions
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import logging
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RetryableSession:
|
||||
"""
|
||||
Wrapper around `neo4j.Session` that retries `neo4j.exceptions.ServiceUnavailable` errors.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: Callable[[], neo4j.Session],
|
||||
max_retries: int,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._max_retries = max(0, max_retries)
|
||||
self._session = self._session_factory()
|
||||
|
||||
def close(self) -> None:
|
||||
if self._session is not None:
|
||||
self._session.close()
|
||||
self._session = None
|
||||
|
||||
def __enter__(self) -> "RetryableSession":
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self, _: Any, __: Any, ___: Any
|
||||
) -> None: # Unused args: exc_type, exc, exc_tb
|
||||
self.close()
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("run", *args, **kwargs)
|
||||
|
||||
def write_transaction(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("write_transaction", *args, **kwargs)
|
||||
|
||||
def read_transaction(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("read_transaction", *args, **kwargs)
|
||||
|
||||
def execute_write(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("execute_write", *args, **kwargs)
|
||||
|
||||
def execute_read(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return self._call_with_retry("execute_read", *args, **kwargs)
|
||||
|
||||
def __getattr__(self, item: str) -> Any:
|
||||
return getattr(self._session, item)
|
||||
|
||||
def _call_with_retry(self, method_name: str, *args: Any, **kwargs: Any) -> Any:
|
||||
attempt = 0
|
||||
last_exc: Exception | None = None
|
||||
|
||||
while attempt <= self._max_retries:
|
||||
try:
|
||||
method = getattr(self._session, method_name)
|
||||
return method(*args, **kwargs)
|
||||
|
||||
except (
|
||||
BrokenPipeError,
|
||||
ConnectionResetError,
|
||||
neo4j.exceptions.ServiceUnavailable,
|
||||
) as exc: # pragma: no cover - depends on infra
|
||||
last_exc = exc
|
||||
attempt += 1
|
||||
|
||||
if attempt > self._max_retries:
|
||||
raise
|
||||
|
||||
logger.warning(
|
||||
f"Neo4j session {method_name} failed with {type(exc).__name__} ({attempt}/{self._max_retries} attempts). Retrying..."
|
||||
)
|
||||
self._refresh_session()
|
||||
|
||||
raise last_exc if last_exc else RuntimeError("Unexpected retry loop exit")
|
||||
|
||||
def _refresh_session(self) -> None:
|
||||
if self._session is not None:
|
||||
try:
|
||||
self._session.close()
|
||||
except Exception:
|
||||
# Best-effort close; failures just mean we open a new session below
|
||||
pass
|
||||
|
||||
self._session = self._session_factory()
|
||||
@@ -0,0 +1,143 @@
|
||||
import logging
|
||||
|
||||
from typing import Any
|
||||
|
||||
from rest_framework.exceptions import APIException, ValidationError
|
||||
|
||||
from api.attack_paths import database as graph_database, AttackPathsQueryDefinition
|
||||
from api.models import AttackPathsScan
|
||||
from config.custom_logging import BackendLogger
|
||||
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
|
||||
def normalize_run_payload(raw_data):
|
||||
if not isinstance(raw_data, dict): # Let the serializer handle this
|
||||
return raw_data
|
||||
|
||||
if "data" in raw_data and isinstance(raw_data.get("data"), dict):
|
||||
data_section = raw_data.get("data") or {}
|
||||
attributes = data_section.get("attributes") or {}
|
||||
payload = {
|
||||
"id": attributes.get("id", data_section.get("id")),
|
||||
"parameters": attributes.get("parameters"),
|
||||
}
|
||||
|
||||
# Remove `None` parameters to allow defaults downstream
|
||||
if payload.get("parameters") is None:
|
||||
payload.pop("parameters")
|
||||
return payload
|
||||
|
||||
return raw_data
|
||||
|
||||
|
||||
def prepare_query_parameters(
|
||||
definition: AttackPathsQueryDefinition,
|
||||
provided_parameters: dict[str, Any],
|
||||
provider_uid: str,
|
||||
) -> dict[str, Any]:
|
||||
parameters = dict(provided_parameters or {})
|
||||
expected_names = {parameter.name for parameter in definition.parameters}
|
||||
provided_names = set(parameters.keys())
|
||||
|
||||
unexpected = provided_names - expected_names
|
||||
if unexpected:
|
||||
raise ValidationError(
|
||||
{"parameters": f"Unknown parameter(s): {', '.join(sorted(unexpected))}"}
|
||||
)
|
||||
|
||||
missing = expected_names - provided_names
|
||||
if missing:
|
||||
raise ValidationError(
|
||||
{
|
||||
"parameters": f"Missing required parameter(s): {', '.join(sorted(missing))}"
|
||||
}
|
||||
)
|
||||
|
||||
clean_parameters = {
|
||||
"provider_uid": str(provider_uid),
|
||||
}
|
||||
|
||||
for definition_parameter in definition.parameters:
|
||||
raw_value = provided_parameters[definition_parameter.name]
|
||||
|
||||
try:
|
||||
casted_value = definition_parameter.cast(raw_value)
|
||||
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise ValidationError(
|
||||
{
|
||||
"parameters": (
|
||||
f"Invalid value for parameter `{definition_parameter.name}`: {str(exc)}"
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
clean_parameters[definition_parameter.name] = casted_value
|
||||
|
||||
return clean_parameters
|
||||
|
||||
|
||||
def execute_attack_paths_query(
|
||||
attack_paths_scan: AttackPathsScan,
|
||||
definition: AttackPathsQueryDefinition,
|
||||
parameters: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
with graph_database.get_session(attack_paths_scan.graph_database) as session:
|
||||
result = session.run(definition.cypher, parameters)
|
||||
return _serialize_graph(result.graph())
|
||||
|
||||
except graph_database.GraphDatabaseQueryException as exc:
|
||||
logger.error(f"Query failed for Attack Paths query `{definition.id}`: {exc}")
|
||||
raise APIException(
|
||||
"Attack Paths query execution failed due to a database error"
|
||||
)
|
||||
|
||||
|
||||
def _serialize_graph(graph):
|
||||
nodes = []
|
||||
for node in graph.nodes:
|
||||
nodes.append(
|
||||
{
|
||||
"id": node.element_id,
|
||||
"labels": list(node.labels),
|
||||
"properties": _serialize_properties(node._properties),
|
||||
},
|
||||
)
|
||||
|
||||
relationships = []
|
||||
for relationship in graph.relationships:
|
||||
relationships.append(
|
||||
{
|
||||
"id": relationship.element_id,
|
||||
"label": relationship.type,
|
||||
"source": relationship.start_node.element_id,
|
||||
"target": relationship.end_node.element_id,
|
||||
"properties": _serialize_properties(relationship._properties),
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"nodes": nodes,
|
||||
"relationships": relationships,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_properties(properties: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Convert Neo4j property values into JSON-serializable primitives."""
|
||||
|
||||
def _serialize_value(value: Any) -> Any:
|
||||
# Neo4j temporal and spatial values expose `to_native` returning Python primitives
|
||||
if hasattr(value, "to_native") and callable(value.to_native):
|
||||
return _serialize_value(value.to_native())
|
||||
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_serialize_value(item) for item in value]
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {key: _serialize_value(val) for key, val in value.items()}
|
||||
|
||||
return value
|
||||
|
||||
return {key: _serialize_value(val) for key, val in properties.items()}
|
||||
@@ -1,15 +1,99 @@
|
||||
from types import MappingProxyType
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
from api.models import Provider
|
||||
from prowler.config.config import get_available_compliance_frameworks
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.models import CheckMetadata
|
||||
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = {}
|
||||
PROWLER_CHECKS = {}
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS = {}
|
||||
|
||||
|
||||
class LazyComplianceTemplate(Mapping):
|
||||
"""Lazy-load compliance templates per provider on first access."""
|
||||
|
||||
def __init__(self, provider_types: Iterable[str] | None = None) -> None:
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
self._provider_types = tuple(provider_types)
|
||||
self._provider_types_set = set(self._provider_types)
|
||||
self._cache: dict[str, dict] = {}
|
||||
|
||||
def _load_provider(self, provider_type: str) -> dict:
|
||||
if provider_type not in self._provider_types_set:
|
||||
raise KeyError(provider_type)
|
||||
cached = self._cache.get(provider_type)
|
||||
if cached is not None:
|
||||
return cached
|
||||
_ensure_provider_loaded(provider_type)
|
||||
return self._cache[provider_type]
|
||||
|
||||
def __getitem__(self, key: str) -> dict:
|
||||
return self._load_provider(key)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._provider_types)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._provider_types)
|
||||
|
||||
def __contains__(self, key: object) -> bool:
|
||||
return key in self._provider_types_set
|
||||
|
||||
def get(self, key: str, default=None):
|
||||
if key not in self._provider_types_set:
|
||||
return default
|
||||
return self._load_provider(key)
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - debugging helper
|
||||
loaded = ", ".join(sorted(self._cache))
|
||||
return f"{self.__class__.__name__}(loaded=[{loaded}])"
|
||||
|
||||
|
||||
class LazyChecksMapping(Mapping):
|
||||
"""Lazy-load checks mapping per provider on first access."""
|
||||
|
||||
def __init__(self, provider_types: Iterable[str] | None = None) -> None:
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
self._provider_types = tuple(provider_types)
|
||||
self._provider_types_set = set(self._provider_types)
|
||||
self._cache: dict[str, dict] = {}
|
||||
|
||||
def _load_provider(self, provider_type: str) -> dict:
|
||||
if provider_type not in self._provider_types_set:
|
||||
raise KeyError(provider_type)
|
||||
cached = self._cache.get(provider_type)
|
||||
if cached is not None:
|
||||
return cached
|
||||
_ensure_provider_loaded(provider_type)
|
||||
return self._cache[provider_type]
|
||||
|
||||
def __getitem__(self, key: str) -> dict:
|
||||
return self._load_provider(key)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._provider_types)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._provider_types)
|
||||
|
||||
def __contains__(self, key: object) -> bool:
|
||||
return key in self._provider_types_set
|
||||
|
||||
def get(self, key: str, default=None):
|
||||
if key not in self._provider_types_set:
|
||||
return default
|
||||
return self._load_provider(key)
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - debugging helper
|
||||
loaded = ", ".join(sorted(self._cache))
|
||||
return f"{self.__class__.__name__}(loaded=[{loaded}])"
|
||||
|
||||
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = LazyComplianceTemplate()
|
||||
PROWLER_CHECKS = LazyChecksMapping()
|
||||
|
||||
|
||||
def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]:
|
||||
"""
|
||||
Retrieve and cache the list of available compliance frameworks for a specific cloud provider.
|
||||
@@ -70,28 +154,35 @@ def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) ->
|
||||
return Compliance.get_bulk(provider_type)
|
||||
|
||||
|
||||
def load_prowler_compliance():
|
||||
"""
|
||||
Load and initialize the Prowler compliance data and checks for all provider types.
|
||||
|
||||
This function retrieves compliance data for all supported provider types,
|
||||
generates a compliance overview template, and populates the global variables
|
||||
`PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE` and `PROWLER_CHECKS` with read-only mappings
|
||||
of the compliance templates and checks, respectively.
|
||||
"""
|
||||
global PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
|
||||
global PROWLER_CHECKS
|
||||
|
||||
prowler_compliance = {
|
||||
provider_type: get_prowler_provider_compliance(provider_type)
|
||||
for provider_type in Provider.ProviderChoices.values
|
||||
}
|
||||
template = generate_compliance_overview_template(prowler_compliance)
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = MappingProxyType(template)
|
||||
PROWLER_CHECKS = MappingProxyType(load_prowler_checks(prowler_compliance))
|
||||
def _load_provider_assets(provider_type: Provider.ProviderChoices) -> tuple[dict, dict]:
|
||||
prowler_compliance = {provider_type: get_prowler_provider_compliance(provider_type)}
|
||||
template = generate_compliance_overview_template(
|
||||
prowler_compliance, provider_types=[provider_type]
|
||||
)
|
||||
checks = load_prowler_checks(prowler_compliance, provider_types=[provider_type])
|
||||
return template.get(provider_type, {}), checks.get(provider_type, {})
|
||||
|
||||
|
||||
def load_prowler_checks(prowler_compliance):
|
||||
def _ensure_provider_loaded(provider_type: Provider.ProviderChoices) -> None:
|
||||
if (
|
||||
provider_type in PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache
|
||||
and provider_type in PROWLER_CHECKS._cache
|
||||
):
|
||||
return
|
||||
template_cached = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache.get(provider_type)
|
||||
checks_cached = PROWLER_CHECKS._cache.get(provider_type)
|
||||
if template_cached is not None and checks_cached is not None:
|
||||
return
|
||||
template, checks = _load_provider_assets(provider_type)
|
||||
if template_cached is None:
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache[provider_type] = template
|
||||
if checks_cached is None:
|
||||
PROWLER_CHECKS._cache[provider_type] = checks
|
||||
|
||||
|
||||
def load_prowler_checks(
|
||||
prowler_compliance, provider_types: Iterable[str] | None = None
|
||||
):
|
||||
"""
|
||||
Generate a mapping of checks to the compliance frameworks that include them.
|
||||
|
||||
@@ -100,21 +191,25 @@ def load_prowler_checks(prowler_compliance):
|
||||
of compliance names that include that check.
|
||||
|
||||
Args:
|
||||
prowler_compliance (dict): The compliance data for all provider types,
|
||||
prowler_compliance (dict): The compliance data for provider types,
|
||||
as returned by `get_prowler_provider_compliance`.
|
||||
provider_types (Iterable[str] | None): Optional subset of provider types to
|
||||
process. Defaults to all providers.
|
||||
|
||||
Returns:
|
||||
dict: A nested dictionary where the first-level keys are provider types,
|
||||
and the values are dictionaries mapping check IDs to sets of compliance names.
|
||||
"""
|
||||
checks = {}
|
||||
for provider_type in Provider.ProviderChoices.values:
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
for provider_type in provider_types:
|
||||
checks[provider_type] = {
|
||||
check_id: set() for check_id in get_prowler_provider_checks(provider_type)
|
||||
}
|
||||
for compliance_name, compliance_data in prowler_compliance[
|
||||
provider_type
|
||||
].items():
|
||||
for compliance_name, compliance_data in prowler_compliance.get(
|
||||
provider_type, {}
|
||||
).items():
|
||||
for requirement in compliance_data.Requirements:
|
||||
for check in requirement.Checks:
|
||||
try:
|
||||
@@ -163,7 +258,9 @@ def generate_scan_compliance(
|
||||
] += 1
|
||||
|
||||
|
||||
def generate_compliance_overview_template(prowler_compliance: dict):
|
||||
def generate_compliance_overview_template(
|
||||
prowler_compliance: dict, provider_types: Iterable[str] | None = None
|
||||
):
|
||||
"""
|
||||
Generate a compliance overview template for all provider types.
|
||||
|
||||
@@ -173,17 +270,21 @@ def generate_compliance_overview_template(prowler_compliance: dict):
|
||||
counts for requirements status.
|
||||
|
||||
Args:
|
||||
prowler_compliance (dict): The compliance data for all provider types,
|
||||
prowler_compliance (dict): The compliance data for provider types,
|
||||
as returned by `get_prowler_provider_compliance`.
|
||||
provider_types (Iterable[str] | None): Optional subset of provider types to
|
||||
process. Defaults to all providers.
|
||||
|
||||
Returns:
|
||||
dict: A nested dictionary representing the compliance overview template,
|
||||
structured by provider type and compliance framework.
|
||||
"""
|
||||
template = {}
|
||||
for provider_type in Provider.ProviderChoices.values:
|
||||
if provider_types is None:
|
||||
provider_types = Provider.ProviderChoices.values
|
||||
for provider_type in provider_types:
|
||||
provider_compliance = template.setdefault(provider_type, {})
|
||||
compliance_data_dict = prowler_compliance[provider_type]
|
||||
compliance_data_dict = prowler_compliance.get(provider_type, {})
|
||||
|
||||
for compliance_name, compliance_data in compliance_data_dict.items():
|
||||
compliance_requirements = {}
|
||||
|
||||
@@ -29,6 +29,7 @@ from api.models import (
|
||||
Finding,
|
||||
Integration,
|
||||
Invitation,
|
||||
AttackPathsScan,
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseProviderModels,
|
||||
Membership,
|
||||
@@ -37,12 +38,15 @@ from api.models import (
|
||||
PermissionChoices,
|
||||
Processor,
|
||||
Provider,
|
||||
ProviderComplianceScore,
|
||||
ProviderGroup,
|
||||
ProviderSecret,
|
||||
Resource,
|
||||
ResourceTag,
|
||||
Role,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
SeverityChoices,
|
||||
StateChoices,
|
||||
@@ -91,10 +95,62 @@ class ChoiceInFilter(BaseInFilter, ChoiceFilter):
|
||||
pass
|
||||
|
||||
|
||||
class BaseProviderFilter(FilterSet):
|
||||
"""
|
||||
Abstract base filter for models with direct FK to Provider.
|
||||
|
||||
Provides standard provider_id and provider_type filters.
|
||||
Subclasses must define Meta.model.
|
||||
"""
|
||||
|
||||
provider_id = UUIDFilter(field_name="provider__id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="provider__id", lookup_expr="in")
|
||||
provider_type = ChoiceFilter(
|
||||
field_name="provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_type__in = ChoiceInFilter(
|
||||
field_name="provider__provider",
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
fields = {}
|
||||
|
||||
|
||||
class BaseScanProviderFilter(FilterSet):
|
||||
"""
|
||||
Abstract base filter for models with FK to Scan (and Scan has FK to Provider).
|
||||
|
||||
Provides standard provider_id and provider_type filters via scan relationship.
|
||||
Subclasses must define Meta.model.
|
||||
"""
|
||||
|
||||
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
|
||||
provider_type = ChoiceFilter(
|
||||
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_type__in = ChoiceInFilter(
|
||||
field_name="scan__provider__provider",
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
fields = {}
|
||||
|
||||
|
||||
class CommonFindingFilters(FilterSet):
|
||||
# We filter providers from the scan in findings
|
||||
# Both 'provider' and 'provider_id' parameters are supported for API consistency
|
||||
# Frontend uses 'provider_id' uniformly across all endpoints
|
||||
provider = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
|
||||
provider__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
|
||||
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
|
||||
provider_type = ChoiceFilter(
|
||||
choices=Provider.ProviderChoices.choices, field_name="scan__provider__provider"
|
||||
)
|
||||
@@ -157,6 +213,12 @@ class CommonFindingFilters(FilterSet):
|
||||
field_name="resources__type", lookup_expr="icontains"
|
||||
)
|
||||
|
||||
category = CharFilter(method="filter_category")
|
||||
category__in = CharInFilter(field_name="categories", lookup_expr="overlap")
|
||||
|
||||
resource_groups = CharFilter(field_name="resource_groups", lookup_expr="exact")
|
||||
resource_groups__in = CharInFilter(field_name="resource_groups", lookup_expr="in")
|
||||
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# resource_tag_key = CharFilter(field_name="resources__tags__key")
|
||||
# resource_tag_key__in = CharInFilter(
|
||||
@@ -188,6 +250,9 @@ class CommonFindingFilters(FilterSet):
|
||||
def filter_resource_type(self, queryset, name, value):
|
||||
return queryset.filter(resource_types__contains=[value])
|
||||
|
||||
def filter_category(self, queryset, name, value):
|
||||
return queryset.filter(categories__contains=[value])
|
||||
|
||||
def filter_resource_tag(self, queryset, name, value):
|
||||
overall_query = Q()
|
||||
for key_value_pair in value:
|
||||
@@ -332,6 +397,23 @@ class ScanFilter(ProviderRelationshipFilterSet):
|
||||
}
|
||||
|
||||
|
||||
class AttackPathsScanFilter(ProviderRelationshipFilterSet):
|
||||
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
|
||||
completed_at = DateFilter(field_name="completed_at", lookup_expr="date")
|
||||
started_at = DateFilter(field_name="started_at", lookup_expr="date")
|
||||
state = ChoiceFilter(choices=StateChoices.choices)
|
||||
state__in = ChoiceInFilter(
|
||||
field_name="state", choices=StateChoices.choices, lookup_expr="in"
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = AttackPathsScan
|
||||
fields = {
|
||||
"provider": ["exact", "in"],
|
||||
"scan": ["exact", "in"],
|
||||
}
|
||||
|
||||
|
||||
class TaskFilter(FilterSet):
|
||||
name = CharFilter(field_name="task_runner_task__task_name", lookup_expr="exact")
|
||||
name__icontains = CharFilter(
|
||||
@@ -379,6 +461,8 @@ class ResourceFilter(ProviderRelationshipFilterSet):
|
||||
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
|
||||
scan = UUIDFilter(field_name="provider__scan", lookup_expr="exact")
|
||||
scan__in = UUIDInFilter(field_name="provider__scan", lookup_expr="in")
|
||||
groups = CharFilter(method="filter_groups")
|
||||
groups__in = CharInFilter(field_name="groups", lookup_expr="overlap")
|
||||
|
||||
class Meta:
|
||||
model = Resource
|
||||
@@ -393,6 +477,9 @@ class ResourceFilter(ProviderRelationshipFilterSet):
|
||||
"updated_at": ["gte", "lte"],
|
||||
}
|
||||
|
||||
def filter_groups(self, queryset, name, value):
|
||||
return queryset.filter(groups__contains=[value])
|
||||
|
||||
def filter_queryset(self, queryset):
|
||||
if not (self.data.get("scan") or self.data.get("scan__in")) and not (
|
||||
self.data.get("updated_at")
|
||||
@@ -457,6 +544,8 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
|
||||
tag_value = CharFilter(method="filter_tag_value")
|
||||
tag = CharFilter(method="filter_tag")
|
||||
tags = CharFilter(method="filter_tag")
|
||||
groups = CharFilter(method="filter_groups")
|
||||
groups__in = CharInFilter(field_name="groups", lookup_expr="overlap")
|
||||
|
||||
class Meta:
|
||||
model = Resource
|
||||
@@ -469,6 +558,9 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
|
||||
"type": ["exact", "icontains", "in"],
|
||||
}
|
||||
|
||||
def filter_groups(self, queryset, name, value):
|
||||
return queryset.filter(groups__contains=[value])
|
||||
|
||||
def filter_tag_key(self, queryset, name, value):
|
||||
return queryset.filter(Q(tags__key=value) | Q(tags__key__icontains=value))
|
||||
|
||||
@@ -762,7 +854,7 @@ class RoleFilter(FilterSet):
|
||||
|
||||
class ComplianceOverviewFilter(FilterSet):
|
||||
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
|
||||
scan_id = UUIDFilter(field_name="scan_id")
|
||||
scan_id = UUIDFilter(field_name="scan_id", required=True)
|
||||
region = CharFilter(field_name="region")
|
||||
|
||||
class Meta:
|
||||
@@ -1079,9 +1171,25 @@ class ThreatScoreSnapshotFilter(FilterSet):
|
||||
}
|
||||
|
||||
|
||||
class AttackSurfaceOverviewFilter(FilterSet):
|
||||
class AttackSurfaceOverviewFilter(BaseScanProviderFilter):
|
||||
"""Filter for attack surface overview aggregations by provider."""
|
||||
|
||||
class Meta(BaseScanProviderFilter.Meta):
|
||||
model = AttackSurfaceOverview
|
||||
|
||||
|
||||
class CategoryOverviewFilter(BaseScanProviderFilter):
|
||||
"""Filter for category overview aggregations by provider."""
|
||||
|
||||
category = CharFilter(field_name="category", lookup_expr="exact")
|
||||
category__in = CharInFilter(field_name="category", lookup_expr="in")
|
||||
|
||||
class Meta(BaseScanProviderFilter.Meta):
|
||||
model = ScanCategorySummary
|
||||
fields = {}
|
||||
|
||||
|
||||
class ResourceGroupOverviewFilter(FilterSet):
|
||||
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
|
||||
provider_type = ChoiceFilter(
|
||||
@@ -1092,7 +1200,16 @@ class AttackSurfaceOverviewFilter(FilterSet):
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
resource_group = CharFilter(field_name="resource_group", lookup_expr="exact")
|
||||
resource_group__in = CharInFilter(field_name="resource_group", lookup_expr="in")
|
||||
|
||||
class Meta:
|
||||
model = AttackSurfaceOverview
|
||||
model = ScanGroupSummary
|
||||
fields = {}
|
||||
|
||||
|
||||
class ComplianceWatchlistFilter(BaseProviderFilter):
|
||||
"""Filter for compliance watchlist overview by provider."""
|
||||
|
||||
class Meta(BaseProviderFilter.Meta):
|
||||
model = ProviderComplianceScore
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
[
|
||||
{
|
||||
"model": "api.attackpathsscan",
|
||||
"pk": "a7f0f6de-6f8e-4b3a-8cbe-3f6dd9012345",
|
||||
"fields": {
|
||||
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
|
||||
"provider": "b85601a8-4b45-4194-8135-03fb980ef428",
|
||||
"scan": "01920573-aa9c-73c9-bcda-f2e35c9b19d2",
|
||||
"state": "completed",
|
||||
"progress": 100,
|
||||
"update_tag": 1693586667,
|
||||
"graph_database": "db-a7f0f6de-6f8e-4b3a-8cbe-3f6dd9012345",
|
||||
"is_graph_database_deleted": false,
|
||||
"task": null,
|
||||
"inserted_at": "2024-09-01T17:24:37Z",
|
||||
"updated_at": "2024-09-01T17:44:37Z",
|
||||
"started_at": "2024-09-01T17:34:37Z",
|
||||
"completed_at": "2024-09-01T17:44:37Z",
|
||||
"duration": 269,
|
||||
"ingestion_exceptions": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"model": "api.attackpathsscan",
|
||||
"pk": "4a2fb2af-8a60-4d7d-9cae-4ca65e098765",
|
||||
"fields": {
|
||||
"tenant": "12646005-9067-4d2a-a098-8bb378604362",
|
||||
"provider": "15fce1fa-ecaa-433f-a9dc-62553f3a2555",
|
||||
"scan": "01929f3b-ed2e-7623-ad63-7c37cd37828f",
|
||||
"state": "executing",
|
||||
"progress": 48,
|
||||
"update_tag": 1697625000,
|
||||
"graph_database": "db-4a2fb2af-8a60-4d7d-9cae-4ca65e098765",
|
||||
"is_graph_database_deleted": false,
|
||||
"task": null,
|
||||
"inserted_at": "2024-10-18T10:55:57Z",
|
||||
"updated_at": "2024-10-18T10:56:15Z",
|
||||
"started_at": "2024-10-18T10:56:05Z"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Generated by Django 5.1.14 on 2025-12-10
|
||||
|
||||
from django.db import migrations
|
||||
from tasks.tasks import backfill_daily_severity_summaries_task
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.rls import Tenant
|
||||
|
||||
|
||||
def trigger_backfill_task(apps, schema_editor):
|
||||
"""
|
||||
Trigger the backfill task for all tenants.
|
||||
|
||||
This dispatches backfill_daily_severity_summaries_task for each tenant
|
||||
in the system to populate DailySeveritySummary records from historical scans.
|
||||
"""
|
||||
tenant_ids = Tenant.objects.using(MainRouter.admin_db).values_list("id", flat=True)
|
||||
|
||||
for tenant_id in tenant_ids:
|
||||
backfill_daily_severity_summaries_task.delay(tenant_id=str(tenant_id), days=90)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0061_daily_severity_summary"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(trigger_backfill_task, migrations.RunPython.noop),
|
||||
]
|
||||
@@ -0,0 +1,111 @@
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.db_utils
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0062_backfill_daily_severity_summaries"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="ScanCategorySummary",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to="api.tenant",
|
||||
),
|
||||
),
|
||||
(
|
||||
"inserted_at",
|
||||
models.DateTimeField(auto_now_add=True),
|
||||
),
|
||||
(
|
||||
"scan",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="category_summaries",
|
||||
related_query_name="category_summary",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
(
|
||||
"category",
|
||||
models.CharField(max_length=100),
|
||||
),
|
||||
(
|
||||
"severity",
|
||||
api.db_utils.SeverityEnumField(
|
||||
choices=[
|
||||
("critical", "Critical"),
|
||||
("high", "High"),
|
||||
("medium", "Medium"),
|
||||
("low", "Low"),
|
||||
("informational", "Informational"),
|
||||
],
|
||||
),
|
||||
),
|
||||
(
|
||||
"total_findings",
|
||||
models.IntegerField(
|
||||
default=0, help_text="Non-muted findings (PASS + FAIL)"
|
||||
),
|
||||
),
|
||||
(
|
||||
"failed_findings",
|
||||
models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL findings (subset of total_findings)",
|
||||
),
|
||||
),
|
||||
(
|
||||
"new_failed_findings",
|
||||
models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "scan_category_summaries",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="scancategorysummary",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "scan"], name="scs_tenant_scan_idx"
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scancategorysummary",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "category", "severity"),
|
||||
name="unique_category_severity_per_scan",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scancategorysummary",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_scancategorysummary",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
import django.contrib.postgres.fields
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0063_scan_category_summary"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="finding",
|
||||
name="categories",
|
||||
field=django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=100),
|
||||
blank=True,
|
||||
null=True,
|
||||
size=None,
|
||||
help_text="Categories from check metadata for efficient filtering",
|
||||
),
|
||||
),
|
||||
]
|
||||
+1
-1
@@ -7,7 +7,7 @@ import api.db_utils
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0061_daily_severity_summary"),
|
||||
("api", "0064_finding_categories"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
@@ -0,0 +1,94 @@
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.db_utils
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0065_alibabacloud_provider"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="ProviderComplianceScore",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("compliance_id", models.TextField()),
|
||||
("requirement_id", models.TextField()),
|
||||
(
|
||||
"requirement_status",
|
||||
api.db_utils.StatusEnumField(
|
||||
choices=[
|
||||
("FAIL", "Fail"),
|
||||
("PASS", "Pass"),
|
||||
("MANUAL", "Manual"),
|
||||
]
|
||||
),
|
||||
),
|
||||
("scan_completed_at", models.DateTimeField()),
|
||||
(
|
||||
"provider",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="compliance_scores",
|
||||
related_query_name="compliance_score",
|
||||
to="api.provider",
|
||||
),
|
||||
),
|
||||
(
|
||||
"scan",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="compliance_scores",
|
||||
related_query_name="compliance_score",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to="api.tenant",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "provider_compliance_scores",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="providercompliancescore",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "provider_id", "compliance_id", "requirement_id"),
|
||||
name="unique_provider_compliance_req",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="providercompliancescore",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_providercompliancescore",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="providercompliancescore",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "provider_id", "compliance_id"],
|
||||
name="pcs_tenant_prov_comp_idx",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,61 @@
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0066_provider_compliance_score"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="TenantComplianceSummary",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("compliance_id", models.TextField()),
|
||||
("requirements_passed", models.IntegerField(default=0)),
|
||||
("requirements_failed", models.IntegerField(default=0)),
|
||||
("requirements_manual", models.IntegerField(default=0)),
|
||||
("total_requirements", models.IntegerField(default=0)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to="api.tenant",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "tenant_compliance_summaries",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="tenantcompliancesummary",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "compliance_id"),
|
||||
name="unique_tenant_compliance_summary",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="tenantcompliancesummary",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_tenantcompliancesummary",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,126 @@
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.db_utils
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0067_tenant_compliance_summary"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="finding",
|
||||
name="resource_groups",
|
||||
field=models.TextField(
|
||||
blank=True,
|
||||
help_text="Resource group from check metadata for efficient filtering",
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="ScanGroupSummary",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to="api.tenant",
|
||||
),
|
||||
),
|
||||
(
|
||||
"inserted_at",
|
||||
models.DateTimeField(auto_now_add=True),
|
||||
),
|
||||
(
|
||||
"scan",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="resource_group_summaries",
|
||||
related_query_name="resource_group_summary",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
(
|
||||
"resource_group",
|
||||
models.CharField(max_length=50),
|
||||
),
|
||||
(
|
||||
"severity",
|
||||
api.db_utils.SeverityEnumField(
|
||||
choices=[
|
||||
("critical", "Critical"),
|
||||
("high", "High"),
|
||||
("medium", "Medium"),
|
||||
("low", "Low"),
|
||||
("informational", "Informational"),
|
||||
],
|
||||
),
|
||||
),
|
||||
(
|
||||
"total_findings",
|
||||
models.IntegerField(
|
||||
default=0, help_text="Non-muted findings (PASS + FAIL)"
|
||||
),
|
||||
),
|
||||
(
|
||||
"failed_findings",
|
||||
models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL findings (subset of total_findings)",
|
||||
),
|
||||
),
|
||||
(
|
||||
"new_failed_findings",
|
||||
models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
|
||||
),
|
||||
),
|
||||
(
|
||||
"resources_count",
|
||||
models.IntegerField(
|
||||
default=0, help_text="Count of distinct resource_uid values"
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "scan_resource_group_summaries",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="scangroupsummary",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "scan"], name="srgs_tenant_scan_idx"
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scangroupsummary",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "resource_group", "severity"),
|
||||
name="unique_resource_group_severity_per_scan",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scangroupsummary",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_scangroupsummary",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,21 @@
|
||||
from django.contrib.postgres.fields import ArrayField
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0068_finding_resource_group_scangroupsummary"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="resource",
|
||||
name="groups",
|
||||
field=ArrayField(
|
||||
models.CharField(max_length=100),
|
||||
blank=True,
|
||||
help_text="Groups for categorization (e.g., compute, storage, IAM)",
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,154 @@
|
||||
# Generated by Django 5.1.13 on 2025-11-06 16:20
|
||||
|
||||
import django.db.models.deletion
|
||||
|
||||
from django.db import migrations, models
|
||||
from uuid6 import uuid7
|
||||
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0069_resource_resource_group"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="AttackPathsScan",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid7,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("inserted_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"state",
|
||||
api.db_utils.StateEnumField(
|
||||
choices=[
|
||||
("available", "Available"),
|
||||
("scheduled", "Scheduled"),
|
||||
("executing", "Executing"),
|
||||
("completed", "Completed"),
|
||||
("failed", "Failed"),
|
||||
("cancelled", "Cancelled"),
|
||||
],
|
||||
default="available",
|
||||
),
|
||||
),
|
||||
("progress", models.IntegerField(default=0)),
|
||||
("started_at", models.DateTimeField(blank=True, null=True)),
|
||||
("completed_at", models.DateTimeField(blank=True, null=True)),
|
||||
(
|
||||
"duration",
|
||||
models.IntegerField(
|
||||
blank=True, help_text="Duration in seconds", null=True
|
||||
),
|
||||
),
|
||||
(
|
||||
"update_tag",
|
||||
models.BigIntegerField(
|
||||
blank=True,
|
||||
help_text="Cartography update tag (epoch)",
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
(
|
||||
"graph_database",
|
||||
models.CharField(blank=True, max_length=63, null=True),
|
||||
),
|
||||
(
|
||||
"is_graph_database_deleted",
|
||||
models.BooleanField(default=False),
|
||||
),
|
||||
(
|
||||
"ingestion_exceptions",
|
||||
models.JSONField(blank=True, default=dict, null=True),
|
||||
),
|
||||
(
|
||||
"provider",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
to="api.provider",
|
||||
),
|
||||
),
|
||||
(
|
||||
"scan",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
(
|
||||
"task",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
to="api.task",
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "attack_paths_scans",
|
||||
"abstract": False,
|
||||
"indexes": [
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-inserted_at"],
|
||||
name="aps_prov_ins_desc_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "state", "-inserted_at"],
|
||||
name="aps_state_ins_desc_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "scan_id"],
|
||||
name="aps_scan_lookup_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id"],
|
||||
name="aps_active_graph_idx",
|
||||
include=["graph_database", "id"],
|
||||
condition=models.Q(("is_graph_database_deleted", False)),
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-completed_at"],
|
||||
name="aps_completed_graph_idx",
|
||||
include=["graph_database", "id"],
|
||||
condition=models.Q(
|
||||
("state", "completed"),
|
||||
("is_graph_database_deleted", False),
|
||||
),
|
||||
),
|
||||
],
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="attackpathsscan",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_attackpathsscan",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -626,6 +626,101 @@ class Scan(RowLevelSecurityProtectedModel):
|
||||
resource_name = "scans"
|
||||
|
||||
|
||||
class AttackPathsScan(RowLevelSecurityProtectedModel):
|
||||
objects = ActiveProviderManager()
|
||||
all_objects = models.Manager()
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid7, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
updated_at = models.DateTimeField(auto_now=True, editable=False)
|
||||
|
||||
state = StateEnumField(choices=StateChoices.choices, default=StateChoices.AVAILABLE)
|
||||
progress = models.IntegerField(default=0)
|
||||
|
||||
# Timing
|
||||
started_at = models.DateTimeField(null=True, blank=True)
|
||||
completed_at = models.DateTimeField(null=True, blank=True)
|
||||
duration = models.IntegerField(
|
||||
null=True, blank=True, help_text="Duration in seconds"
|
||||
)
|
||||
|
||||
# Relationship to the provider and optional prowler Scan and celery Task
|
||||
provider = models.ForeignKey(
|
||||
"Provider",
|
||||
on_delete=models.CASCADE,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
)
|
||||
scan = models.ForeignKey(
|
||||
"Scan",
|
||||
on_delete=models.SET_NULL,
|
||||
null=True,
|
||||
blank=True,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
)
|
||||
task = models.ForeignKey(
|
||||
"Task",
|
||||
on_delete=models.SET_NULL,
|
||||
null=True,
|
||||
blank=True,
|
||||
related_name="attack_paths_scans",
|
||||
related_query_name="attack_paths_scan",
|
||||
)
|
||||
|
||||
# Cartography specific metadata
|
||||
update_tag = models.BigIntegerField(
|
||||
null=True, blank=True, help_text="Cartography update tag (epoch)"
|
||||
)
|
||||
graph_database = models.CharField(max_length=63, null=True, blank=True)
|
||||
is_graph_database_deleted = models.BooleanField(default=False)
|
||||
ingestion_exceptions = models.JSONField(default=dict, null=True, blank=True)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "attack_paths_scans"
|
||||
|
||||
constraints = [
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
indexes = [
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-inserted_at"],
|
||||
name="aps_prov_ins_desc_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "state", "-inserted_at"],
|
||||
name="aps_state_ins_desc_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "scan_id"],
|
||||
name="aps_scan_lookup_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id"],
|
||||
name="aps_active_graph_idx",
|
||||
include=["graph_database", "id"],
|
||||
condition=Q(is_graph_database_deleted=False),
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "-completed_at"],
|
||||
name="aps_completed_graph_idx",
|
||||
include=["graph_database", "id"],
|
||||
condition=Q(
|
||||
state=StateChoices.COMPLETED,
|
||||
is_graph_database_deleted=False,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-scans"
|
||||
|
||||
|
||||
class ResourceTag(RowLevelSecurityProtectedModel):
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
@@ -704,6 +799,12 @@ class Resource(RowLevelSecurityProtectedModel):
|
||||
metadata = models.TextField(blank=True, null=True)
|
||||
details = models.TextField(blank=True, null=True)
|
||||
partition = models.TextField(blank=True, null=True)
|
||||
groups = ArrayField(
|
||||
models.CharField(max_length=100),
|
||||
blank=True,
|
||||
null=True,
|
||||
help_text="Groups for categorization (e.g., compute, storage, IAM)",
|
||||
)
|
||||
|
||||
failed_findings_count = models.IntegerField(default=0)
|
||||
|
||||
@@ -726,14 +827,19 @@ class Resource(RowLevelSecurityProtectedModel):
|
||||
self.clear_tags()
|
||||
return
|
||||
|
||||
# Add new relationships with the tenant_id field
|
||||
# Add new relationships with the tenant_id field; avoid touching the
|
||||
# Resource row unless a mapping is actually created to prevent noisy
|
||||
# updates during scans.
|
||||
mapping_created = False
|
||||
for tag in tags:
|
||||
ResourceTagMapping.objects.update_or_create(
|
||||
_, created = ResourceTagMapping.objects.update_or_create(
|
||||
tag=tag, resource=self, tenant_id=self.tenant_id
|
||||
)
|
||||
mapping_created = mapping_created or created
|
||||
|
||||
# Save the instance
|
||||
self.save()
|
||||
if mapping_created:
|
||||
# Only bump updated_at when the tag set truly changed
|
||||
self.save(update_fields=["updated_at"])
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "resources"
|
||||
@@ -878,6 +984,19 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
|
||||
null=True,
|
||||
)
|
||||
|
||||
# Check metadata denormalization
|
||||
categories = ArrayField(
|
||||
models.CharField(max_length=100),
|
||||
blank=True,
|
||||
null=True,
|
||||
help_text="Categories from check metadata for efficient filtering",
|
||||
)
|
||||
resource_groups = models.TextField(
|
||||
blank=True,
|
||||
null=True,
|
||||
help_text="Resource group from check metadata for efficient filtering",
|
||||
)
|
||||
|
||||
# Relationships
|
||||
scan = models.ForeignKey(to=Scan, related_name="findings", on_delete=models.CASCADE)
|
||||
|
||||
@@ -1961,6 +2080,125 @@ class ResourceScanSummary(RowLevelSecurityProtectedModel):
|
||||
]
|
||||
|
||||
|
||||
class ScanCategorySummary(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Pre-aggregated category metrics per scan by severity.
|
||||
|
||||
Stores one row per (category, severity) combination per scan for efficient
|
||||
overview queries. Categories come from check_metadata.categories.
|
||||
|
||||
Count relationships (each is a subset of the previous):
|
||||
- total_findings >= failed_findings >= new_failed_findings
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
|
||||
scan = models.ForeignKey(
|
||||
Scan,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="category_summaries",
|
||||
related_query_name="category_summary",
|
||||
)
|
||||
|
||||
category = models.CharField(max_length=100)
|
||||
severity = SeverityEnumField(choices=SeverityChoices)
|
||||
|
||||
total_findings = models.IntegerField(
|
||||
default=0, help_text="Non-muted findings (PASS + FAIL)"
|
||||
)
|
||||
failed_findings = models.IntegerField(
|
||||
default=0, help_text="Non-muted FAIL findings (subset of total_findings)"
|
||||
)
|
||||
new_failed_findings = models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
|
||||
)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "scan_category_summaries"
|
||||
|
||||
indexes = [
|
||||
models.Index(fields=["tenant_id", "scan"], name="scs_tenant_scan_idx"),
|
||||
]
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "category", "severity"),
|
||||
name="unique_category_severity_per_scan",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "scan-category-summaries"
|
||||
|
||||
|
||||
class ScanGroupSummary(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Pre-aggregated resource group metrics per scan by severity.
|
||||
|
||||
Stores one row per (resource_group, severity) combination per scan for efficient
|
||||
overview queries. Resource groups come from check_metadata.Group.
|
||||
|
||||
Count relationships (each is a subset of the previous):
|
||||
- total_findings >= failed_findings >= new_failed_findings
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
|
||||
scan = models.ForeignKey(
|
||||
Scan,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="resource_group_summaries",
|
||||
related_query_name="resource_group_summary",
|
||||
)
|
||||
|
||||
resource_group = models.CharField(max_length=50)
|
||||
severity = SeverityEnumField(choices=SeverityChoices)
|
||||
|
||||
total_findings = models.IntegerField(
|
||||
default=0, help_text="Non-muted findings (PASS + FAIL)"
|
||||
)
|
||||
failed_findings = models.IntegerField(
|
||||
default=0, help_text="Non-muted FAIL findings (subset of total_findings)"
|
||||
)
|
||||
new_failed_findings = models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
|
||||
)
|
||||
resources_count = models.IntegerField(
|
||||
default=0, help_text="Count of distinct resource_uid values"
|
||||
)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "scan_resource_group_summaries"
|
||||
|
||||
indexes = [
|
||||
models.Index(fields=["tenant_id", "scan"], name="srgs_tenant_scan_idx"),
|
||||
]
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "resource_group", "severity"),
|
||||
name="unique_resource_group_severity_per_scan",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "scan-resource-group-summaries"
|
||||
|
||||
|
||||
class LighthouseConfiguration(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Stores configuration and API keys for LLM services.
|
||||
@@ -2534,3 +2772,92 @@ class AttackSurfaceOverview(RowLevelSecurityProtectedModel):
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-surface-overviews"
|
||||
|
||||
|
||||
class ProviderComplianceScore(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Compliance requirement status from latest completed scan per provider.
|
||||
|
||||
Used for efficient compliance watchlist queries with FAIL-dominant aggregation
|
||||
across multiple providers. Updated via atomic upsert after each scan completion.
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
|
||||
scan = models.ForeignKey(
|
||||
Scan,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="compliance_scores",
|
||||
related_query_name="compliance_score",
|
||||
)
|
||||
|
||||
provider = models.ForeignKey(
|
||||
Provider,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="compliance_scores",
|
||||
related_query_name="compliance_score",
|
||||
)
|
||||
|
||||
compliance_id = models.TextField()
|
||||
requirement_id = models.TextField()
|
||||
requirement_status = StatusEnumField(choices=StatusChoices)
|
||||
|
||||
scan_completed_at = models.DateTimeField()
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "provider_compliance_scores"
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "provider_id", "compliance_id", "requirement_id"),
|
||||
name="unique_provider_compliance_req",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
indexes = [
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id", "compliance_id"],
|
||||
name="pcs_tenant_prov_comp_idx",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class TenantComplianceSummary(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Pre-aggregated compliance counts per tenant with FAIL-dominant logic applied.
|
||||
|
||||
One row per (tenant, compliance_id). Used for fast watchlist queries when
|
||||
no provider filter is applied. Recalculated after each scan by aggregating
|
||||
across all providers with FAIL-dominant logic at requirement level.
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
|
||||
compliance_id = models.TextField()
|
||||
|
||||
requirements_passed = models.IntegerField(default=0)
|
||||
requirements_failed = models.IntegerField(default=0)
|
||||
requirements_manual = models.IntegerField(default=0)
|
||||
total_requirements = models.IntegerField(default=0)
|
||||
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "tenant_compliance_summaries"
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "compliance_id"),
|
||||
name="unique_tenant_compliance_summary",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
+2046
-94
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,13 @@
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from django.conf import settings
|
||||
|
||||
import api
|
||||
import api.apps as api_apps_module
|
||||
from api.apps import (
|
||||
ApiConfig,
|
||||
@@ -150,3 +153,82 @@ def test_ensure_crypto_keys_skips_when_env_vars(monkeypatch, tmp_path):
|
||||
|
||||
# Assert: orchestrator did not trigger generation when env present
|
||||
assert called["ensure"] is False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def stub_api_modules():
|
||||
"""Provide dummy modules imported during ApiConfig.ready()."""
|
||||
created = []
|
||||
for name in ("api.schema_extensions", "api.signals"):
|
||||
if name not in sys.modules:
|
||||
sys.modules[name] = types.ModuleType(name)
|
||||
created.append(name)
|
||||
|
||||
yield
|
||||
|
||||
for name in created:
|
||||
sys.modules.pop(name, None)
|
||||
|
||||
|
||||
def _set_argv(monkeypatch, argv):
|
||||
monkeypatch.setattr(sys, "argv", argv, raising=False)
|
||||
|
||||
|
||||
def _set_testing(monkeypatch, value):
|
||||
monkeypatch.setattr(settings, "TESTING", value, raising=False)
|
||||
|
||||
|
||||
def _make_app():
|
||||
return ApiConfig("api", api)
|
||||
|
||||
|
||||
def test_ready_initializes_driver_for_api_process(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch(
|
||||
"api.attack_paths.database.init_driver"
|
||||
) as init_driver:
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_called_once()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_celery(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["celery", "-A", "api"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch(
|
||||
"api.attack_paths.database.init_driver"
|
||||
) as init_driver:
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_manage_py_skip_command(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["manage.py", "migrate"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch(
|
||||
"api.attack_paths.database.init_driver"
|
||||
) as init_driver:
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_when_testing(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, True)
|
||||
|
||||
with patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None), patch(
|
||||
"api.attack_paths.database.init_driver"
|
||||
) as init_driver:
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from rest_framework.exceptions import APIException, ValidationError
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.attack_paths import views_helpers
|
||||
|
||||
|
||||
def test_normalize_run_payload_extracts_attributes_section():
|
||||
payload = {
|
||||
"data": {
|
||||
"id": "ignored",
|
||||
"attributes": {
|
||||
"id": "aws-rds",
|
||||
"parameters": {"ip": "192.0.2.0"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
result = views_helpers.normalize_run_payload(payload)
|
||||
|
||||
assert result == {"id": "aws-rds", "parameters": {"ip": "192.0.2.0"}}
|
||||
|
||||
|
||||
def test_normalize_run_payload_passthrough_for_non_dict():
|
||||
sentinel = "not-a-dict"
|
||||
assert views_helpers.normalize_run_payload(sentinel) is sentinel
|
||||
|
||||
|
||||
def test_prepare_query_parameters_includes_provider_and_casts(
|
||||
attack_paths_query_definition_factory,
|
||||
):
|
||||
definition = attack_paths_query_definition_factory(cast_type=int)
|
||||
result = views_helpers.prepare_query_parameters(
|
||||
definition,
|
||||
{"limit": "5"},
|
||||
provider_uid="123456789012",
|
||||
)
|
||||
|
||||
assert result["provider_uid"] == "123456789012"
|
||||
assert result["limit"] == 5
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provided,expected_message",
|
||||
[
|
||||
({}, "Missing required parameter"),
|
||||
({"limit": 10, "extra": True}, "Unknown parameter"),
|
||||
],
|
||||
)
|
||||
def test_prepare_query_parameters_validates_names(
|
||||
attack_paths_query_definition_factory, provided, expected_message
|
||||
):
|
||||
definition = attack_paths_query_definition_factory()
|
||||
|
||||
with pytest.raises(ValidationError) as exc:
|
||||
views_helpers.prepare_query_parameters(definition, provided, provider_uid="1")
|
||||
|
||||
assert expected_message in str(exc.value)
|
||||
|
||||
|
||||
def test_prepare_query_parameters_validates_cast(
|
||||
attack_paths_query_definition_factory,
|
||||
):
|
||||
definition = attack_paths_query_definition_factory(cast_type=int)
|
||||
|
||||
with pytest.raises(ValidationError) as exc:
|
||||
views_helpers.prepare_query_parameters(
|
||||
definition,
|
||||
{"limit": "not-an-int"},
|
||||
provider_uid="1",
|
||||
)
|
||||
|
||||
assert "Invalid value" in str(exc.value)
|
||||
|
||||
|
||||
def test_execute_attack_paths_query_serializes_graph(
|
||||
attack_paths_query_definition_factory, attack_paths_graph_stub_classes
|
||||
):
|
||||
definition = attack_paths_query_definition_factory(
|
||||
id="aws-rds",
|
||||
name="RDS",
|
||||
description="",
|
||||
cypher="MATCH (n) RETURN n",
|
||||
parameters=[],
|
||||
)
|
||||
parameters = {"provider_uid": "123"}
|
||||
attack_paths_scan = SimpleNamespace(graph_database="tenant-db")
|
||||
|
||||
node = attack_paths_graph_stub_classes.Node(
|
||||
element_id="node-1",
|
||||
labels=["AWSAccount"],
|
||||
properties={
|
||||
"name": "account",
|
||||
"complex": {
|
||||
"items": [
|
||||
attack_paths_graph_stub_classes.NativeValue("value"),
|
||||
{"nested": 1},
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
relationship = attack_paths_graph_stub_classes.Relationship(
|
||||
element_id="rel-1",
|
||||
rel_type="OWNS",
|
||||
start_node=node,
|
||||
end_node=attack_paths_graph_stub_classes.Node("node-2", ["RDSInstance"], {}),
|
||||
properties={"weight": 1},
|
||||
)
|
||||
graph = SimpleNamespace(nodes=[node], relationships=[relationship])
|
||||
|
||||
run_result = MagicMock()
|
||||
run_result.graph.return_value = graph
|
||||
|
||||
session = MagicMock()
|
||||
session.run.return_value = run_result
|
||||
|
||||
session_ctx = MagicMock()
|
||||
session_ctx.__enter__.return_value = session
|
||||
session_ctx.__exit__.return_value = False
|
||||
|
||||
with patch(
|
||||
"api.attack_paths.views_helpers.graph_database.get_session",
|
||||
return_value=session_ctx,
|
||||
) as mock_get_session:
|
||||
result = views_helpers.execute_attack_paths_query(
|
||||
attack_paths_scan, definition, parameters
|
||||
)
|
||||
|
||||
mock_get_session.assert_called_once_with("tenant-db")
|
||||
session.run.assert_called_once_with(definition.cypher, parameters)
|
||||
assert result["nodes"][0]["id"] == "node-1"
|
||||
assert result["nodes"][0]["properties"]["complex"]["items"][0] == "value"
|
||||
assert result["relationships"][0]["label"] == "OWNS"
|
||||
|
||||
|
||||
def test_execute_attack_paths_query_wraps_graph_errors(
|
||||
attack_paths_query_definition_factory,
|
||||
):
|
||||
definition = attack_paths_query_definition_factory(
|
||||
id="aws-rds",
|
||||
name="RDS",
|
||||
description="",
|
||||
cypher="MATCH (n) RETURN n",
|
||||
parameters=[],
|
||||
)
|
||||
attack_paths_scan = SimpleNamespace(graph_database="tenant-db")
|
||||
parameters = {"provider_uid": "123"}
|
||||
|
||||
class ExplodingContext:
|
||||
def __enter__(self):
|
||||
raise graph_database.GraphDatabaseQueryException("boom")
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
with (
|
||||
patch(
|
||||
"api.attack_paths.views_helpers.graph_database.get_session",
|
||||
return_value=ExplodingContext(),
|
||||
),
|
||||
patch("api.attack_paths.views_helpers.logger") as mock_logger,
|
||||
):
|
||||
with pytest.raises(APIException):
|
||||
views_helpers.execute_attack_paths_query(
|
||||
attack_paths_scan, definition, parameters
|
||||
)
|
||||
|
||||
mock_logger.error.assert_called_once()
|
||||
@@ -0,0 +1,303 @@
|
||||
"""
|
||||
Tests for Neo4j database lazy initialization.
|
||||
|
||||
The Neo4j driver connects on first use by default. API processes may
|
||||
eagerly initialize the driver during app startup, while Celery workers
|
||||
remain lazy. These tests validate the database module behavior itself.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestLazyInitialization:
|
||||
"""Test that Neo4j driver is initialized lazily on first use."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
"""Reset module-level singleton state before each test."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
original_driver = db_module._driver
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
|
||||
def test_driver_not_initialized_at_import(self):
|
||||
"""Driver should be None after module import (no eager connection)."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
assert db_module._driver is None
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_creates_connection_on_first_call(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""init_driver() should create connection only when called."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
assert db_module._driver is None
|
||||
|
||||
result = db_module.init_driver()
|
||||
|
||||
mock_driver_factory.assert_called_once()
|
||||
mock_driver.verify_connectivity.assert_called_once()
|
||||
assert result is mock_driver
|
||||
assert db_module._driver is mock_driver
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_returns_cached_driver_on_subsequent_calls(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""Subsequent calls should return cached driver without reconnecting."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
first_result = db_module.init_driver()
|
||||
second_result = db_module.init_driver()
|
||||
third_result = db_module.init_driver()
|
||||
|
||||
# Only one connection attempt
|
||||
assert mock_driver_factory.call_count == 1
|
||||
assert mock_driver.verify_connectivity.call_count == 1
|
||||
|
||||
# All calls return same instance
|
||||
assert first_result is second_result is third_result
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_get_driver_delegates_to_init_driver(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""get_driver() should use init_driver() for lazy initialization."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
result = db_module.get_driver()
|
||||
|
||||
assert result is mock_driver
|
||||
mock_driver_factory.assert_called_once()
|
||||
|
||||
|
||||
class TestAtexitRegistration:
|
||||
"""Test that atexit cleanup handler is registered correctly."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
"""Reset module-level singleton state before each test."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
original_driver = db_module._driver
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.atexit.register")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_atexit_registered_on_first_init(
|
||||
self, mock_driver_factory, mock_atexit_register, mock_settings
|
||||
):
|
||||
"""atexit.register should be called on first initialization."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver_factory.return_value = MagicMock()
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
db_module.init_driver()
|
||||
|
||||
mock_atexit_register.assert_called_once_with(db_module.close_driver)
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.atexit.register")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_atexit_registered_only_once(
|
||||
self, mock_driver_factory, mock_atexit_register, mock_settings
|
||||
):
|
||||
"""atexit.register should only be called once across multiple inits.
|
||||
|
||||
The double-checked locking on _driver ensures the atexit registration
|
||||
block only executes once (when _driver is first created).
|
||||
"""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver_factory.return_value = MagicMock()
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
db_module.init_driver()
|
||||
db_module.init_driver()
|
||||
db_module.init_driver()
|
||||
|
||||
# Only registered once because subsequent calls hit the fast path
|
||||
assert mock_atexit_register.call_count == 1
|
||||
|
||||
|
||||
class TestCloseDriver:
|
||||
"""Test driver cleanup functionality."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
"""Reset module-level singleton state before each test."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
original_driver = db_module._driver
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
|
||||
def test_close_driver_closes_and_clears_driver(self):
|
||||
"""close_driver() should close the driver and set it to None."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
db_module._driver = mock_driver
|
||||
|
||||
db_module.close_driver()
|
||||
|
||||
mock_driver.close.assert_called_once()
|
||||
assert db_module._driver is None
|
||||
|
||||
def test_close_driver_handles_none_driver(self):
|
||||
"""close_driver() should handle case where driver is None."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
# Should not raise
|
||||
db_module.close_driver()
|
||||
|
||||
assert db_module._driver is None
|
||||
|
||||
def test_close_driver_clears_driver_even_on_close_error(self):
|
||||
"""Driver should be cleared even if close() raises an exception."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver.close.side_effect = Exception("Connection error")
|
||||
db_module._driver = mock_driver
|
||||
|
||||
with pytest.raises(Exception, match="Connection error"):
|
||||
db_module.close_driver()
|
||||
|
||||
# Driver should still be cleared
|
||||
assert db_module._driver is None
|
||||
|
||||
|
||||
class TestThreadSafety:
|
||||
"""Test thread-safe initialization."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
"""Reset module-level singleton state before each test."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
original_driver = db_module._driver
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_concurrent_init_creates_single_driver(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""Multiple threads calling init_driver() should create only one driver."""
|
||||
import api.attack_paths.database as db_module
|
||||
|
||||
mock_driver = MagicMock()
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
def call_init():
|
||||
try:
|
||||
result = db_module.init_driver()
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
errors.append(e)
|
||||
|
||||
threads = [threading.Thread(target=call_init) for _ in range(10)]
|
||||
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert not errors, f"Threads raised errors: {errors}"
|
||||
|
||||
# Only one driver created
|
||||
assert mock_driver_factory.call_count == 1
|
||||
|
||||
# All threads got the same driver instance
|
||||
assert all(r is mock_driver for r in results)
|
||||
assert len(results) == 10
|
||||
@@ -6,7 +6,6 @@ from api.compliance import (
|
||||
get_prowler_provider_checks,
|
||||
get_prowler_provider_compliance,
|
||||
load_prowler_checks,
|
||||
load_prowler_compliance,
|
||||
)
|
||||
from api.models import Provider
|
||||
|
||||
@@ -35,55 +34,6 @@ class TestCompliance:
|
||||
assert compliance_data == mock_compliance.get_bulk.return_value
|
||||
mock_compliance.get_bulk.assert_called_once_with(provider_type)
|
||||
|
||||
@patch("api.models.Provider.ProviderChoices")
|
||||
@patch("api.compliance.get_prowler_provider_compliance")
|
||||
@patch("api.compliance.generate_compliance_overview_template")
|
||||
@patch("api.compliance.load_prowler_checks")
|
||||
def test_load_prowler_compliance(
|
||||
self,
|
||||
mock_load_prowler_checks,
|
||||
mock_generate_compliance_overview_template,
|
||||
mock_get_prowler_provider_compliance,
|
||||
mock_provider_choices,
|
||||
):
|
||||
mock_provider_choices.values = ["aws", "azure"]
|
||||
|
||||
compliance_data_aws = {"compliance_aws": MagicMock()}
|
||||
compliance_data_azure = {"compliance_azure": MagicMock()}
|
||||
|
||||
compliance_data_dict = {
|
||||
"aws": compliance_data_aws,
|
||||
"azure": compliance_data_azure,
|
||||
}
|
||||
|
||||
def mock_get_compliance(provider_type):
|
||||
return compliance_data_dict[provider_type]
|
||||
|
||||
mock_get_prowler_provider_compliance.side_effect = mock_get_compliance
|
||||
|
||||
mock_generate_compliance_overview_template.return_value = {
|
||||
"template_key": "template_value"
|
||||
}
|
||||
|
||||
mock_load_prowler_checks.return_value = {"checks_key": "checks_value"}
|
||||
|
||||
load_prowler_compliance()
|
||||
|
||||
from api.compliance import PROWLER_CHECKS, PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
|
||||
|
||||
assert PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE == {
|
||||
"template_key": "template_value"
|
||||
}
|
||||
assert PROWLER_CHECKS == {"checks_key": "checks_value"}
|
||||
|
||||
expected_prowler_compliance = compliance_data_dict
|
||||
mock_get_prowler_provider_compliance.assert_any_call("aws")
|
||||
mock_get_prowler_provider_compliance.assert_any_call("azure")
|
||||
mock_generate_compliance_overview_template.assert_called_once_with(
|
||||
expected_prowler_compliance
|
||||
)
|
||||
mock_load_prowler_checks.assert_called_once_with(expected_prowler_compliance)
|
||||
|
||||
@patch("api.compliance.get_prowler_provider_checks")
|
||||
@patch("api.models.Provider.ProviderChoices")
|
||||
def test_load_prowler_checks(
|
||||
|
||||
@@ -1,9 +1,21 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from allauth.socialaccount.models import SocialApp
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import IntegrityError
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.models import Resource, ResourceTag, SAMLConfiguration, SAMLDomainIndex
|
||||
from api.models import (
|
||||
ProviderComplianceScore,
|
||||
Resource,
|
||||
ResourceTag,
|
||||
SAMLConfiguration,
|
||||
SAMLDomainIndex,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
TenantComplianceSummary,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -324,3 +336,159 @@ class TestSAMLConfigurationModel:
|
||||
errors = exc_info.value.message_dict
|
||||
assert "metadata_xml" in errors
|
||||
assert "There is a problem with your metadata." in errors["metadata_xml"][0]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestProviderComplianceScoreModel:
|
||||
def test_create_provider_compliance_score(self, providers_fixture, scans_fixture):
|
||||
provider = providers_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
scan.completed_at = datetime.now(timezone.utc)
|
||||
scan.save()
|
||||
|
||||
score = ProviderComplianceScore.objects.create(
|
||||
tenant_id=provider.tenant_id,
|
||||
provider=provider,
|
||||
scan=scan,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_1",
|
||||
requirement_status=StatusChoices.PASS,
|
||||
scan_completed_at=scan.completed_at,
|
||||
)
|
||||
|
||||
assert score.compliance_id == "aws_cis_2.0"
|
||||
assert score.requirement_id == "req_1"
|
||||
assert score.requirement_status == StatusChoices.PASS
|
||||
|
||||
def test_unique_constraint_per_provider_compliance_requirement(
|
||||
self, providers_fixture, scans_fixture
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
scan.completed_at = datetime.now(timezone.utc)
|
||||
scan.save()
|
||||
|
||||
ProviderComplianceScore.objects.create(
|
||||
tenant_id=provider.tenant_id,
|
||||
provider=provider,
|
||||
scan=scan,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_1",
|
||||
requirement_status=StatusChoices.PASS,
|
||||
scan_completed_at=scan.completed_at,
|
||||
)
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
ProviderComplianceScore.objects.create(
|
||||
tenant_id=provider.tenant_id,
|
||||
provider=provider,
|
||||
scan=scan,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_1",
|
||||
requirement_status=StatusChoices.FAIL,
|
||||
scan_completed_at=scan.completed_at,
|
||||
)
|
||||
|
||||
def test_different_providers_same_requirement_allowed(
|
||||
self, providers_fixture, scans_fixture
|
||||
):
|
||||
provider1, provider2, *_ = providers_fixture
|
||||
scan1 = scans_fixture[0]
|
||||
scan1.completed_at = datetime.now(timezone.utc)
|
||||
scan1.save()
|
||||
|
||||
scan2 = scans_fixture[2]
|
||||
scan2.state = StateChoices.COMPLETED
|
||||
scan2.completed_at = datetime.now(timezone.utc)
|
||||
scan2.save()
|
||||
|
||||
score1 = ProviderComplianceScore.objects.create(
|
||||
tenant_id=provider1.tenant_id,
|
||||
provider=provider1,
|
||||
scan=scan1,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_1",
|
||||
requirement_status=StatusChoices.PASS,
|
||||
scan_completed_at=scan1.completed_at,
|
||||
)
|
||||
|
||||
score2 = ProviderComplianceScore.objects.create(
|
||||
tenant_id=provider2.tenant_id,
|
||||
provider=provider2,
|
||||
scan=scan2,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_1",
|
||||
requirement_status=StatusChoices.FAIL,
|
||||
scan_completed_at=scan2.completed_at,
|
||||
)
|
||||
|
||||
assert score1.id != score2.id
|
||||
assert score1.requirement_status != score2.requirement_status
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestTenantComplianceSummaryModel:
|
||||
def test_create_tenant_compliance_summary(self, tenants_fixture):
|
||||
tenant = tenants_fixture[0]
|
||||
|
||||
summary = TenantComplianceSummary.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirements_passed=5,
|
||||
requirements_failed=2,
|
||||
requirements_manual=1,
|
||||
total_requirements=8,
|
||||
)
|
||||
|
||||
assert summary.compliance_id == "aws_cis_2.0"
|
||||
assert summary.requirements_passed == 5
|
||||
assert summary.requirements_failed == 2
|
||||
assert summary.requirements_manual == 1
|
||||
assert summary.total_requirements == 8
|
||||
assert summary.updated_at is not None
|
||||
|
||||
def test_unique_constraint_per_tenant_compliance(self, tenants_fixture):
|
||||
tenant = tenants_fixture[0]
|
||||
|
||||
TenantComplianceSummary.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirements_passed=5,
|
||||
requirements_failed=2,
|
||||
requirements_manual=1,
|
||||
total_requirements=8,
|
||||
)
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
TenantComplianceSummary.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirements_passed=3,
|
||||
requirements_failed=4,
|
||||
requirements_manual=1,
|
||||
total_requirements=8,
|
||||
)
|
||||
|
||||
def test_different_tenants_same_compliance_allowed(self, tenants_fixture):
|
||||
tenant1, tenant2, *_ = tenants_fixture
|
||||
|
||||
summary1 = TenantComplianceSummary.objects.create(
|
||||
tenant_id=tenant1.id,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirements_passed=5,
|
||||
requirements_failed=2,
|
||||
requirements_manual=1,
|
||||
total_requirements=8,
|
||||
)
|
||||
|
||||
summary2 = TenantComplianceSummary.objects.create(
|
||||
tenant_id=tenant2.id,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirements_passed=3,
|
||||
requirements_failed=4,
|
||||
requirements_manual=1,
|
||||
total_requirements=8,
|
||||
)
|
||||
|
||||
assert summary1.id != summary2.id
|
||||
assert summary1.requirements_passed != summary2.requirements_passed
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from allauth.socialaccount.providers.oauth2.client import OAuth2Client
|
||||
from django.contrib.postgres.aggregates import ArrayAgg
|
||||
@@ -11,19 +14,25 @@ from api.exceptions import InvitationTokenExpiredException
|
||||
from api.models import Integration, Invitation, Processor, Provider, Resource
|
||||
from api.v1.serializers import FindingMetadataSerializer
|
||||
from prowler.lib.outputs.jira.jira import Jira, JiraBasicAuthError
|
||||
from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.aws.lib.s3.s3 import S3
|
||||
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub
|
||||
from prowler.providers.azure.azure_provider import AzureProvider
|
||||
from prowler.providers.common.models import Connection
|
||||
from prowler.providers.gcp.gcp_provider import GcpProvider
|
||||
from prowler.providers.github.github_provider import GithubProvider
|
||||
from prowler.providers.iac.iac_provider import IacProvider
|
||||
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
|
||||
from prowler.providers.m365.m365_provider import M365Provider
|
||||
from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider
|
||||
from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prowler.providers.alibabacloud.alibabacloud_provider import (
|
||||
AlibabacloudProvider,
|
||||
)
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.azure.azure_provider import AzureProvider
|
||||
from prowler.providers.gcp.gcp_provider import GcpProvider
|
||||
from prowler.providers.github.github_provider import GithubProvider
|
||||
from prowler.providers.iac.iac_provider import IacProvider
|
||||
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
|
||||
from prowler.providers.m365.m365_provider import M365Provider
|
||||
from prowler.providers.mongodbatlas.mongodbatlas_provider import (
|
||||
MongodbatlasProvider,
|
||||
)
|
||||
from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
|
||||
|
||||
|
||||
class CustomOAuth2Client(OAuth2Client):
|
||||
@@ -89,24 +98,52 @@ def return_prowler_provider(
|
||||
"""
|
||||
match provider.provider:
|
||||
case Provider.ProviderChoices.AWS.value:
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
|
||||
prowler_provider = AwsProvider
|
||||
case Provider.ProviderChoices.GCP.value:
|
||||
from prowler.providers.gcp.gcp_provider import GcpProvider
|
||||
|
||||
prowler_provider = GcpProvider
|
||||
case Provider.ProviderChoices.AZURE.value:
|
||||
from prowler.providers.azure.azure_provider import AzureProvider
|
||||
|
||||
prowler_provider = AzureProvider
|
||||
case Provider.ProviderChoices.KUBERNETES.value:
|
||||
from prowler.providers.kubernetes.kubernetes_provider import (
|
||||
KubernetesProvider,
|
||||
)
|
||||
|
||||
prowler_provider = KubernetesProvider
|
||||
case Provider.ProviderChoices.M365.value:
|
||||
from prowler.providers.m365.m365_provider import M365Provider
|
||||
|
||||
prowler_provider = M365Provider
|
||||
case Provider.ProviderChoices.GITHUB.value:
|
||||
from prowler.providers.github.github_provider import GithubProvider
|
||||
|
||||
prowler_provider = GithubProvider
|
||||
case Provider.ProviderChoices.MONGODBATLAS.value:
|
||||
from prowler.providers.mongodbatlas.mongodbatlas_provider import (
|
||||
MongodbatlasProvider,
|
||||
)
|
||||
|
||||
prowler_provider = MongodbatlasProvider
|
||||
case Provider.ProviderChoices.IAC.value:
|
||||
from prowler.providers.iac.iac_provider import IacProvider
|
||||
|
||||
prowler_provider = IacProvider
|
||||
case Provider.ProviderChoices.ORACLECLOUD.value:
|
||||
from prowler.providers.oraclecloud.oraclecloud_provider import (
|
||||
OraclecloudProvider,
|
||||
)
|
||||
|
||||
prowler_provider = OraclecloudProvider
|
||||
case Provider.ProviderChoices.ALIBABACLOUD.value:
|
||||
from prowler.providers.alibabacloud.alibabacloud_provider import (
|
||||
AlibabacloudProvider,
|
||||
)
|
||||
|
||||
prowler_provider = AlibabacloudProvider
|
||||
case _:
|
||||
raise ValueError(f"Provider type {provider.provider} not supported")
|
||||
@@ -386,10 +423,28 @@ def get_findings_metadata_no_aggregations(tenant_id: str, filtered_queryset):
|
||||
regions = sorted({region for region in aggregation["regions"] or [] if region})
|
||||
resource_types = sorted(set(aggregation["resource_types"] or []))
|
||||
|
||||
# Aggregate categories from findings
|
||||
categories_set = set()
|
||||
for categories_list in filtered_queryset.values_list("categories", flat=True):
|
||||
if categories_list:
|
||||
categories_set.update(categories_list)
|
||||
categories = sorted(categories_set)
|
||||
|
||||
# Aggregate groups from findings
|
||||
groups = list(
|
||||
filtered_queryset.exclude(resource_groups__isnull=True)
|
||||
.exclude(resource_groups__exact="")
|
||||
.values_list("resource_groups", flat=True)
|
||||
.distinct()
|
||||
.order_by("resource_groups")
|
||||
)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"resource_types": resource_types,
|
||||
"categories": categories,
|
||||
"groups": groups,
|
||||
}
|
||||
|
||||
serializer = FindingMetadataSerializer(data=result)
|
||||
|
||||
@@ -21,6 +21,7 @@ from rest_framework_simplejwt.tokens import RefreshToken
|
||||
from api.db_router import MainRouter
|
||||
from api.exceptions import ConflictException
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
Finding,
|
||||
Integration,
|
||||
IntegrationProviderRelationship,
|
||||
@@ -1132,6 +1133,109 @@ class ScanComplianceReportSerializer(BaseSerializerV1):
|
||||
fields = ["id", "name"]
|
||||
|
||||
|
||||
class AttackPathsScanSerializer(RLSSerializer):
|
||||
state = StateEnumSerializerField(read_only=True)
|
||||
provider_alias = serializers.SerializerMethodField(read_only=True)
|
||||
provider_type = serializers.SerializerMethodField(read_only=True)
|
||||
provider_uid = serializers.SerializerMethodField(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = AttackPathsScan
|
||||
fields = [
|
||||
"id",
|
||||
"state",
|
||||
"progress",
|
||||
"provider",
|
||||
"provider_alias",
|
||||
"provider_type",
|
||||
"provider_uid",
|
||||
"scan",
|
||||
"task",
|
||||
"inserted_at",
|
||||
"started_at",
|
||||
"completed_at",
|
||||
"duration",
|
||||
]
|
||||
|
||||
included_serializers = {
|
||||
"provider": "api.v1.serializers.ProviderIncludeSerializer",
|
||||
"scan": "api.v1.serializers.ScanIncludeSerializer",
|
||||
"task": "api.v1.serializers.TaskSerializer",
|
||||
}
|
||||
|
||||
def get_provider_alias(self, obj):
|
||||
provider = getattr(obj, "provider", None)
|
||||
return provider.alias if provider else None
|
||||
|
||||
def get_provider_type(self, obj):
|
||||
provider = getattr(obj, "provider", None)
|
||||
return provider.provider if provider else None
|
||||
|
||||
def get_provider_uid(self, obj):
|
||||
provider = getattr(obj, "provider", None)
|
||||
return provider.uid if provider else None
|
||||
|
||||
|
||||
class AttackPathsQueryParameterSerializer(BaseSerializerV1):
|
||||
name = serializers.CharField()
|
||||
label = serializers.CharField()
|
||||
data_type = serializers.CharField(default="string")
|
||||
description = serializers.CharField(allow_null=True, required=False)
|
||||
placeholder = serializers.CharField(allow_null=True, required=False)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-parameters"
|
||||
|
||||
|
||||
class AttackPathsQuerySerializer(BaseSerializerV1):
|
||||
id = serializers.CharField()
|
||||
name = serializers.CharField()
|
||||
description = serializers.CharField()
|
||||
provider = serializers.CharField()
|
||||
parameters = AttackPathsQueryParameterSerializer(many=True)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-queries"
|
||||
|
||||
|
||||
class AttackPathsQueryRunRequestSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField()
|
||||
parameters = serializers.DictField(
|
||||
child=serializers.JSONField(), allow_empty=True, required=False
|
||||
)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-run-requests"
|
||||
|
||||
|
||||
class AttackPathsNodeSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField()
|
||||
labels = serializers.ListField(child=serializers.CharField())
|
||||
properties = serializers.DictField(child=serializers.JSONField())
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-result-nodes"
|
||||
|
||||
|
||||
class AttackPathsRelationshipSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField()
|
||||
label = serializers.CharField()
|
||||
source = serializers.CharField()
|
||||
target = serializers.CharField()
|
||||
properties = serializers.DictField(child=serializers.JSONField())
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-result-relationships"
|
||||
|
||||
|
||||
class AttackPathsQueryResultSerializer(BaseSerializerV1):
|
||||
nodes = AttackPathsNodeSerializer(many=True)
|
||||
relationships = AttackPathsRelationshipSerializer(many=True)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-paths-query-results"
|
||||
|
||||
|
||||
class ResourceTagSerializer(RLSSerializer):
|
||||
"""
|
||||
Serializer for the ResourceTag model
|
||||
@@ -1175,6 +1279,7 @@ class ResourceSerializer(RLSSerializer):
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
"groups",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
@@ -1183,6 +1288,7 @@ class ResourceSerializer(RLSSerializer):
|
||||
"metadata": {"read_only": True},
|
||||
"details": {"read_only": True},
|
||||
"partition": {"read_only": True},
|
||||
"groups": {"read_only": True},
|
||||
}
|
||||
|
||||
included_serializers = {
|
||||
@@ -1276,6 +1382,7 @@ class ResourceMetadataSerializer(BaseSerializerV1):
|
||||
services = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
types = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
groups = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
|
||||
|
||||
@@ -1301,6 +1408,8 @@ class FindingSerializer(RLSSerializer):
|
||||
"severity",
|
||||
"check_id",
|
||||
"check_metadata",
|
||||
"categories",
|
||||
"resource_groups",
|
||||
"raw_result",
|
||||
"inserted_at",
|
||||
"updated_at",
|
||||
@@ -1356,6 +1465,10 @@ class FindingMetadataSerializer(BaseSerializerV1):
|
||||
resource_types = serializers.ListField(
|
||||
child=serializers.CharField(), allow_empty=True
|
||||
)
|
||||
categories = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
groups = serializers.ListField(
|
||||
child=serializers.CharField(), allow_empty=True, required=False, default=list
|
||||
)
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
|
||||
|
||||
@@ -1551,14 +1664,8 @@ class AlibabaCloudProviderSecret(serializers.Serializer):
|
||||
|
||||
|
||||
class AlibabaCloudRoleAssumptionProviderSecret(serializers.Serializer):
|
||||
"""Serializer for Alibaba Cloud RAM Role Assumption credentials.
|
||||
|
||||
This allows assuming a RAM role using access keys, similar to AWS STS AssumeRole.
|
||||
The SDK will automatically manage the STS token refresh.
|
||||
"""
|
||||
|
||||
role_arn = serializers.CharField(
|
||||
help_text="ARN of the RAM role to assume (e.g., acs:ram::1234567890123456:role/ProwlerRole)"
|
||||
help_text="Access Key ID of the RAM user that will assume the role"
|
||||
)
|
||||
access_key_id = serializers.CharField(
|
||||
help_text="Access Key ID of the RAM user that will assume the role"
|
||||
@@ -2287,14 +2394,56 @@ class AttackSurfaceOverviewSerializer(BaseSerializerV1):
|
||||
total_findings = serializers.IntegerField()
|
||||
failed_findings = serializers.IntegerField()
|
||||
muted_failed_findings = serializers.IntegerField()
|
||||
check_ids = serializers.ListField(
|
||||
child=serializers.CharField(), allow_empty=True, default=list, read_only=True
|
||||
)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-surface-overviews"
|
||||
|
||||
|
||||
class CategoryOverviewSerializer(BaseSerializerV1):
|
||||
"""Serializer for category overview aggregations."""
|
||||
|
||||
id = serializers.CharField(source="category")
|
||||
total_findings = serializers.IntegerField()
|
||||
failed_findings = serializers.IntegerField()
|
||||
new_failed_findings = serializers.IntegerField()
|
||||
severity = serializers.JSONField(
|
||||
help_text="Severity breakdown: {informational, low, medium, high, critical}"
|
||||
)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "category-overviews"
|
||||
|
||||
|
||||
class ResourceGroupOverviewSerializer(BaseSerializerV1):
|
||||
"""Serializer for resource group overview aggregations."""
|
||||
|
||||
id = serializers.CharField(source="resource_group")
|
||||
total_findings = serializers.IntegerField()
|
||||
failed_findings = serializers.IntegerField()
|
||||
new_failed_findings = serializers.IntegerField()
|
||||
resources_count = serializers.IntegerField()
|
||||
severity = serializers.JSONField(
|
||||
help_text="Severity breakdown: {informational, low, medium, high, critical}"
|
||||
)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "resource-group-overviews"
|
||||
|
||||
|
||||
class ComplianceWatchlistOverviewSerializer(BaseSerializerV1):
|
||||
"""Serializer for compliance watchlist overview with FAIL-dominant aggregation."""
|
||||
|
||||
id = serializers.CharField(source="compliance_id")
|
||||
compliance_id = serializers.CharField()
|
||||
requirements_passed = serializers.IntegerField()
|
||||
requirements_failed = serializers.IntegerField()
|
||||
requirements_manual = serializers.IntegerField()
|
||||
total_requirements = serializers.IntegerField()
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "compliance-watchlist-overviews"
|
||||
|
||||
|
||||
class OverviewRegionSerializer(serializers.Serializer):
|
||||
id = serializers.SerializerMethodField()
|
||||
provider_type = serializers.CharField()
|
||||
|
||||
@@ -4,6 +4,7 @@ from drf_spectacular.views import SpectacularRedocView
|
||||
from rest_framework_nested import routers
|
||||
|
||||
from api.v1.views import (
|
||||
AttackPathsScanViewSet,
|
||||
ComplianceOverviewViewSet,
|
||||
CustomSAMLLoginView,
|
||||
CustomTokenObtainView,
|
||||
@@ -53,6 +54,9 @@ router.register(r"tenants", TenantViewSet, basename="tenant")
|
||||
router.register(r"providers", ProviderViewSet, basename="provider")
|
||||
router.register(r"provider-groups", ProviderGroupViewSet, basename="providergroup")
|
||||
router.register(r"scans", ScanViewSet, basename="scan")
|
||||
router.register(
|
||||
r"attack-paths-scans", AttackPathsScanViewSet, basename="attack-paths-scans"
|
||||
)
|
||||
router.register(r"tasks", TaskViewSet, basename="task")
|
||||
router.register(r"resources", ResourceViewSet, basename="resource")
|
||||
router.register(r"findings", FindingViewSet, basename="finding")
|
||||
|
||||
+657
-82
@@ -3,6 +3,7 @@ import glob
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
from collections import defaultdict
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -10,6 +11,7 @@ from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import sentry_sdk
|
||||
|
||||
from allauth.socialaccount.models import SocialAccount, SocialApp
|
||||
from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
|
||||
from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter
|
||||
@@ -41,8 +43,9 @@ from django.db.models import (
|
||||
Sum,
|
||||
Value,
|
||||
When,
|
||||
Window,
|
||||
)
|
||||
from django.db.models.functions import Coalesce
|
||||
from django.db.models.functions import Coalesce, RowNumber
|
||||
from django.http import HttpResponse, QueryDict
|
||||
from django.shortcuts import redirect
|
||||
from django.urls import reverse
|
||||
@@ -72,24 +75,13 @@ from rest_framework.generics import GenericAPIView, get_object_or_404
|
||||
from rest_framework.permissions import SAFE_METHODS
|
||||
from rest_framework_json_api.views import RelationshipView, Response
|
||||
from rest_framework_simplejwt.exceptions import InvalidToken, TokenError
|
||||
from tasks.beat import schedule_provider_scan
|
||||
from tasks.jobs.export import get_s3_client
|
||||
from tasks.jobs.scan import _get_attack_surface_mapping_from_provider
|
||||
from tasks.tasks import (
|
||||
backfill_compliance_summaries_task,
|
||||
backfill_scan_resource_summaries_task,
|
||||
check_integration_connection_task,
|
||||
check_lighthouse_connection_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
check_provider_connection_task,
|
||||
delete_provider_task,
|
||||
delete_tenant_task,
|
||||
jira_integration_task,
|
||||
mute_historical_findings_task,
|
||||
perform_scan_task,
|
||||
refresh_lighthouse_provider_models_task,
|
||||
)
|
||||
|
||||
from api.attack_paths import (
|
||||
database as graph_database,
|
||||
get_queries_for_provider,
|
||||
get_query_by_id,
|
||||
views_helpers as attack_paths_views_helpers,
|
||||
)
|
||||
from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset
|
||||
from api.compliance import (
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
|
||||
@@ -99,8 +91,11 @@ from api.db_router import MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.exceptions import TaskFailedException
|
||||
from api.filters import (
|
||||
AttackPathsScanFilter,
|
||||
AttackSurfaceOverviewFilter,
|
||||
CategoryOverviewFilter,
|
||||
ComplianceOverviewFilter,
|
||||
ComplianceWatchlistFilter,
|
||||
CustomDjangoFilterBackend,
|
||||
DailySeveritySummaryFilter,
|
||||
FindingFilter,
|
||||
@@ -118,6 +113,7 @@ from api.filters import (
|
||||
ProviderGroupFilter,
|
||||
ProviderSecretFilter,
|
||||
ResourceFilter,
|
||||
ResourceGroupOverviewFilter,
|
||||
RoleFilter,
|
||||
ScanFilter,
|
||||
ScanSummaryFilter,
|
||||
@@ -129,6 +125,7 @@ from api.filters import (
|
||||
UserFilter,
|
||||
)
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
AttackSurfaceOverview,
|
||||
ComplianceOverviewSummary,
|
||||
ComplianceRequirementOverview,
|
||||
@@ -144,6 +141,7 @@ from api.models import (
|
||||
MuteRule,
|
||||
Processor,
|
||||
Provider,
|
||||
ProviderComplianceScore,
|
||||
ProviderGroup,
|
||||
ProviderGroupMembership,
|
||||
ProviderSecret,
|
||||
@@ -157,11 +155,14 @@ from api.models import (
|
||||
SAMLDomainIndex,
|
||||
SAMLToken,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
SeverityChoices,
|
||||
StateChoices,
|
||||
Task,
|
||||
TenantAPIKey,
|
||||
TenantComplianceSummary,
|
||||
ThreatScoreSnapshot,
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
@@ -177,12 +178,18 @@ from api.utils import (
|
||||
from api.uuid_utils import datetime_to_uuid7, uuid7_start
|
||||
from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin
|
||||
from api.v1.serializers import (
|
||||
AttackPathsQueryResultSerializer,
|
||||
AttackPathsQueryRunRequestSerializer,
|
||||
AttackPathsQuerySerializer,
|
||||
AttackPathsScanSerializer,
|
||||
AttackSurfaceOverviewSerializer,
|
||||
CategoryOverviewSerializer,
|
||||
ComplianceOverviewAttributesSerializer,
|
||||
ComplianceOverviewDetailSerializer,
|
||||
ComplianceOverviewDetailThreatscoreSerializer,
|
||||
ComplianceOverviewMetadataSerializer,
|
||||
ComplianceOverviewSerializer,
|
||||
ComplianceWatchlistOverviewSerializer,
|
||||
FindingDynamicFilterSerializer,
|
||||
FindingMetadataSerializer,
|
||||
FindingSerializer,
|
||||
@@ -227,6 +234,7 @@ from api.v1.serializers import (
|
||||
ProviderSecretUpdateSerializer,
|
||||
ProviderSerializer,
|
||||
ProviderUpdateSerializer,
|
||||
ResourceGroupOverviewSerializer,
|
||||
ResourceMetadataSerializer,
|
||||
ResourceSerializer,
|
||||
RoleCreateSerializer,
|
||||
@@ -256,6 +264,23 @@ from api.v1.serializers import (
|
||||
UserSerializer,
|
||||
UserUpdateSerializer,
|
||||
)
|
||||
from tasks.beat import schedule_provider_scan
|
||||
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
|
||||
from tasks.jobs.export import get_s3_client
|
||||
from tasks.tasks import (
|
||||
backfill_compliance_summaries_task,
|
||||
backfill_scan_resource_summaries_task,
|
||||
check_integration_connection_task,
|
||||
check_lighthouse_connection_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
check_provider_connection_task,
|
||||
delete_provider_task,
|
||||
delete_tenant_task,
|
||||
jira_integration_task,
|
||||
mute_historical_findings_task,
|
||||
perform_scan_task,
|
||||
refresh_lighthouse_provider_models_task,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
@@ -357,7 +382,7 @@ class SchemaView(SpectacularAPIView):
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
spectacular_settings.TITLE = "Prowler API"
|
||||
spectacular_settings.VERSION = "1.16.0"
|
||||
spectacular_settings.VERSION = "1.19.0"
|
||||
spectacular_settings.DESCRIPTION = (
|
||||
"Prowler API specification.\n\nThis file is auto-generated."
|
||||
)
|
||||
@@ -399,6 +424,10 @@ class SchemaView(SpectacularAPIView):
|
||||
"name": "Scan",
|
||||
"description": "Endpoints for triggering manual scans and viewing scan results.",
|
||||
},
|
||||
{
|
||||
"name": "Attack Paths",
|
||||
"description": "Endpoints for Attack Paths scan status and executing Attack Paths queries.",
|
||||
},
|
||||
{
|
||||
"name": "Schedule",
|
||||
"description": "Endpoints for managing scan schedules, allowing configuration of automated "
|
||||
@@ -2149,6 +2178,12 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
},
|
||||
)
|
||||
|
||||
attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id=self.request.tenant_id,
|
||||
scan_id=str(scan.id),
|
||||
provider_id=str(scan.provider_id),
|
||||
)
|
||||
|
||||
prowler_task = Task.objects.get(id=task.id)
|
||||
scan.task_id = task.id
|
||||
scan.save(update_fields=["task_id"])
|
||||
@@ -2229,6 +2264,188 @@ class TaskViewSet(BaseRLSViewSet):
|
||||
)
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
tags=["Attack Paths"],
|
||||
summary="List Attack Paths scans",
|
||||
description="Retrieve Attack Paths scans for the tenant with support for filtering, ordering, and pagination.",
|
||||
),
|
||||
retrieve=extend_schema(
|
||||
tags=["Attack Paths"],
|
||||
summary="Retrieve Attack Paths scan details",
|
||||
description="Fetch full details for a specific Attack Paths scan.",
|
||||
),
|
||||
attack_paths_queries=extend_schema(
|
||||
tags=["Attack Paths"],
|
||||
summary="List attack paths queries",
|
||||
description="Retrieve the catalog of Attack Paths queries available for this Attack Paths scan.",
|
||||
responses={
|
||||
200: OpenApiResponse(AttackPathsQuerySerializer(many=True)),
|
||||
404: OpenApiResponse(
|
||||
description="No queries found for the selected provider"
|
||||
),
|
||||
},
|
||||
),
|
||||
run_attack_paths_query=extend_schema(
|
||||
tags=["Attack Paths"],
|
||||
summary="Execute an Attack Paths query",
|
||||
description="Execute the selected Attack Paths query against the Attack Paths graph and return the resulting subgraph.",
|
||||
request=AttackPathsQueryRunRequestSerializer,
|
||||
responses={
|
||||
200: OpenApiResponse(AttackPathsQueryResultSerializer),
|
||||
400: OpenApiResponse(
|
||||
description="Bad request (e.g., Unknown Attack Paths query for the selected provider)"
|
||||
),
|
||||
404: OpenApiResponse(
|
||||
description="No attack paths found for the given query and parameters"
|
||||
),
|
||||
500: OpenApiResponse(
|
||||
description="Attack Paths query execution failed due to a database error"
|
||||
),
|
||||
},
|
||||
),
|
||||
)
|
||||
class AttackPathsScanViewSet(BaseRLSViewSet):
|
||||
queryset = AttackPathsScan.objects.all()
|
||||
serializer_class = AttackPathsScanSerializer
|
||||
http_method_names = ["get", "post"]
|
||||
filterset_class = AttackPathsScanFilter
|
||||
ordering = ["-inserted_at"]
|
||||
ordering_fields = [
|
||||
"inserted_at",
|
||||
"started_at",
|
||||
]
|
||||
# RBAC required permissions
|
||||
required_permissions = [Permissions.MANAGE_SCANS]
|
||||
|
||||
def set_required_permissions(self):
|
||||
if self.request.method in SAFE_METHODS:
|
||||
self.required_permissions = []
|
||||
|
||||
else:
|
||||
self.required_permissions = [Permissions.MANAGE_SCANS]
|
||||
|
||||
def get_serializer_class(self):
|
||||
if self.action == "run_attack_paths_query":
|
||||
return AttackPathsQueryRunRequestSerializer
|
||||
|
||||
return super().get_serializer_class()
|
||||
|
||||
def get_queryset(self):
|
||||
user_roles = get_role(self.request.user)
|
||||
base_queryset = AttackPathsScan.objects.filter(tenant_id=self.request.tenant_id)
|
||||
|
||||
if user_roles.unlimited_visibility:
|
||||
queryset = base_queryset
|
||||
|
||||
else:
|
||||
queryset = base_queryset.filter(provider__in=get_providers(user_roles))
|
||||
|
||||
return queryset.select_related("provider", "scan", "task")
|
||||
|
||||
def list(self, request, *args, **kwargs):
|
||||
queryset = self.filter_queryset(self.get_queryset())
|
||||
|
||||
latest_per_provider = queryset.annotate(
|
||||
latest_scan_rank=Window(
|
||||
expression=RowNumber(),
|
||||
partition_by=[F("provider_id")],
|
||||
order_by=[F("inserted_at").desc()],
|
||||
)
|
||||
).filter(latest_scan_rank=1)
|
||||
|
||||
page = self.paginate_queryset(latest_per_provider)
|
||||
if page is not None:
|
||||
serializer = self.get_serializer(page, many=True)
|
||||
return self.get_paginated_response(serializer.data)
|
||||
|
||||
serializer = self.get_serializer(latest_per_provider, many=True)
|
||||
return Response(serializer.data)
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
def create(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="POST")
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
def destroy(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="DELETE")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_path="queries",
|
||||
url_name="queries",
|
||||
)
|
||||
def attack_paths_queries(self, request, pk=None):
|
||||
attack_paths_scan = self.get_object()
|
||||
queries = get_queries_for_provider(attack_paths_scan.provider.provider)
|
||||
|
||||
if not queries:
|
||||
return Response(
|
||||
{"detail": "No queries found for the selected provider"},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
serializer = AttackPathsQuerySerializer(queries, many=True)
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["post"],
|
||||
url_path="queries/run",
|
||||
url_name="queries-run",
|
||||
)
|
||||
def run_attack_paths_query(self, request, pk=None):
|
||||
attack_paths_scan = self.get_object()
|
||||
|
||||
if attack_paths_scan.state != StateChoices.COMPLETED:
|
||||
raise ValidationError(
|
||||
{
|
||||
"detail": "The Attack Paths scan must be completed before running Attack Paths queries"
|
||||
}
|
||||
)
|
||||
|
||||
if not attack_paths_scan.graph_database:
|
||||
logger.error(
|
||||
f"The Attack Paths Scan {attack_paths_scan.id} does not reference a graph database"
|
||||
)
|
||||
return Response(
|
||||
{"detail": "The Attack Paths scan does not reference a graph database"},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
payload = attack_paths_views_helpers.normalize_run_payload(request.data)
|
||||
serializer = AttackPathsQueryRunRequestSerializer(data=payload)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
|
||||
query_definition = get_query_by_id(serializer.validated_data["id"])
|
||||
if (
|
||||
query_definition is None
|
||||
or query_definition.provider != attack_paths_scan.provider.provider
|
||||
):
|
||||
raise ValidationError(
|
||||
{"id": "Unknown Attack Paths query for the selected provider"}
|
||||
)
|
||||
|
||||
parameters = attack_paths_views_helpers.prepare_query_parameters(
|
||||
query_definition,
|
||||
serializer.validated_data.get("parameters", {}),
|
||||
attack_paths_scan.provider.uid,
|
||||
)
|
||||
|
||||
graph = attack_paths_views_helpers.execute_attack_paths_query(
|
||||
attack_paths_scan, query_definition, parameters
|
||||
)
|
||||
graph_database.clear_cache(attack_paths_scan.graph_database)
|
||||
|
||||
status_code = status.HTTP_200_OK
|
||||
if not graph.get("nodes"):
|
||||
status_code = status.HTTP_404_NOT_FOUND
|
||||
|
||||
response_serializer = AttackPathsQueryResultSerializer(graph)
|
||||
return Response(response_serializer.data, status=status_code)
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
tags=["Resource"],
|
||||
@@ -2521,10 +2738,20 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
.order_by("resource_type")
|
||||
)
|
||||
|
||||
# Get groups from Resource model (flatten ArrayField)
|
||||
all_groups = Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
groups__isnull=False,
|
||||
).values_list("groups", flat=True)
|
||||
groups = sorted(
|
||||
set(g for groups_list in all_groups if groups_list for g in groups_list)
|
||||
)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"types": resource_types,
|
||||
"groups": groups,
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
@@ -2581,10 +2808,20 @@ class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
.order_by("resource_type")
|
||||
)
|
||||
|
||||
# Get groups from Resource model for resources in latest scans (flatten ArrayField)
|
||||
all_groups = Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
groups__isnull=False,
|
||||
).values_list("groups", flat=True)
|
||||
groups = sorted(
|
||||
set(g for groups_list in all_groups if groups_list for g in groups_list)
|
||||
)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"types": resource_types,
|
||||
"groups": groups,
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
@@ -2760,12 +2997,15 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
|
||||
queryset = ResourceScanSummary.objects.filter(tenant_id=tenant_id)
|
||||
scan_based_filters = {}
|
||||
category_scan_filters = {} # Filters for ScanCategorySummary
|
||||
|
||||
if scans := query_params.get("filter[scan__in]") or query_params.get(
|
||||
"filter[scan]"
|
||||
):
|
||||
queryset = queryset.filter(scan_id__in=scans.split(","))
|
||||
scan_based_filters = {"id__in": scans.split(",")}
|
||||
scan_ids_list = scans.split(",")
|
||||
queryset = queryset.filter(scan_id__in=scan_ids_list)
|
||||
scan_based_filters = {"id__in": scan_ids_list}
|
||||
category_scan_filters = {"scan_id__in": scan_ids_list}
|
||||
else:
|
||||
exact = query_params.get("filter[inserted_at]")
|
||||
gte = query_params.get("filter[inserted_at__gte]")
|
||||
@@ -2809,6 +3049,7 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
scan_based_filters = {
|
||||
key.lstrip("scan_"): value for key, value in date_filters.items()
|
||||
}
|
||||
category_scan_filters = date_filters
|
||||
|
||||
# ToRemove: Temporary fallback mechanism
|
||||
if not queryset.exists():
|
||||
@@ -2855,10 +3096,31 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
.order_by("resource_type")
|
||||
)
|
||||
|
||||
# Get categories from ScanCategorySummary using same scan filters
|
||||
categories = list(
|
||||
ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id, **category_scan_filters
|
||||
)
|
||||
.values_list("category", flat=True)
|
||||
.distinct()
|
||||
.order_by("category")
|
||||
)
|
||||
|
||||
# Fallback to finding aggregation if no ScanCategorySummary exists
|
||||
if not categories:
|
||||
categories_set = set()
|
||||
for categories_list in filtered_queryset.values_list(
|
||||
"categories", flat=True
|
||||
):
|
||||
if categories_list:
|
||||
categories_set.update(categories_list)
|
||||
categories = sorted(categories_set)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"resource_types": resource_types,
|
||||
"categories": categories,
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
@@ -2963,10 +3225,48 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
.order_by("resource_type")
|
||||
)
|
||||
|
||||
# Get categories from ScanCategorySummary for latest scans
|
||||
categories = list(
|
||||
ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id__in=latest_scans_queryset.values_list("id", flat=True),
|
||||
)
|
||||
.values_list("category", flat=True)
|
||||
.distinct()
|
||||
.order_by("category")
|
||||
)
|
||||
|
||||
# Fallback to finding aggregation if no ScanCategorySummary exists
|
||||
if not categories:
|
||||
filtered_queryset = self.filter_queryset(self.get_queryset()).filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id__in=latest_scans_queryset.values_list("id", flat=True),
|
||||
)
|
||||
categories_set = set()
|
||||
for categories_list in filtered_queryset.values_list(
|
||||
"categories", flat=True
|
||||
):
|
||||
if categories_list:
|
||||
categories_set.update(categories_list)
|
||||
categories = sorted(categories_set)
|
||||
|
||||
# Get groups from ScanGroupSummary for latest scans
|
||||
groups = list(
|
||||
ScanGroupSummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id__in=latest_scans_queryset.values_list("id", flat=True),
|
||||
)
|
||||
.values_list("resource_group", flat=True)
|
||||
.distinct()
|
||||
.order_by("resource_group")
|
||||
)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"resource_types": resource_types,
|
||||
"categories": categories,
|
||||
"groups": groups,
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
@@ -3901,7 +4201,7 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
# If we couldn't determine from database, try each provider type
|
||||
if not provider_type:
|
||||
for pt in Provider.ProviderChoices.values:
|
||||
if compliance_id in PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE.get(pt, {}):
|
||||
if compliance_id in get_compliance_frameworks(pt):
|
||||
provider_type = pt
|
||||
break
|
||||
|
||||
@@ -4026,32 +4326,43 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
summary="Get attack surface overview",
|
||||
description="Retrieve aggregated attack surface metrics from latest completed scans per provider.",
|
||||
tags=["Overview"],
|
||||
parameters=[
|
||||
OpenApiParameter(
|
||||
name="filter[provider_id]",
|
||||
type=OpenApiTypes.UUID,
|
||||
location=OpenApiParameter.QUERY,
|
||||
description="Filter by specific provider ID",
|
||||
),
|
||||
OpenApiParameter(
|
||||
name="filter[provider_id.in]",
|
||||
type=OpenApiTypes.STR,
|
||||
location=OpenApiParameter.QUERY,
|
||||
description="Filter by multiple provider IDs (comma-separated UUIDs)",
|
||||
),
|
||||
OpenApiParameter(
|
||||
name="filter[provider_type]",
|
||||
type=OpenApiTypes.STR,
|
||||
location=OpenApiParameter.QUERY,
|
||||
description="Filter by provider type (aws, azure, gcp, etc.)",
|
||||
),
|
||||
OpenApiParameter(
|
||||
name="filter[provider_type.in]",
|
||||
type=OpenApiTypes.STR,
|
||||
location=OpenApiParameter.QUERY,
|
||||
description="Filter by multiple provider types (comma-separated)",
|
||||
),
|
||||
],
|
||||
filters=True,
|
||||
responses={200: AttackSurfaceOverviewSerializer(many=True)},
|
||||
),
|
||||
categories=extend_schema(
|
||||
summary="Get category overview",
|
||||
description=(
|
||||
"Retrieve aggregated category metrics from latest completed scans per provider. "
|
||||
"Returns one row per category with total, failed, and new failed findings counts, "
|
||||
"plus a severity breakdown showing failed findings per severity level. "
|
||||
),
|
||||
tags=["Overview"],
|
||||
filters=True,
|
||||
responses={200: CategoryOverviewSerializer(many=True)},
|
||||
),
|
||||
resource_groups=extend_schema(
|
||||
summary="Get resource group overview",
|
||||
description=(
|
||||
"Retrieve aggregated resource group metrics from latest completed scans per provider. "
|
||||
"Returns one row per resource group with total, failed, and new failed findings counts, "
|
||||
"plus a severity breakdown showing failed findings per severity level, "
|
||||
"and a count of distinct resources evaluated per group."
|
||||
),
|
||||
tags=["Overview"],
|
||||
filters=True,
|
||||
responses={200: ResourceGroupOverviewSerializer(many=True)},
|
||||
),
|
||||
compliance_watchlist=extend_schema(
|
||||
summary="Get compliance watchlist overview",
|
||||
description=(
|
||||
"Retrieve compliance metrics with FAIL-dominant aggregation. "
|
||||
"Without filters: uses pre-aggregated TenantComplianceSummary. "
|
||||
"With provider filters: queries ProviderComplianceScore with FAIL-dominant logic "
|
||||
"where any FAIL in a requirement marks it as failed."
|
||||
),
|
||||
tags=["Overview"],
|
||||
filters=True,
|
||||
responses={200: ComplianceWatchlistOverviewSerializer(many=True)},
|
||||
),
|
||||
)
|
||||
@method_decorator(CACHE_DECORATOR, name="list")
|
||||
@@ -4100,6 +4411,12 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
return ThreatScoreSnapshotSerializer
|
||||
elif self.action == "attack_surface":
|
||||
return AttackSurfaceOverviewSerializer
|
||||
elif self.action == "categories":
|
||||
return CategoryOverviewSerializer
|
||||
elif self.action == "resource_groups":
|
||||
return ResourceGroupOverviewSerializer
|
||||
elif self.action == "compliance_watchlist":
|
||||
return ComplianceWatchlistOverviewSerializer
|
||||
return super().get_serializer_class()
|
||||
|
||||
def get_filterset_class(self):
|
||||
@@ -4111,6 +4428,14 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
return ScanSummarySeverityFilter
|
||||
elif self.action == "findings_severity_timeseries":
|
||||
return DailySeveritySummaryFilter
|
||||
elif self.action == "categories":
|
||||
return CategoryOverviewFilter
|
||||
elif self.action == "resource_groups":
|
||||
return ResourceGroupOverviewFilter
|
||||
elif self.action == "attack_surface":
|
||||
return AttackSurfaceOverviewFilter
|
||||
elif self.action == "compliance_watchlist":
|
||||
return ComplianceWatchlistFilter
|
||||
return None
|
||||
|
||||
def filter_queryset(self, queryset):
|
||||
@@ -4194,31 +4519,56 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
self.request.query_params, exclude_keys=set(exclude_keys or [])
|
||||
)
|
||||
filterset = filterset_class(normalized_params, queryset=queryset)
|
||||
if not filterset.is_valid():
|
||||
raise ValidationError(filterset.errors)
|
||||
return filterset.qs
|
||||
|
||||
def _latest_scan_ids_for_allowed_providers(self, tenant_id):
|
||||
def _latest_scan_ids_for_allowed_providers(self, tenant_id, provider_filters=None):
|
||||
provider_filter = self._get_provider_filter()
|
||||
queryset = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
if provider_filters:
|
||||
queryset = queryset.filter(**provider_filters)
|
||||
return (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
queryset.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
|
||||
def _attack_surface_check_ids_by_provider_types(self, provider_types):
|
||||
check_ids_by_type = {
|
||||
attack_surface_type: set()
|
||||
for attack_surface_type in AttackSurfaceOverview.AttackSurfaceTypeChoices.values
|
||||
}
|
||||
for provider_type in provider_types:
|
||||
attack_surface_mapping = _get_attack_surface_mapping_from_provider(
|
||||
provider_type=provider_type
|
||||
)
|
||||
for attack_surface_type, check_ids in attack_surface_mapping.items():
|
||||
check_ids_by_type[attack_surface_type].update(check_ids)
|
||||
return check_ids_by_type
|
||||
def _extract_provider_filters_from_params(self):
|
||||
"""Extract and validate provider filters from query params."""
|
||||
params = self.request.query_params
|
||||
filters = {}
|
||||
valid_provider_types = {c[0] for c in Provider.ProviderChoices.choices}
|
||||
|
||||
provider_id = params.get("filter[provider_id]")
|
||||
if provider_id:
|
||||
filters["provider_id"] = provider_id
|
||||
|
||||
provider_id_in = params.get("filter[provider_id__in]")
|
||||
if provider_id_in:
|
||||
filters["provider_id__in"] = provider_id_in.split(",")
|
||||
|
||||
provider_type = params.get("filter[provider_type]")
|
||||
if provider_type:
|
||||
if provider_type not in valid_provider_types:
|
||||
raise ValidationError(
|
||||
{"provider_type": f"Invalid choice: {provider_type}"}
|
||||
)
|
||||
filters["provider__provider"] = provider_type
|
||||
|
||||
provider_type_in = params.get("filter[provider_type__in]")
|
||||
if provider_type_in:
|
||||
types = provider_type_in.split(",")
|
||||
invalid = [t for t in types if t not in valid_provider_types]
|
||||
if invalid:
|
||||
raise ValidationError(
|
||||
{"provider_type__in": f"Invalid choices: {', '.join(invalid)}"}
|
||||
)
|
||||
filters["provider__provider__in"] = types
|
||||
|
||||
return filters
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="providers")
|
||||
def providers(self, request):
|
||||
@@ -4825,22 +5175,13 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
tenant_id = request.tenant_id
|
||||
latest_scan_ids = self._latest_scan_ids_for_allowed_providers(tenant_id)
|
||||
|
||||
# Build base queryset and apply user filters via FilterSet
|
||||
base_queryset = AttackSurfaceOverview.objects.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
filtered_queryset = self._apply_filterset(
|
||||
base_queryset, AttackSurfaceOverviewFilter
|
||||
)
|
||||
provider_types = list(
|
||||
filtered_queryset.values_list(
|
||||
"scan__provider__provider", flat=True
|
||||
).distinct()
|
||||
)
|
||||
attack_surface_check_ids = self._attack_surface_check_ids_by_provider_types(
|
||||
provider_types
|
||||
)
|
||||
# Aggregate attack surface data
|
||||
|
||||
aggregation = filtered_queryset.values("attack_surface_type").annotate(
|
||||
total_findings=Coalesce(Sum("total_findings"), 0),
|
||||
failed_findings=Coalesce(Sum("failed_findings"), 0),
|
||||
@@ -4863,12 +5204,7 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
}
|
||||
|
||||
response_data = [
|
||||
{
|
||||
"attack_surface_type": key,
|
||||
**value,
|
||||
"check_ids": attack_surface_check_ids.get(key, []),
|
||||
}
|
||||
for key, value in results.items()
|
||||
{"attack_surface_type": key, **value} for key, value in results.items()
|
||||
]
|
||||
|
||||
return Response(
|
||||
@@ -4876,6 +5212,245 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="categories")
|
||||
def categories(self, request):
|
||||
tenant_id = request.tenant_id
|
||||
provider_filters = self._extract_provider_filters_from_params()
|
||||
latest_scan_ids = self._latest_scan_ids_for_allowed_providers(
|
||||
tenant_id, provider_filters
|
||||
)
|
||||
|
||||
base_queryset = ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
provider_filter_keys = {
|
||||
"provider_id",
|
||||
"provider_id__in",
|
||||
"provider_type",
|
||||
"provider_type__in",
|
||||
}
|
||||
filtered_queryset = self._apply_filterset(
|
||||
base_queryset, CategoryOverviewFilter, exclude_keys=provider_filter_keys
|
||||
)
|
||||
|
||||
aggregation = (
|
||||
filtered_queryset.values("category", "severity")
|
||||
.annotate(
|
||||
total=Coalesce(Sum("total_findings"), 0),
|
||||
failed=Coalesce(Sum("failed_findings"), 0),
|
||||
new_failed=Coalesce(Sum("new_failed_findings"), 0),
|
||||
)
|
||||
.order_by("category", "severity")
|
||||
)
|
||||
|
||||
category_data = defaultdict(
|
||||
lambda: {
|
||||
"total_findings": 0,
|
||||
"failed_findings": 0,
|
||||
"new_failed_findings": 0,
|
||||
"severity": {
|
||||
"informational": 0,
|
||||
"low": 0,
|
||||
"medium": 0,
|
||||
"high": 0,
|
||||
"critical": 0,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
for row in aggregation:
|
||||
cat = row["category"]
|
||||
sev = row["severity"]
|
||||
category_data[cat]["total_findings"] += row["total"]
|
||||
category_data[cat]["failed_findings"] += row["failed"]
|
||||
category_data[cat]["new_failed_findings"] += row["new_failed"]
|
||||
if sev in category_data[cat]["severity"]:
|
||||
category_data[cat]["severity"][sev] = row["failed"]
|
||||
|
||||
response_data = [
|
||||
{"category": cat, **data} for cat, data in sorted(category_data.items())
|
||||
]
|
||||
|
||||
return Response(
|
||||
self.get_serializer(response_data, many=True).data,
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
@action(
|
||||
detail=False,
|
||||
methods=["get"],
|
||||
url_name="resource-groups",
|
||||
url_path="resource-groups",
|
||||
)
|
||||
def resource_groups(self, request):
|
||||
tenant_id = request.tenant_id
|
||||
provider_filters = self._extract_provider_filters_from_params()
|
||||
latest_scan_ids = self._latest_scan_ids_for_allowed_providers(
|
||||
tenant_id, provider_filters
|
||||
)
|
||||
|
||||
base_queryset = ScanGroupSummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
provider_filter_keys = {
|
||||
"provider_id",
|
||||
"provider_id__in",
|
||||
"provider_type",
|
||||
"provider_type__in",
|
||||
}
|
||||
filtered_queryset = self._apply_filterset(
|
||||
base_queryset,
|
||||
ResourceGroupOverviewFilter,
|
||||
exclude_keys=provider_filter_keys,
|
||||
)
|
||||
|
||||
aggregation = (
|
||||
filtered_queryset.values("resource_group", "severity")
|
||||
.annotate(
|
||||
total=Coalesce(Sum("total_findings"), 0),
|
||||
failed=Coalesce(Sum("failed_findings"), 0),
|
||||
new_failed=Coalesce(Sum("new_failed_findings"), 0),
|
||||
)
|
||||
.order_by("resource_group", "severity")
|
||||
)
|
||||
|
||||
# Get resource_group-level resources_count:
|
||||
# 1. Max per (scan, resource_group) to deduplicate within-scan severity rows
|
||||
# 2. Sum across scans for cross-provider aggregation
|
||||
scan_resource_group_resources = filtered_queryset.values(
|
||||
"scan_id", "resource_group"
|
||||
).annotate(resources=Coalesce(Max("resources_count"), 0))
|
||||
resources_by_resource_group = defaultdict(int)
|
||||
for row in scan_resource_group_resources:
|
||||
resources_by_resource_group[row["resource_group"]] += row["resources"]
|
||||
|
||||
resource_group_data = defaultdict(
|
||||
lambda: {
|
||||
"total_findings": 0,
|
||||
"failed_findings": 0,
|
||||
"new_failed_findings": 0,
|
||||
"resources_count": 0,
|
||||
"severity": {
|
||||
"informational": 0,
|
||||
"low": 0,
|
||||
"medium": 0,
|
||||
"high": 0,
|
||||
"critical": 0,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
for row in aggregation:
|
||||
grp = row["resource_group"]
|
||||
sev = row["severity"]
|
||||
resource_group_data[grp]["total_findings"] += row["total"]
|
||||
resource_group_data[grp]["failed_findings"] += row["failed"]
|
||||
resource_group_data[grp]["new_failed_findings"] += row["new_failed"]
|
||||
if sev in resource_group_data[grp]["severity"]:
|
||||
resource_group_data[grp]["severity"][sev] = row["failed"]
|
||||
|
||||
# Set resources_count from resource_group-level aggregation
|
||||
for grp in resource_group_data:
|
||||
resource_group_data[grp]["resources_count"] = (
|
||||
resources_by_resource_group.get(grp, 0)
|
||||
)
|
||||
|
||||
response_data = [
|
||||
{"resource_group": grp, **data}
|
||||
for grp, data in sorted(resource_group_data.items())
|
||||
]
|
||||
|
||||
return Response(
|
||||
self.get_serializer(response_data, many=True).data,
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
@action(
|
||||
detail=False,
|
||||
methods=["get"],
|
||||
url_name="compliance-watchlist",
|
||||
url_path="compliance-watchlist",
|
||||
)
|
||||
def compliance_watchlist(self, request):
|
||||
"""
|
||||
Get compliance watchlist overview with FAIL-dominant aggregation.
|
||||
|
||||
Without filters: uses pre-aggregated TenantComplianceSummary (~70 rows).
|
||||
With provider filters: queries ProviderComplianceScore with FAIL-dominant logic.
|
||||
"""
|
||||
tenant_id = request.tenant_id
|
||||
rbac_filter = self._get_provider_filter()
|
||||
query_params = request.query_params
|
||||
|
||||
has_provider_filter = any(
|
||||
key.startswith("filter[provider") for key in query_params.keys()
|
||||
)
|
||||
has_rbac_restriction = bool(rbac_filter)
|
||||
|
||||
if not has_provider_filter and not has_rbac_restriction:
|
||||
response_data = list(
|
||||
TenantComplianceSummary.objects.filter(tenant_id=tenant_id)
|
||||
.values(
|
||||
"compliance_id",
|
||||
"requirements_passed",
|
||||
"requirements_failed",
|
||||
"requirements_manual",
|
||||
"total_requirements",
|
||||
)
|
||||
.order_by("compliance_id")
|
||||
)
|
||||
else:
|
||||
base_queryset = ProviderComplianceScore.objects.filter(
|
||||
tenant_id=tenant_id, **rbac_filter
|
||||
)
|
||||
|
||||
filtered_queryset = self._apply_filterset(
|
||||
base_queryset, ComplianceWatchlistFilter
|
||||
)
|
||||
|
||||
aggregation = (
|
||||
filtered_queryset.values("compliance_id", "requirement_id")
|
||||
.annotate(
|
||||
has_fail=Sum(
|
||||
Case(When(requirement_status="FAIL", then=1), default=0)
|
||||
),
|
||||
has_manual=Sum(
|
||||
Case(When(requirement_status="MANUAL", then=1), default=0)
|
||||
),
|
||||
)
|
||||
.values("compliance_id", "requirement_id", "has_fail", "has_manual")
|
||||
)
|
||||
|
||||
compliance_data = defaultdict(
|
||||
lambda: {
|
||||
"requirements_passed": 0,
|
||||
"requirements_failed": 0,
|
||||
"requirements_manual": 0,
|
||||
"total_requirements": 0,
|
||||
}
|
||||
)
|
||||
|
||||
for row in aggregation:
|
||||
cid = row["compliance_id"]
|
||||
compliance_data[cid]["total_requirements"] += 1
|
||||
|
||||
if row["has_fail"] and row["has_fail"] > 0:
|
||||
compliance_data[cid]["requirements_failed"] += 1
|
||||
elif row["has_manual"] and row["has_manual"] > 0:
|
||||
compliance_data[cid]["requirements_manual"] += 1
|
||||
else:
|
||||
compliance_data[cid]["requirements_passed"] += 1
|
||||
|
||||
response_data = [
|
||||
{"compliance_id": cid, **data}
|
||||
for cid, data in sorted(compliance_data.items())
|
||||
]
|
||||
|
||||
return Response(
|
||||
self.get_serializer(response_data, many=True).data,
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["Schedule"])
|
||||
@extend_schema_view(
|
||||
@@ -5545,7 +6120,7 @@ class TenantApiKeyViewSet(BaseRLSViewSet):
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
def destroy(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="DESTROY")
|
||||
raise MethodNotAllowed(method="DELETE")
|
||||
|
||||
@action(detail=True, methods=["delete"])
|
||||
def revoke(self, request, *args, **kwargs):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import warnings
|
||||
|
||||
from celery import Celery, Task
|
||||
|
||||
from config.env import env
|
||||
|
||||
# Suppress specific warnings from django-rest-auth: https://github.com/iMerica/dj-rest-auth/issues/684
|
||||
|
||||
@@ -44,6 +44,12 @@ DATABASES = {
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
"neo4j": {
|
||||
"HOST": env.str("NEO4J_HOST", "neo4j"),
|
||||
"PORT": env.str("NEO4J_PORT", "7687"),
|
||||
"USER": env.str("NEO4J_USER", "neo4j"),
|
||||
"PASSWORD": env.str("NEO4J_PASSWORD", "neo4j_password"),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
@@ -45,6 +45,12 @@ DATABASES = {
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
"neo4j": {
|
||||
"HOST": env.str("NEO4J_HOST"),
|
||||
"PORT": env.str("NEO4J_PORT"),
|
||||
"USER": env.str("NEO4J_USER"),
|
||||
"PASSWORD": env.str("NEO4J_PASSWORD"),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
@@ -19,8 +19,6 @@ PORT = env("DJANGO_PORT", default=8000)
|
||||
|
||||
# Server settings
|
||||
bind = f"{BIND_ADDRESS}:{PORT}"
|
||||
# TODO: Remove after the category filter is implemented
|
||||
limit_request_line = 0
|
||||
|
||||
workers = env.int("DJANGO_WORKERS", default=multiprocessing.cpu_count() * 2 + 1)
|
||||
reload = DEBUG
|
||||
|
||||
+455
-8
@@ -1,8 +1,11 @@
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from allauth.socialaccount.models import SocialLogin
|
||||
from django.conf import settings
|
||||
from django.db import connection as django_connection
|
||||
@@ -11,10 +14,14 @@ from django.urls import reverse
|
||||
from django_celery_results.models import TaskResult
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
from tasks.jobs.backfill import backfill_resource_scan_summaries
|
||||
|
||||
from api.attack_paths import (
|
||||
AttackPathsQueryDefinition,
|
||||
AttackPathsQueryParameterDefinition,
|
||||
)
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
AttackSurfaceOverview,
|
||||
ComplianceOverview,
|
||||
ComplianceRequirementOverview,
|
||||
@@ -27,6 +34,7 @@ from api.models import (
|
||||
MuteRule,
|
||||
Processor,
|
||||
Provider,
|
||||
ProviderComplianceScore,
|
||||
ProviderGroup,
|
||||
ProviderSecret,
|
||||
Resource,
|
||||
@@ -36,11 +44,14 @@ from api.models import (
|
||||
SAMLConfiguration,
|
||||
SAMLDomainIndex,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
Task,
|
||||
TenantAPIKey,
|
||||
TenantComplianceSummary,
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
@@ -48,6 +59,11 @@ from api.rls import Tenant
|
||||
from api.v1.serializers import TokenSerializer
|
||||
from prowler.lib.check.models import Severity
|
||||
from prowler.lib.outputs.finding import Status
|
||||
from tasks.jobs.backfill import (
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
backfill_scan_resource_group_summaries,
|
||||
)
|
||||
|
||||
TODAY = str(datetime.today().date())
|
||||
API_JSON_CONTENT_TYPE = "application/vnd.api+json"
|
||||
@@ -160,22 +176,20 @@ def create_test_user_rbac_no_roles(django_db_setup, django_db_blocker, tenants_f
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def create_test_user_rbac_limited(django_db_setup, django_db_blocker):
|
||||
def create_test_user_rbac_limited(django_db_setup, django_db_blocker, tenants_fixture):
|
||||
with django_db_blocker.unblock():
|
||||
user = User.objects.create_user(
|
||||
name="testing_limited",
|
||||
email="rbac_limited@rbac.com",
|
||||
password=TEST_PASSWORD,
|
||||
)
|
||||
tenant = Tenant.objects.create(
|
||||
name="Tenant Test",
|
||||
)
|
||||
tenant = tenants_fixture[0]
|
||||
Membership.objects.create(
|
||||
user=user,
|
||||
tenant=tenant,
|
||||
role=Membership.RoleChoices.OWNER,
|
||||
)
|
||||
Role.objects.create(
|
||||
role = Role.objects.create(
|
||||
name="limited",
|
||||
tenant_id=tenant.id,
|
||||
manage_users=False,
|
||||
@@ -188,7 +202,7 @@ def create_test_user_rbac_limited(django_db_setup, django_db_blocker):
|
||||
)
|
||||
UserRoleRelationship.objects.create(
|
||||
user=user,
|
||||
role=Role.objects.get(name="limited"),
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
return user
|
||||
@@ -733,6 +747,7 @@ def resources_fixture(providers_fixture):
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="prowler-test",
|
||||
groups=["compute"],
|
||||
)
|
||||
|
||||
resource1.upsert_or_delete_tags(tags)
|
||||
@@ -745,6 +760,7 @@ def resources_fixture(providers_fixture):
|
||||
region="eu-west-1",
|
||||
service="s3",
|
||||
type="prowler-test",
|
||||
groups=["storage"],
|
||||
)
|
||||
resource2.upsert_or_delete_tags(tags)
|
||||
|
||||
@@ -756,6 +772,7 @@ def resources_fixture(providers_fixture):
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="test",
|
||||
groups=["compute"],
|
||||
)
|
||||
|
||||
tags = [
|
||||
@@ -1228,7 +1245,7 @@ def lighthouse_config_fixture(authenticated_client, tenants_fixture):
|
||||
return LighthouseConfiguration.objects.create(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
name="OpenAI",
|
||||
api_key_decoded="sk-test1234567890T3BlbkFJtest1234567890",
|
||||
api_key_decoded="sk-fake-test-key-for-unit-testing-only",
|
||||
model="gpt-4o",
|
||||
temperature=0,
|
||||
max_tokens=4000,
|
||||
@@ -1278,6 +1295,115 @@ def latest_scan_finding(authenticated_client, providers_fixture, resources_fixtu
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_categories(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource = resources_fixture[0]
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_with_categories_1",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="genai_check",
|
||||
check_metadata={"CheckId": "genai_check"},
|
||||
categories=["gen-ai", "security"],
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_multiple_categories(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource1, resource2 = resources_fixture[:2]
|
||||
|
||||
finding1 = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_multi_cat_1",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="genai_check",
|
||||
check_metadata={"CheckId": "genai_check"},
|
||||
categories=["gen-ai", "security"],
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding1.add_resources([resource1])
|
||||
|
||||
finding2 = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_multi_cat_2",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status 2",
|
||||
impact=Severity.high,
|
||||
impact_extended="test impact 2",
|
||||
severity=Severity.high,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="iam_check",
|
||||
check_metadata={"CheckId": "iam_check"},
|
||||
categories=["iam", "security"],
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding2.add_resources([resource2])
|
||||
|
||||
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
|
||||
return finding1, finding2
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def latest_scan_finding_with_categories(
|
||||
authenticated_client, providers_fixture, resources_fixture
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
tenant_id = str(providers_fixture[0].tenant_id)
|
||||
resource = resources_fixture[0]
|
||||
scan = Scan.objects.create(
|
||||
name="latest completed scan with categories",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
uid="latest_finding_with_categories",
|
||||
scan=scan,
|
||||
delta="new",
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="genai_iam_check",
|
||||
check_metadata={"CheckId": "genai_iam_check"},
|
||||
categories=["gen-ai", "iam"],
|
||||
resource_groups="ai_ml",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
backfill_resource_scan_summaries(tenant_id, str(scan.id))
|
||||
backfill_scan_category_summaries(tenant_id, str(scan.id))
|
||||
backfill_scan_resource_group_summaries(tenant_id, str(scan.id))
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def latest_scan_resource(authenticated_client, providers_fixture):
|
||||
provider = providers_fixture[0]
|
||||
@@ -1477,6 +1603,104 @@ def mute_rules_fixture(tenants_fixture, create_test_user, findings_fixture):
|
||||
return mute_rule1, mute_rule2
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_attack_paths_scan():
|
||||
"""Factory fixture to create Attack Paths scans for tests."""
|
||||
|
||||
def _create(
|
||||
provider,
|
||||
*,
|
||||
scan=None,
|
||||
state=StateChoices.COMPLETED,
|
||||
progress=0,
|
||||
graph_database="tenant-db",
|
||||
**extra_fields,
|
||||
):
|
||||
scan_instance = scan or Scan.objects.create(
|
||||
name=extra_fields.pop("scan_name", "Attack Paths Supporting Scan"),
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=extra_fields.pop("scan_state", StateChoices.COMPLETED),
|
||||
tenant_id=provider.tenant_id,
|
||||
)
|
||||
|
||||
payload = {
|
||||
"tenant_id": provider.tenant_id,
|
||||
"provider": provider,
|
||||
"scan": scan_instance,
|
||||
"state": state,
|
||||
"progress": progress,
|
||||
"graph_database": graph_database,
|
||||
}
|
||||
payload.update(extra_fields)
|
||||
|
||||
return AttackPathsScan.objects.create(**payload)
|
||||
|
||||
return _create
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def attack_paths_query_definition_factory():
|
||||
"""Factory fixture for building Attack Paths query definitions."""
|
||||
|
||||
def _create(**overrides):
|
||||
cast_type = overrides.pop("cast_type", str)
|
||||
parameters = overrides.pop(
|
||||
"parameters",
|
||||
[
|
||||
AttackPathsQueryParameterDefinition(
|
||||
name="limit",
|
||||
label="Limit",
|
||||
cast=cast_type,
|
||||
)
|
||||
],
|
||||
)
|
||||
definition_payload = {
|
||||
"id": "aws-test",
|
||||
"name": "Attack Paths Test Query",
|
||||
"description": "Synthetic Attack Paths definition for tests.",
|
||||
"provider": "aws",
|
||||
"cypher": "RETURN 1",
|
||||
"parameters": parameters,
|
||||
}
|
||||
definition_payload.update(overrides)
|
||||
return AttackPathsQueryDefinition(**definition_payload)
|
||||
|
||||
return _create
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def attack_paths_graph_stub_classes():
|
||||
"""Provide lightweight graph element stubs for Attack Paths serialization tests."""
|
||||
|
||||
class AttackPathsNativeValue:
|
||||
def __init__(self, value):
|
||||
self._value = value
|
||||
|
||||
def to_native(self):
|
||||
return self._value
|
||||
|
||||
class AttackPathsNode:
|
||||
def __init__(self, element_id, labels, properties):
|
||||
self.element_id = element_id
|
||||
self.labels = labels
|
||||
self._properties = properties
|
||||
|
||||
class AttackPathsRelationship:
|
||||
def __init__(self, element_id, rel_type, start_node, end_node, properties):
|
||||
self.element_id = element_id
|
||||
self.type = rel_type
|
||||
self.start_node = start_node
|
||||
self.end_node = end_node
|
||||
self._properties = properties
|
||||
|
||||
return SimpleNamespace(
|
||||
NativeValue=AttackPathsNativeValue,
|
||||
Node=AttackPathsNode,
|
||||
Relationship=AttackPathsRelationship,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_attack_surface_overview():
|
||||
def _create(tenant, scan, attack_surface_type, total=10, failed=5, muted_failed=2):
|
||||
@@ -1492,10 +1716,233 @@ def create_attack_surface_overview():
|
||||
return _create
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_scan_category_summary():
|
||||
def _create(
|
||||
tenant,
|
||||
scan,
|
||||
category,
|
||||
severity,
|
||||
total_findings=10,
|
||||
failed_findings=5,
|
||||
new_failed_findings=2,
|
||||
):
|
||||
return ScanCategorySummary.objects.create(
|
||||
tenant=tenant,
|
||||
scan=scan,
|
||||
category=category,
|
||||
severity=severity,
|
||||
total_findings=total_findings,
|
||||
failed_findings=failed_findings,
|
||||
new_failed_findings=new_failed_findings,
|
||||
)
|
||||
|
||||
return _create
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_group(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource = resources_fixture[0]
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_with_group_1",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="storage_check",
|
||||
check_metadata={"CheckId": "storage_check"},
|
||||
resource_groups="storage",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_multiple_groups(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource1, resource2 = resources_fixture[:2]
|
||||
|
||||
finding1 = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_multi_grp_1",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="storage_check",
|
||||
check_metadata={"CheckId": "storage_check"},
|
||||
resource_groups="storage",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding1.add_resources([resource1])
|
||||
|
||||
finding2 = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_multi_grp_2",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status 2",
|
||||
impact=Severity.high,
|
||||
impact_extended="test impact 2",
|
||||
severity=Severity.high,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="security_check",
|
||||
check_metadata={"CheckId": "security_check"},
|
||||
resource_groups="security",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding2.add_resources([resource2])
|
||||
|
||||
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
|
||||
return finding1, finding2
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_scan_resource_group_summary():
|
||||
def _create(
|
||||
tenant,
|
||||
scan,
|
||||
resource_group,
|
||||
severity,
|
||||
total_findings=10,
|
||||
failed_findings=5,
|
||||
new_failed_findings=2,
|
||||
resources_count=3,
|
||||
):
|
||||
return ScanGroupSummary.objects.create(
|
||||
tenant=tenant,
|
||||
scan=scan,
|
||||
resource_group=resource_group,
|
||||
severity=severity,
|
||||
total_findings=total_findings,
|
||||
failed_findings=failed_findings,
|
||||
new_failed_findings=new_failed_findings,
|
||||
resources_count=resources_count,
|
||||
)
|
||||
|
||||
return _create
|
||||
|
||||
|
||||
def get_authorization_header(access_token: str) -> dict:
|
||||
return {"Authorization": f"Bearer {access_token}"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def provider_compliance_scores_fixture(
|
||||
tenants_fixture, providers_fixture, scans_fixture
|
||||
):
|
||||
"""Create ProviderComplianceScore entries for compliance watchlist tests."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider1, provider2, *_ = providers_fixture
|
||||
scan1, _, scan3 = scans_fixture
|
||||
|
||||
scan1.completed_at = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
scan1.save()
|
||||
scan3.state = StateChoices.COMPLETED
|
||||
scan3.completed_at = datetime.now(timezone.utc)
|
||||
scan3.save()
|
||||
|
||||
scores = [
|
||||
ProviderComplianceScore.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider1,
|
||||
scan=scan1,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_1",
|
||||
requirement_status=StatusChoices.PASS,
|
||||
scan_completed_at=scan1.completed_at,
|
||||
),
|
||||
ProviderComplianceScore.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider1,
|
||||
scan=scan1,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_2",
|
||||
requirement_status=StatusChoices.FAIL,
|
||||
scan_completed_at=scan1.completed_at,
|
||||
),
|
||||
ProviderComplianceScore.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider1,
|
||||
scan=scan1,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_3",
|
||||
requirement_status=StatusChoices.MANUAL,
|
||||
scan_completed_at=scan1.completed_at,
|
||||
),
|
||||
ProviderComplianceScore.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider2,
|
||||
scan=scan3,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_1",
|
||||
requirement_status=StatusChoices.FAIL,
|
||||
scan_completed_at=scan3.completed_at,
|
||||
),
|
||||
ProviderComplianceScore.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider2,
|
||||
scan=scan3,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirement_id="req_2",
|
||||
requirement_status=StatusChoices.PASS,
|
||||
scan_completed_at=scan3.completed_at,
|
||||
),
|
||||
ProviderComplianceScore.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider1,
|
||||
scan=scan1,
|
||||
compliance_id="gdpr_aws",
|
||||
requirement_id="gdpr_req_1",
|
||||
requirement_status=StatusChoices.PASS,
|
||||
scan_completed_at=scan1.completed_at,
|
||||
),
|
||||
]
|
||||
|
||||
return scores
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tenant_compliance_summary_fixture(tenants_fixture):
|
||||
"""Create TenantComplianceSummary entries for compliance watchlist tests."""
|
||||
tenant = tenants_fixture[0]
|
||||
|
||||
summaries = [
|
||||
TenantComplianceSummary.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
compliance_id="aws_cis_2.0",
|
||||
requirements_passed=1,
|
||||
requirements_failed=2,
|
||||
requirements_manual=1,
|
||||
total_requirements=4,
|
||||
),
|
||||
TenantComplianceSummary.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
compliance_id="gdpr_aws",
|
||||
requirements_passed=5,
|
||||
requirements_failed=0,
|
||||
requirements_manual=2,
|
||||
total_requirements=7,
|
||||
),
|
||||
]
|
||||
|
||||
return summaries
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(items):
|
||||
"""Ensure test_rbac.py is executed first."""
|
||||
items.sort(key=lambda item: 0 if "test_rbac.py" in item.nodeid else 1)
|
||||
|
||||
@@ -7,6 +7,7 @@ from tasks.tasks import perform_scheduled_scan_task
|
||||
from api.db_utils import rls_transaction
|
||||
from api.exceptions import ConflictException
|
||||
from api.models import Provider, Scan, StateChoices
|
||||
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
|
||||
|
||||
|
||||
def schedule_provider_scan(provider_instance: Provider):
|
||||
@@ -39,6 +40,12 @@ def schedule_provider_scan(provider_instance: Provider):
|
||||
scheduled_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
attack_paths_db_utils.create_attack_paths_scan(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=str(scheduled_scan.id),
|
||||
provider_id=provider_id,
|
||||
)
|
||||
|
||||
# Schedule the task
|
||||
periodic_task_instance = PeriodicTask.objects.create(
|
||||
interval=schedule,
|
||||
@@ -61,4 +68,5 @@ def schedule_provider_scan(provider_instance: Provider):
|
||||
"tenant_id": str(provider_instance.tenant_id),
|
||||
"provider_id": provider_id,
|
||||
},
|
||||
countdown=5, # Avoid race conditions between the worker and the database
|
||||
)
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
from tasks.jobs.attack_paths.db_utils import can_provider_run_attack_paths_scan
|
||||
from tasks.jobs.attack_paths.scan import run as attack_paths_scan
|
||||
|
||||
__all__ = [
|
||||
"attack_paths_scan",
|
||||
"can_provider_run_attack_paths_scan",
|
||||
]
|
||||
@@ -0,0 +1,253 @@
|
||||
# Portions of this file are based on code from the Cartography project
|
||||
# (https://github.com/cartography-cncf/cartography), which is licensed under the Apache 2.0 License.
|
||||
|
||||
from typing import Any
|
||||
|
||||
import aioboto3
|
||||
import boto3
|
||||
import neo4j
|
||||
|
||||
from cartography.config import Config as CartographyConfig
|
||||
from cartography.intel import aws as cartography_aws
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from api.models import (
|
||||
AttackPathsScan as ProwlerAPIAttackPathsScan,
|
||||
Provider as ProwlerAPIProvider,
|
||||
)
|
||||
from prowler.providers.common.provider import Provider as ProwlerSDKProvider
|
||||
from tasks.jobs.attack_paths import db_utils, utils
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def start_aws_ingestion(
|
||||
neo4j_session: neo4j.Session,
|
||||
cartography_config: CartographyConfig,
|
||||
prowler_api_provider: ProwlerAPIProvider,
|
||||
prowler_sdk_provider: ProwlerSDKProvider,
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
) -> dict[str, dict[str, str]]:
|
||||
"""
|
||||
Code based on Cartography version 0.122.0, specifically on `cartography.intel.aws.__init__.py`.
|
||||
|
||||
For the scan progress updates:
|
||||
- The caller of this function (`tasks.jobs.attack_paths.scan.run`) has set it to 2.
|
||||
- When the control returns to the caller, it will be set to 95.
|
||||
"""
|
||||
|
||||
# Initialize variables common to all jobs
|
||||
common_job_parameters = {
|
||||
"UPDATE_TAG": cartography_config.update_tag,
|
||||
"permission_relationships_file": cartography_config.permission_relationships_file,
|
||||
"aws_guardduty_severity_threshold": cartography_config.aws_guardduty_severity_threshold,
|
||||
"aws_cloudtrail_management_events_lookback_hours": cartography_config.aws_cloudtrail_management_events_lookback_hours,
|
||||
"experimental_aws_inspector_batch": cartography_config.experimental_aws_inspector_batch,
|
||||
}
|
||||
|
||||
boto3_session = get_boto3_session(prowler_api_provider, prowler_sdk_provider)
|
||||
regions: list[str] = list(prowler_sdk_provider._enabled_regions)
|
||||
requested_syncs = list(cartography_aws.RESOURCE_FUNCTIONS.keys())
|
||||
|
||||
sync_args = cartography_aws._build_aws_sync_kwargs(
|
||||
neo4j_session,
|
||||
boto3_session,
|
||||
regions,
|
||||
prowler_api_provider.uid,
|
||||
cartography_config.update_tag,
|
||||
common_job_parameters,
|
||||
)
|
||||
|
||||
# Starting with sync functions
|
||||
logger.info(f"Syncing organizations for AWS account {prowler_api_provider.uid}")
|
||||
cartography_aws.organizations.sync(
|
||||
neo4j_session,
|
||||
{prowler_api_provider.alias: prowler_api_provider.uid},
|
||||
cartography_config.update_tag,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 3)
|
||||
|
||||
# Adding an extra field
|
||||
common_job_parameters["AWS_ID"] = prowler_api_provider.uid
|
||||
|
||||
cartography_aws._autodiscover_accounts(
|
||||
neo4j_session,
|
||||
boto3_session,
|
||||
prowler_api_provider.uid,
|
||||
cartography_config.update_tag,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 4)
|
||||
|
||||
failed_syncs = sync_aws_account(
|
||||
prowler_api_provider, requested_syncs, sync_args, attack_paths_scan
|
||||
)
|
||||
|
||||
if "permission_relationships" in requested_syncs:
|
||||
logger.info(
|
||||
f"Syncing function permission_relationships for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_aws.RESOURCE_FUNCTIONS["permission_relationships"](**sync_args)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 88)
|
||||
|
||||
if "resourcegroupstaggingapi" in requested_syncs:
|
||||
logger.info(
|
||||
f"Syncing function resourcegroupstaggingapi for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_aws.RESOURCE_FUNCTIONS["resourcegroupstaggingapi"](**sync_args)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 89)
|
||||
|
||||
logger.info(
|
||||
f"Syncing ec2_iaminstanceprofile scoped analysis for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_aws.run_scoped_analysis_job(
|
||||
"aws_ec2_iaminstanceprofile.json",
|
||||
neo4j_session,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 90)
|
||||
|
||||
logger.info(
|
||||
f"Syncing lambda_ecr analysis for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_aws.run_analysis_job(
|
||||
"aws_lambda_ecr.json",
|
||||
neo4j_session,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 91)
|
||||
|
||||
logger.info(f"Syncing metadata for AWS account {prowler_api_provider.uid}")
|
||||
cartography_aws.merge_module_sync_metadata(
|
||||
neo4j_session,
|
||||
group_type="AWSAccount",
|
||||
group_id=prowler_api_provider.uid,
|
||||
synced_type="AWSAccount",
|
||||
update_tag=cartography_config.update_tag,
|
||||
stat_handler=cartography_aws.stat_handler,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 92)
|
||||
|
||||
# Removing the added extra field
|
||||
del common_job_parameters["AWS_ID"]
|
||||
|
||||
logger.info(f"Syncing cleanup_job for AWS account {prowler_api_provider.uid}")
|
||||
cartography_aws.run_cleanup_job(
|
||||
"aws_post_ingestion_principals_cleanup.json",
|
||||
neo4j_session,
|
||||
common_job_parameters,
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 93)
|
||||
|
||||
logger.info(f"Syncing analysis for AWS account {prowler_api_provider.uid}")
|
||||
cartography_aws._perform_aws_analysis(
|
||||
requested_syncs, neo4j_session, common_job_parameters
|
||||
)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 94)
|
||||
|
||||
return failed_syncs
|
||||
|
||||
|
||||
def get_boto3_session(
|
||||
prowler_api_provider: ProwlerAPIProvider, prowler_sdk_provider: ProwlerSDKProvider
|
||||
) -> boto3.Session:
|
||||
boto3_session = prowler_sdk_provider.session.current_session
|
||||
|
||||
aws_accounts_from_session = cartography_aws.organizations.get_aws_account_default(
|
||||
boto3_session
|
||||
)
|
||||
if not aws_accounts_from_session:
|
||||
raise Exception(
|
||||
"No valid AWS credentials could be found. No AWS accounts can be synced."
|
||||
)
|
||||
|
||||
aws_account_id_from_session = list(aws_accounts_from_session.values())[0]
|
||||
if prowler_api_provider.uid != aws_account_id_from_session:
|
||||
raise Exception(
|
||||
f"Provider {prowler_api_provider.uid} doesn't match AWS account {aws_account_id_from_session}."
|
||||
)
|
||||
|
||||
if boto3_session.region_name is None:
|
||||
global_region = prowler_sdk_provider.get_global_region()
|
||||
boto3_session._session.set_config_variable("region", global_region)
|
||||
|
||||
return boto3_session
|
||||
|
||||
|
||||
def get_aioboto3_session(boto3_session: boto3.Session) -> aioboto3.Session:
|
||||
return aioboto3.Session(botocore_session=boto3_session._session)
|
||||
|
||||
|
||||
def sync_aws_account(
|
||||
prowler_api_provider: ProwlerAPIProvider,
|
||||
requested_syncs: list[str],
|
||||
sync_args: dict[str, Any],
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
) -> dict[str, str]:
|
||||
current_progress = 4 # `cartography_aws._autodiscover_accounts`
|
||||
max_progress = (
|
||||
87 # `cartography_aws.RESOURCE_FUNCTIONS["permission_relationships"]` - 1
|
||||
)
|
||||
n_steps = (
|
||||
len(requested_syncs) - 2
|
||||
) # Excluding `permission_relationships` and `resourcegroupstaggingapi`
|
||||
progress_step = (max_progress - current_progress) / n_steps
|
||||
|
||||
failed_syncs = {}
|
||||
|
||||
for func_name in requested_syncs:
|
||||
if func_name in cartography_aws.RESOURCE_FUNCTIONS:
|
||||
logger.info(
|
||||
f"Syncing function {func_name} for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
|
||||
# Updating progress, not really the right place but good enough
|
||||
current_progress += progress_step
|
||||
db_utils.update_attack_paths_scan_progress(
|
||||
attack_paths_scan, int(current_progress)
|
||||
)
|
||||
|
||||
try:
|
||||
# `ecr:image_layers` uses `aioboto3_session` instead of `boto3_session`
|
||||
if func_name == "ecr:image_layers":
|
||||
cartography_aws.RESOURCE_FUNCTIONS[func_name](
|
||||
neo4j_session=sync_args.get("neo4j_session"),
|
||||
aioboto3_session=get_aioboto3_session(
|
||||
sync_args.get("boto3_session")
|
||||
),
|
||||
regions=sync_args.get("regions"),
|
||||
current_aws_account_id=sync_args.get("current_aws_account_id"),
|
||||
update_tag=sync_args.get("update_tag"),
|
||||
common_job_parameters=sync_args.get("common_job_parameters"),
|
||||
)
|
||||
|
||||
# Skip permission relationships and tags for now because they rely on data already being in the graph
|
||||
elif func_name in [
|
||||
"permission_relationships",
|
||||
"resourcegroupstaggingapi",
|
||||
]:
|
||||
continue
|
||||
|
||||
else:
|
||||
cartography_aws.RESOURCE_FUNCTIONS[func_name](**sync_args)
|
||||
|
||||
except Exception as e:
|
||||
exception_message = utils.stringify_exception(
|
||||
e, f"Exception for AWS sync function: {func_name}"
|
||||
)
|
||||
failed_syncs[func_name] = exception_message
|
||||
|
||||
logger.warning(
|
||||
f"Caught exception syncing function {func_name} from AWS account {prowler_api_provider.uid}. We "
|
||||
"are continuing on to the next AWS sync function.",
|
||||
)
|
||||
|
||||
continue
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f'AWS sync function "{func_name}" was specified but does not exist. Did you misspell it?'
|
||||
)
|
||||
|
||||
return failed_syncs
|
||||
@@ -0,0 +1,168 @@
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from django.db.models import Q
|
||||
from cartography.config import Config as CartographyConfig
|
||||
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
AttackPathsScan as ProwlerAPIAttackPathsScan,
|
||||
Provider as ProwlerAPIProvider,
|
||||
StateChoices,
|
||||
)
|
||||
from tasks.jobs.attack_paths.providers import is_provider_available
|
||||
|
||||
|
||||
def can_provider_run_attack_paths_scan(tenant_id: str, provider_id: int) -> bool:
|
||||
with rls_transaction(tenant_id):
|
||||
prowler_api_provider = ProwlerAPIProvider.objects.get(id=provider_id)
|
||||
|
||||
return is_provider_available(prowler_api_provider.provider)
|
||||
|
||||
|
||||
def create_attack_paths_scan(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
provider_id: int,
|
||||
) -> ProwlerAPIAttackPathsScan | None:
|
||||
if not can_provider_run_attack_paths_scan(tenant_id, provider_id):
|
||||
return None
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
attack_paths_scan = ProwlerAPIAttackPathsScan.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
scan_id=scan_id,
|
||||
state=StateChoices.SCHEDULED,
|
||||
started_at=datetime.now(tz=timezone.utc),
|
||||
)
|
||||
attack_paths_scan.save()
|
||||
|
||||
return attack_paths_scan
|
||||
|
||||
|
||||
def retrieve_attack_paths_scan(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
) -> ProwlerAPIAttackPathsScan | None:
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
attack_paths_scan = ProwlerAPIAttackPathsScan.objects.get(
|
||||
scan_id=scan_id,
|
||||
)
|
||||
|
||||
return attack_paths_scan
|
||||
|
||||
except ProwlerAPIAttackPathsScan.DoesNotExist:
|
||||
return None
|
||||
|
||||
|
||||
def starting_attack_paths_scan(
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
task_id: str,
|
||||
cartography_config: CartographyConfig,
|
||||
) -> None:
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
attack_paths_scan.task_id = task_id
|
||||
attack_paths_scan.state = StateChoices.EXECUTING
|
||||
attack_paths_scan.started_at = datetime.now(tz=timezone.utc)
|
||||
attack_paths_scan.update_tag = cartography_config.update_tag
|
||||
attack_paths_scan.graph_database = cartography_config.neo4j_database
|
||||
|
||||
attack_paths_scan.save(
|
||||
update_fields=[
|
||||
"task_id",
|
||||
"state",
|
||||
"started_at",
|
||||
"update_tag",
|
||||
"graph_database",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def finish_attack_paths_scan(
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
state: StateChoices,
|
||||
ingestion_exceptions: dict[str, Any],
|
||||
) -> None:
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
duration = int((now - attack_paths_scan.started_at).total_seconds())
|
||||
|
||||
attack_paths_scan.state = state
|
||||
attack_paths_scan.progress = 100
|
||||
attack_paths_scan.completed_at = now
|
||||
attack_paths_scan.duration = duration
|
||||
attack_paths_scan.ingestion_exceptions = ingestion_exceptions
|
||||
|
||||
attack_paths_scan.save(
|
||||
update_fields=[
|
||||
"state",
|
||||
"progress",
|
||||
"completed_at",
|
||||
"duration",
|
||||
"ingestion_exceptions",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def update_attack_paths_scan_progress(
|
||||
attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
progress: int,
|
||||
) -> None:
|
||||
with rls_transaction(attack_paths_scan.tenant_id):
|
||||
attack_paths_scan.progress = progress
|
||||
attack_paths_scan.save(update_fields=["progress"])
|
||||
|
||||
|
||||
def get_old_attack_paths_scans(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
attack_paths_scan_id: str,
|
||||
) -> list[ProwlerAPIAttackPathsScan]:
|
||||
"""
|
||||
An `old_attack_paths_scan` is any `completed` Attack Paths scan for the same provider,
|
||||
with its graph database not deleted, excluding the current Attack Paths scan.
|
||||
"""
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
completed_scans_qs = (
|
||||
ProwlerAPIAttackPathsScan.objects.filter(
|
||||
provider_id=provider_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
is_graph_database_deleted=False,
|
||||
)
|
||||
.exclude(id=attack_paths_scan_id)
|
||||
.all()
|
||||
)
|
||||
|
||||
return list(completed_scans_qs)
|
||||
|
||||
|
||||
def update_old_attack_paths_scan(
|
||||
old_attack_paths_scan: ProwlerAPIAttackPathsScan,
|
||||
) -> None:
|
||||
with rls_transaction(old_attack_paths_scan.tenant_id):
|
||||
old_attack_paths_scan.is_graph_database_deleted = True
|
||||
old_attack_paths_scan.save(update_fields=["is_graph_database_deleted"])
|
||||
|
||||
|
||||
def get_provider_graph_database_names(tenant_id: str, provider_id: str) -> list[str]:
|
||||
"""
|
||||
Return existing graph database names for a tenant/provider.
|
||||
|
||||
Note: For accesing the `AttackPathsScan` we need to use `all_objects` manager because the provider is soft-deleted.
|
||||
"""
|
||||
with rls_transaction(tenant_id):
|
||||
graph_databases_names_qs = (
|
||||
ProwlerAPIAttackPathsScan.all_objects.filter(
|
||||
~Q(graph_database=""),
|
||||
graph_database__isnull=False,
|
||||
provider_id=provider_id,
|
||||
is_graph_database_deleted=False,
|
||||
)
|
||||
.values_list("graph_database", flat=True)
|
||||
.distinct()
|
||||
)
|
||||
|
||||
return list(graph_databases_names_qs)
|
||||
@@ -0,0 +1,23 @@
|
||||
AVAILABLE_PROVIDERS: list[str] = [
|
||||
"aws",
|
||||
]
|
||||
|
||||
ROOT_NODE_LABELS: dict[str, str] = {
|
||||
"aws": "AWSAccount",
|
||||
}
|
||||
|
||||
NODE_UID_FIELDS: dict[str, str] = {
|
||||
"aws": "arn",
|
||||
}
|
||||
|
||||
|
||||
def is_provider_available(provider_type: str) -> bool:
|
||||
return provider_type in AVAILABLE_PROVIDERS
|
||||
|
||||
|
||||
def get_root_node_label(provider_type: str) -> str:
|
||||
return ROOT_NODE_LABELS.get(provider_type, "UnknownProviderAccount")
|
||||
|
||||
|
||||
def get_node_uid_field(provider_type: str) -> str:
|
||||
return NODE_UID_FIELDS.get(provider_type, "UnknownProviderUID")
|
||||
@@ -0,0 +1,290 @@
|
||||
from collections import defaultdict
|
||||
from typing import Generator
|
||||
|
||||
import neo4j
|
||||
from cartography.client.core.tx import run_write_query
|
||||
from cartography.config import Config as CartographyConfig
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.env import env
|
||||
from tasks.jobs.attack_paths.providers import get_node_uid_field, get_root_node_label
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Finding, Provider, ResourceFindingMapping
|
||||
from prowler.config import config as ProwlerConfig
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
BATCH_SIZE = env.int("ATTACK_PATHS_FINDINGS_BATCH_SIZE", 1000)
|
||||
|
||||
INDEX_STATEMENTS = [
|
||||
"CREATE INDEX prowler_finding_id IF NOT EXISTS FOR (n:ProwlerFinding) ON (n.id);",
|
||||
"CREATE INDEX prowler_finding_provider_uid IF NOT EXISTS FOR (n:ProwlerFinding) ON (n.provider_uid);",
|
||||
"CREATE INDEX prowler_finding_lastupdated IF NOT EXISTS FOR (n:ProwlerFinding) ON (n.lastupdated);",
|
||||
"CREATE INDEX prowler_finding_check_id IF NOT EXISTS FOR (n:ProwlerFinding) ON (n.status);",
|
||||
]
|
||||
|
||||
INSERT_STATEMENT_TEMPLATE = """
|
||||
MATCH (account:__ROOT_NODE_LABEL__ {id: $provider_uid})
|
||||
UNWIND $findings_data AS finding_data
|
||||
|
||||
OPTIONAL MATCH (account)-->(resource_by_uid)
|
||||
WHERE resource_by_uid.__NODE_UID_FIELD__ = finding_data.resource_uid
|
||||
WITH account, finding_data, resource_by_uid
|
||||
|
||||
OPTIONAL MATCH (account)-->(resource_by_id)
|
||||
WHERE resource_by_uid IS NULL
|
||||
AND resource_by_id.id = finding_data.resource_uid
|
||||
WITH account, finding_data, COALESCE(resource_by_uid, resource_by_id) AS resource
|
||||
WHERE resource IS NOT NULL
|
||||
|
||||
MERGE (finding:ProwlerFinding {id: finding_data.id})
|
||||
ON CREATE SET
|
||||
finding.id = finding_data.id,
|
||||
finding.uid = finding_data.uid,
|
||||
finding.inserted_at = finding_data.inserted_at,
|
||||
finding.updated_at = finding_data.updated_at,
|
||||
finding.first_seen_at = finding_data.first_seen_at,
|
||||
finding.scan_id = finding_data.scan_id,
|
||||
finding.delta = finding_data.delta,
|
||||
finding.status = finding_data.status,
|
||||
finding.status_extended = finding_data.status_extended,
|
||||
finding.severity = finding_data.severity,
|
||||
finding.check_id = finding_data.check_id,
|
||||
finding.check_title = finding_data.check_title,
|
||||
finding.muted = finding_data.muted,
|
||||
finding.muted_reason = finding_data.muted_reason,
|
||||
finding.provider_uid = $provider_uid,
|
||||
finding.firstseen = timestamp(),
|
||||
finding.lastupdated = $last_updated,
|
||||
finding._module_name = 'cartography:prowler',
|
||||
finding._module_version = $prowler_version
|
||||
ON MATCH SET
|
||||
finding.status = finding_data.status,
|
||||
finding.status_extended = finding_data.status_extended,
|
||||
finding.lastupdated = $last_updated
|
||||
|
||||
MERGE (resource)-[rel:HAS_FINDING]->(finding)
|
||||
ON CREATE SET
|
||||
rel.provider_uid = $provider_uid,
|
||||
rel.firstseen = timestamp(),
|
||||
rel.lastupdated = $last_updated,
|
||||
rel._module_name = 'cartography:prowler',
|
||||
rel._module_version = $prowler_version
|
||||
ON MATCH SET
|
||||
rel.lastupdated = $last_updated
|
||||
"""
|
||||
|
||||
CLEANUP_STATEMENT = """
|
||||
MATCH (finding:ProwlerFinding {provider_uid: $provider_uid})
|
||||
WHERE finding.lastupdated < $last_updated
|
||||
|
||||
WITH finding LIMIT $batch_size
|
||||
|
||||
DETACH DELETE finding
|
||||
|
||||
RETURN COUNT(finding) AS deleted_findings_count
|
||||
"""
|
||||
|
||||
|
||||
def create_indexes(neo4j_session: neo4j.Session) -> None:
|
||||
"""
|
||||
Code based on Cartography version 0.122.0, specifically on `cartography.intel.create_indexes.run`.
|
||||
"""
|
||||
|
||||
logger.info("Creating indexes for Prowler Findings node types")
|
||||
for statement in INDEX_STATEMENTS:
|
||||
run_write_query(neo4j_session, statement)
|
||||
|
||||
|
||||
def analysis(
|
||||
neo4j_session: neo4j.Session,
|
||||
prowler_api_provider: Provider,
|
||||
scan_id: str,
|
||||
config: CartographyConfig,
|
||||
) -> None:
|
||||
findings_data = get_provider_last_scan_findings(prowler_api_provider, scan_id)
|
||||
load_findings(neo4j_session, findings_data, prowler_api_provider, config)
|
||||
cleanup_findings(neo4j_session, prowler_api_provider, config)
|
||||
|
||||
|
||||
def get_provider_last_scan_findings(
|
||||
prowler_api_provider: Provider,
|
||||
scan_id: str,
|
||||
) -> Generator[list[dict[str, str]], None, None]:
|
||||
"""
|
||||
Generator that yields batches of finding-resource pairs.
|
||||
|
||||
Two-step query approach per batch:
|
||||
1. Paginate findings for scan (single table, indexed by scan_id)
|
||||
2. Batch-fetch resource UIDs via mapping table (single join)
|
||||
3. Merge and yield flat structure for Neo4j
|
||||
|
||||
Memory efficient: never holds more than BATCH_SIZE findings in memory.
|
||||
"""
|
||||
|
||||
logger.info(
|
||||
f"Starting findings fetch for scan {scan_id} (tenant {prowler_api_provider.tenant_id}) with batch size {BATCH_SIZE}"
|
||||
)
|
||||
|
||||
iteration = 0
|
||||
last_id = None
|
||||
|
||||
while True:
|
||||
iteration += 1
|
||||
|
||||
with rls_transaction(prowler_api_provider.tenant_id, using=READ_REPLICA_ALIAS):
|
||||
# Use all_objects to avoid the ActiveProviderManager's implicit JOIN
|
||||
# through Scan -> Provider (to check is_deleted=False).
|
||||
# The provider is already validated as active in this context.
|
||||
qs = Finding.all_objects.filter(scan_id=scan_id).order_by("id")
|
||||
if last_id is not None:
|
||||
qs = qs.filter(id__gt=last_id)
|
||||
|
||||
findings_batch = list(
|
||||
qs.values(
|
||||
"id",
|
||||
"uid",
|
||||
"inserted_at",
|
||||
"updated_at",
|
||||
"first_seen_at",
|
||||
"scan_id",
|
||||
"delta",
|
||||
"status",
|
||||
"status_extended",
|
||||
"severity",
|
||||
"check_id",
|
||||
"check_metadata__checktitle",
|
||||
"muted",
|
||||
"muted_reason",
|
||||
)[:BATCH_SIZE]
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Iteration #{iteration} fetched {len(findings_batch)} findings"
|
||||
)
|
||||
|
||||
if not findings_batch:
|
||||
logger.info(
|
||||
f"No findings returned for iteration #{iteration}; stopping pagination"
|
||||
)
|
||||
break
|
||||
|
||||
last_id = findings_batch[-1]["id"]
|
||||
enriched_batch = _enrich_and_flatten_batch(findings_batch)
|
||||
|
||||
# Yield outside the transaction
|
||||
if enriched_batch:
|
||||
yield enriched_batch
|
||||
|
||||
logger.info(f"Finished fetching findings for scan {scan_id}")
|
||||
|
||||
|
||||
def _enrich_and_flatten_batch(
|
||||
findings_batch: list[dict],
|
||||
) -> list[dict[str, str]]:
|
||||
"""
|
||||
Fetch resource UIDs for a batch of findings and return flat structure.
|
||||
|
||||
One finding with 3 resources becomes 3 dicts (same output format as before).
|
||||
Must be called within an RLS transaction context.
|
||||
"""
|
||||
finding_ids = [f["id"] for f in findings_batch]
|
||||
|
||||
# Single join: mapping -> resource
|
||||
resource_mappings = ResourceFindingMapping.objects.filter(
|
||||
finding_id__in=finding_ids
|
||||
).values_list("finding_id", "resource__uid")
|
||||
|
||||
# Build finding_id -> [resource_uids] mapping
|
||||
finding_resources = defaultdict(list)
|
||||
for finding_id, resource_uid in resource_mappings:
|
||||
finding_resources[finding_id].append(resource_uid)
|
||||
|
||||
# Flatten: one dict per (finding, resource) pair
|
||||
results = []
|
||||
for f in findings_batch:
|
||||
resource_uids = finding_resources.get(f["id"], [])
|
||||
|
||||
if not resource_uids:
|
||||
continue
|
||||
|
||||
for resource_uid in resource_uids:
|
||||
results.append(
|
||||
{
|
||||
"resource_uid": str(resource_uid),
|
||||
"id": str(f["id"]),
|
||||
"uid": f["uid"],
|
||||
"inserted_at": f["inserted_at"],
|
||||
"updated_at": f["updated_at"],
|
||||
"first_seen_at": f["first_seen_at"],
|
||||
"scan_id": str(f["scan_id"]),
|
||||
"delta": f["delta"],
|
||||
"status": f["status"],
|
||||
"status_extended": f["status_extended"],
|
||||
"severity": f["severity"],
|
||||
"check_id": str(f["check_id"]),
|
||||
"check_title": f["check_metadata__checktitle"],
|
||||
"muted": f["muted"],
|
||||
"muted_reason": f["muted_reason"],
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def load_findings(
|
||||
neo4j_session: neo4j.Session,
|
||||
findings_batches: Generator[list[dict[str, str]], None, None],
|
||||
prowler_api_provider: Provider,
|
||||
config: CartographyConfig,
|
||||
) -> None:
|
||||
replacements = {
|
||||
"__ROOT_NODE_LABEL__": get_root_node_label(prowler_api_provider.provider),
|
||||
"__NODE_UID_FIELD__": get_node_uid_field(prowler_api_provider.provider),
|
||||
}
|
||||
query = INSERT_STATEMENT_TEMPLATE
|
||||
for replace_key, replace_value in replacements.items():
|
||||
query = query.replace(replace_key, replace_value)
|
||||
|
||||
parameters = {
|
||||
"provider_uid": str(prowler_api_provider.uid),
|
||||
"last_updated": config.update_tag,
|
||||
"prowler_version": ProwlerConfig.prowler_version,
|
||||
}
|
||||
|
||||
batch_num = 0
|
||||
total_records = 0
|
||||
for batch in findings_batches:
|
||||
batch_num += 1
|
||||
batch_size = len(batch)
|
||||
total_records += batch_size
|
||||
|
||||
parameters["findings_data"] = batch
|
||||
|
||||
logger.info(f"Loading findings batch {batch_num} ({batch_size} records)")
|
||||
neo4j_session.run(query, parameters)
|
||||
|
||||
logger.info(f"Finished loading {total_records} records in {batch_num} batches")
|
||||
|
||||
|
||||
def cleanup_findings(
|
||||
neo4j_session: neo4j.Session,
|
||||
prowler_api_provider: Provider,
|
||||
config: CartographyConfig,
|
||||
) -> None:
|
||||
parameters = {
|
||||
"provider_uid": str(prowler_api_provider.uid),
|
||||
"last_updated": config.update_tag,
|
||||
"batch_size": BATCH_SIZE,
|
||||
}
|
||||
|
||||
batch = 1
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
logger.info(f"Cleaning findings batch {batch}")
|
||||
|
||||
result = neo4j_session.run(CLEANUP_STATEMENT, parameters)
|
||||
|
||||
deleted_count = result.single().get("deleted_findings_count", 0)
|
||||
batch += 1
|
||||
@@ -0,0 +1,197 @@
|
||||
import logging
|
||||
import time
|
||||
import asyncio
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
from cartography.config import Config as CartographyConfig
|
||||
from cartography.intel import analysis as cartography_analysis
|
||||
from cartography.intel import create_indexes as cartography_create_indexes
|
||||
from cartography.intel import ontology as cartography_ontology
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
Provider as ProwlerAPIProvider,
|
||||
StateChoices,
|
||||
)
|
||||
from api.utils import initialize_prowler_provider
|
||||
from tasks.jobs.attack_paths import aws, db_utils, prowler, utils
|
||||
|
||||
# Without this Celery goes crazy with Cartography logging
|
||||
logging.getLogger("cartography").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
CARTOGRAPHY_INGESTION_FUNCTIONS: dict[str, Callable] = {
|
||||
"aws": aws.start_aws_ingestion,
|
||||
}
|
||||
|
||||
|
||||
def get_cartography_ingestion_function(provider_type: str) -> Callable | None:
|
||||
return CARTOGRAPHY_INGESTION_FUNCTIONS.get(provider_type)
|
||||
|
||||
|
||||
def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
|
||||
"""
|
||||
Code based on Cartography version 0.122.0, specifically on `cartography.cli.main`, `cartography.cli.CLI.main`,
|
||||
`cartography.sync.run_with_config` and `cartography.sync.Sync.run`.
|
||||
"""
|
||||
ingestion_exceptions = {} # This will hold any exceptions raised during ingestion
|
||||
|
||||
# Prowler necessary objects
|
||||
with rls_transaction(tenant_id):
|
||||
prowler_api_provider = ProwlerAPIProvider.objects.get(scan__pk=scan_id)
|
||||
prowler_sdk_provider = initialize_prowler_provider(prowler_api_provider)
|
||||
|
||||
# Attack Paths Scan necessary objects
|
||||
cartography_ingestion_function = get_cartography_ingestion_function(
|
||||
prowler_api_provider.provider
|
||||
)
|
||||
attack_paths_scan = db_utils.retrieve_attack_paths_scan(tenant_id, scan_id)
|
||||
|
||||
# Checks before starting the scan
|
||||
if not cartography_ingestion_function:
|
||||
ingestion_exceptions = {
|
||||
"global_error": f"Provider {prowler_api_provider.provider} is not supported for Attack Paths scans"
|
||||
}
|
||||
if attack_paths_scan:
|
||||
db_utils.finish_attack_paths_scan(
|
||||
attack_paths_scan, StateChoices.COMPLETED, ingestion_exceptions
|
||||
)
|
||||
|
||||
logger.warning(
|
||||
f"Provider {prowler_api_provider.provider} is not supported for Attack Paths scans"
|
||||
)
|
||||
return ingestion_exceptions
|
||||
|
||||
else:
|
||||
if not attack_paths_scan:
|
||||
logger.warning(
|
||||
f"No Attack Paths Scan found for scan {scan_id} and tenant {tenant_id}, let's create it then"
|
||||
)
|
||||
attack_paths_scan = db_utils.create_attack_paths_scan(
|
||||
tenant_id, scan_id, prowler_api_provider.id
|
||||
)
|
||||
|
||||
# While creating the Cartography configuration, attributes `neo4j_user` and `neo4j_password` are not really needed in this config object
|
||||
cartography_config = CartographyConfig(
|
||||
neo4j_uri=graph_database.get_uri(),
|
||||
neo4j_database=graph_database.get_database_name(attack_paths_scan.id),
|
||||
update_tag=int(time.time()),
|
||||
)
|
||||
|
||||
# Starting the Attack Paths scan
|
||||
db_utils.starting_attack_paths_scan(attack_paths_scan, task_id, cartography_config)
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f"Creating Neo4j database {cartography_config.neo4j_database} for tenant {prowler_api_provider.tenant_id}"
|
||||
)
|
||||
|
||||
graph_database.create_database(cartography_config.neo4j_database)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 1)
|
||||
|
||||
logger.info(
|
||||
f"Starting Cartography ({attack_paths_scan.id}) for "
|
||||
f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}"
|
||||
)
|
||||
with graph_database.get_session(
|
||||
cartography_config.neo4j_database
|
||||
) as neo4j_session:
|
||||
# Indexes creation
|
||||
cartography_create_indexes.run(neo4j_session, cartography_config)
|
||||
prowler.create_indexes(neo4j_session)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 2)
|
||||
|
||||
# The real scan, where iterates over cloud services
|
||||
ingestion_exceptions = _call_within_event_loop(
|
||||
cartography_ingestion_function,
|
||||
neo4j_session,
|
||||
cartography_config,
|
||||
prowler_api_provider,
|
||||
prowler_sdk_provider,
|
||||
attack_paths_scan,
|
||||
)
|
||||
|
||||
# Post-processing: Just keeping it to be more Cartography compliant
|
||||
logger.info(
|
||||
f"Syncing Cartography ontology for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_ontology.run(neo4j_session, cartography_config)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 95)
|
||||
|
||||
logger.info(
|
||||
f"Syncing Cartography analysis for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
cartography_analysis.run(neo4j_session, cartography_config)
|
||||
db_utils.update_attack_paths_scan_progress(attack_paths_scan, 96)
|
||||
|
||||
# Adding Prowler nodes and relationships
|
||||
logger.info(
|
||||
f"Syncing Prowler analysis for AWS account {prowler_api_provider.uid}"
|
||||
)
|
||||
prowler.analysis(
|
||||
neo4j_session, prowler_api_provider, scan_id, cartography_config
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Clearing Neo4j cache for database {cartography_config.neo4j_database}"
|
||||
)
|
||||
graph_database.clear_cache(cartography_config.neo4j_database)
|
||||
|
||||
logger.info(
|
||||
f"Completed Cartography ({attack_paths_scan.id}) for "
|
||||
f"{prowler_api_provider.provider.upper()} provider {prowler_api_provider.id}"
|
||||
)
|
||||
|
||||
# Handling databases changes
|
||||
old_attack_paths_scans = db_utils.get_old_attack_paths_scans(
|
||||
prowler_api_provider.tenant_id,
|
||||
prowler_api_provider.id,
|
||||
attack_paths_scan.id,
|
||||
)
|
||||
for old_attack_paths_scan in old_attack_paths_scans:
|
||||
graph_database.drop_database(old_attack_paths_scan.graph_database)
|
||||
db_utils.update_old_attack_paths_scan(old_attack_paths_scan)
|
||||
|
||||
db_utils.finish_attack_paths_scan(
|
||||
attack_paths_scan, StateChoices.COMPLETED, ingestion_exceptions
|
||||
)
|
||||
return ingestion_exceptions
|
||||
|
||||
except Exception as e:
|
||||
exception_message = utils.stringify_exception(e, "Cartography failed")
|
||||
logger.error(exception_message)
|
||||
ingestion_exceptions["global_cartography_error"] = exception_message
|
||||
|
||||
# Handling databases changes
|
||||
graph_database.drop_database(cartography_config.neo4j_database)
|
||||
db_utils.finish_attack_paths_scan(
|
||||
attack_paths_scan, StateChoices.FAILED, ingestion_exceptions
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def _call_within_event_loop(fn, *args, **kwargs):
|
||||
"""
|
||||
Cartography needs a running event loop, so assuming there is none (Celery task or even regular DRF endpoint),
|
||||
let's create a new one and set it as the current event loop for this thread.
|
||||
"""
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
asyncio.set_event_loop(loop)
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
finally:
|
||||
try:
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to shutdown async generators cleanly: {e}")
|
||||
|
||||
loop.close()
|
||||
asyncio.set_event_loop(None)
|
||||
@@ -0,0 +1,10 @@
|
||||
import traceback
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def stringify_exception(exception: Exception, context: str) -> str:
|
||||
timestamp = datetime.now(tz=timezone.utc)
|
||||
exception_traceback = traceback.TracebackException.from_exception(exception)
|
||||
traceback_string = "".join(exception_traceback.format())
|
||||
return f"{timestamp} - {context}\n{traceback_string}"
|
||||
@@ -1,21 +1,40 @@
|
||||
from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
|
||||
from django.db.models import Sum
|
||||
from celery.utils.log import get_task_logger
|
||||
from django.db.models import OuterRef, Subquery, Sum
|
||||
from django.utils import timezone
|
||||
from tasks.jobs.queries import (
|
||||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL,
|
||||
)
|
||||
from tasks.jobs.scan import aggregate_category_counts, aggregate_resource_group_counts
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.db_utils import (
|
||||
POSTGRES_TENANT_VAR,
|
||||
SET_CONFIG_QUERY,
|
||||
psycopg_connection,
|
||||
rls_transaction,
|
||||
)
|
||||
from api.models import (
|
||||
ComplianceOverviewSummary,
|
||||
ComplianceRequirementOverview,
|
||||
DailySeveritySummary,
|
||||
Finding,
|
||||
ProviderComplianceScore,
|
||||
Resource,
|
||||
ResourceFindingMapping,
|
||||
ResourceScanSummary,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def backfill_resource_scan_summaries(tenant_id: str, scan_id: str):
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
@@ -186,10 +205,6 @@ def backfill_daily_severity_summaries(tenant_id: str, days: int = None):
|
||||
Backfill DailySeveritySummary from completed scans.
|
||||
Groups by provider+date, keeps latest scan per day.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
created_count = 0
|
||||
updated_count = 0
|
||||
|
||||
@@ -276,3 +291,264 @@ def backfill_daily_severity_summaries(tenant_id: str, days: int = None):
|
||||
"updated": updated_count,
|
||||
"total_days": len(latest_scans_by_day),
|
||||
}
|
||||
|
||||
|
||||
def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanCategorySummary for a completed scan.
|
||||
|
||||
Aggregates category counts from all findings in the scan and creates
|
||||
one ScanCategorySummary row per (category, severity) combination.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID
|
||||
scan_id: Scan UUID to backfill
|
||||
|
||||
Returns:
|
||||
dict: Status indicating whether backfill was performed
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
if ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).exists():
|
||||
return {"status": "already backfilled"}
|
||||
|
||||
if not Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id=scan_id,
|
||||
state__in=(StateChoices.COMPLETED, StateChoices.FAILED),
|
||||
).exists():
|
||||
return {"status": "scan is not completed"}
|
||||
|
||||
category_counts: dict[tuple[str, str], dict[str, int]] = {}
|
||||
for finding in Finding.all_objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).values("categories", "severity", "status", "delta", "muted"):
|
||||
aggregate_category_counts(
|
||||
categories=finding.get("categories") or [],
|
||||
severity=finding.get("severity"),
|
||||
status=finding.get("status"),
|
||||
delta=finding.get("delta"),
|
||||
muted=finding.get("muted", False),
|
||||
cache=category_counts,
|
||||
)
|
||||
|
||||
if not category_counts:
|
||||
return {"status": "no categories to backfill"}
|
||||
|
||||
category_summaries = [
|
||||
ScanCategorySummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
category=category,
|
||||
severity=severity,
|
||||
total_findings=counts["total"],
|
||||
failed_findings=counts["failed"],
|
||||
new_failed_findings=counts["new_failed"],
|
||||
)
|
||||
for (category, severity), counts in category_counts.items()
|
||||
]
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
ScanCategorySummary.objects.bulk_create(
|
||||
category_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
|
||||
return {"status": "backfilled", "categories_count": len(category_counts)}
|
||||
|
||||
|
||||
def backfill_scan_resource_group_summaries(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanGroupSummary for a completed scan.
|
||||
|
||||
Aggregates resource group counts from all findings in the scan and creates
|
||||
one ScanGroupSummary row per (resource_group, severity) combination.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID
|
||||
scan_id: Scan UUID to backfill
|
||||
|
||||
Returns:
|
||||
dict: Status indicating whether backfill was performed
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
if ScanGroupSummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).exists():
|
||||
return {"status": "already backfilled"}
|
||||
|
||||
if not Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id=scan_id,
|
||||
state__in=(StateChoices.COMPLETED, StateChoices.FAILED),
|
||||
).exists():
|
||||
return {"status": "scan is not completed"}
|
||||
|
||||
resource_group_counts: dict[tuple[str, str], dict[str, int]] = {}
|
||||
group_resources_cache: dict[str, set] = {}
|
||||
# Get findings with their first resource UID via annotation
|
||||
resource_uid_subquery = ResourceFindingMapping.objects.filter(
|
||||
finding_id=OuterRef("id"), tenant_id=tenant_id
|
||||
).values("resource__uid")[:1]
|
||||
|
||||
for finding in (
|
||||
Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id)
|
||||
.annotate(resource_uid=Subquery(resource_uid_subquery))
|
||||
.values(
|
||||
"resource_groups",
|
||||
"severity",
|
||||
"status",
|
||||
"delta",
|
||||
"muted",
|
||||
"resource_uid",
|
||||
)
|
||||
):
|
||||
aggregate_resource_group_counts(
|
||||
resource_group=finding.get("resource_groups"),
|
||||
severity=finding.get("severity"),
|
||||
status=finding.get("status"),
|
||||
delta=finding.get("delta"),
|
||||
muted=finding.get("muted", False),
|
||||
resource_uid=finding.get("resource_uid") or "",
|
||||
cache=resource_group_counts,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
if not resource_group_counts:
|
||||
return {"status": "no resource groups to backfill"}
|
||||
|
||||
# Compute group-level resource counts (same value for all severity rows in a group)
|
||||
group_resource_counts = {
|
||||
grp: len(uids) for grp, uids in group_resources_cache.items()
|
||||
}
|
||||
resource_group_summaries = [
|
||||
ScanGroupSummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
resource_group=grp,
|
||||
severity=severity,
|
||||
total_findings=counts["total"],
|
||||
failed_findings=counts["failed"],
|
||||
new_failed_findings=counts["new_failed"],
|
||||
resources_count=group_resource_counts.get(grp, 0),
|
||||
)
|
||||
for (grp, severity), counts in resource_group_counts.items()
|
||||
]
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
ScanGroupSummary.objects.bulk_create(
|
||||
resource_group_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
|
||||
return {"status": "backfilled", "resource_groups_count": len(resource_group_counts)}
|
||||
|
||||
|
||||
def backfill_provider_compliance_scores(tenant_id: str) -> dict:
|
||||
"""
|
||||
Backfill ProviderComplianceScore from latest completed scan per provider.
|
||||
|
||||
For each provider with completed scans, finds the most recent scan and
|
||||
upserts compliance requirement statuses with FAIL-dominant aggregation.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID
|
||||
|
||||
Returns:
|
||||
dict: Statistics about the backfill operation
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
completed_scans = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
completed_at__isnull=False,
|
||||
)
|
||||
if not completed_scans.exists():
|
||||
return {"status": "no completed scans"}
|
||||
|
||||
existing_providers = set(
|
||||
ProviderComplianceScore.objects.filter(tenant_id=tenant_id)
|
||||
.values_list("provider_id", flat=True)
|
||||
.distinct()
|
||||
)
|
||||
|
||||
if existing_providers:
|
||||
completed_scans = completed_scans.exclude(
|
||||
provider_id__in=existing_providers
|
||||
)
|
||||
|
||||
scan_info = list(
|
||||
completed_scans.order_by("provider_id", "-completed_at")
|
||||
.distinct("provider_id")
|
||||
.values("id", "provider_id", "completed_at")
|
||||
)
|
||||
|
||||
if not scan_info:
|
||||
return {"status": "no scans to process"}
|
||||
|
||||
total_upserted = 0
|
||||
providers_processed = 0
|
||||
providers_skipped = 0
|
||||
|
||||
for scan in scan_info:
|
||||
provider_id = scan["provider_id"]
|
||||
|
||||
scan_id = scan["id"]
|
||||
|
||||
try:
|
||||
with psycopg_connection(MainRouter.default_db) as connection:
|
||||
connection.autocommit = False
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id]
|
||||
)
|
||||
cursor.execute(
|
||||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
|
||||
[tenant_id, str(scan_id)],
|
||||
)
|
||||
upserted = cursor.rowcount
|
||||
connection.commit()
|
||||
total_upserted += upserted
|
||||
providers_processed += 1
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
except Exception as e:
|
||||
providers_skipped += 1
|
||||
logger.exception(
|
||||
"Error backfilling provider %s for tenant %s: %s",
|
||||
provider_id,
|
||||
tenant_id,
|
||||
e,
|
||||
)
|
||||
|
||||
# Recalculate tenant summary after all providers are backfilled
|
||||
if providers_processed > 0:
|
||||
with psycopg_connection(MainRouter.default_db) as connection:
|
||||
connection.autocommit = False
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id])
|
||||
# Advisory lock to prevent race conditions
|
||||
cursor.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtext(%s))", [tenant_id]
|
||||
)
|
||||
cursor.execute(
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL,
|
||||
[tenant_id, tenant_id],
|
||||
)
|
||||
tenant_summary_count = cursor.rowcount
|
||||
connection.commit()
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
else:
|
||||
tenant_summary_count = 0
|
||||
|
||||
return {
|
||||
"status": "backfilled",
|
||||
"providers_processed": providers_processed,
|
||||
"providers_skipped": providers_skipped,
|
||||
"total_upserted": total_upserted,
|
||||
"tenant_summary_count": tenant_summary_count,
|
||||
}
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
from celery.utils.log import get_task_logger
|
||||
from django.db import DatabaseError
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_router import MainRouter
|
||||
from api.db_utils import batch_delete, rls_transaction
|
||||
from api.models import Finding, Provider, Resource, Scan, ScanSummary, Tenant
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
Finding,
|
||||
Provider,
|
||||
Resource,
|
||||
Scan,
|
||||
ScanSummary,
|
||||
Tenant,
|
||||
)
|
||||
from tasks.jobs.attack_paths.db_utils import get_provider_graph_database_names
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@@ -23,16 +33,27 @@ def delete_provider(tenant_id: str, pk: str):
|
||||
Raises:
|
||||
Provider.DoesNotExist: If no instance with the provided primary key exists.
|
||||
"""
|
||||
# Delete the Attack Paths' graph databases related to the provider
|
||||
graph_database_names = get_provider_graph_database_names(tenant_id, pk)
|
||||
try:
|
||||
for graph_database_name in graph_database_names:
|
||||
graph_database.drop_database(graph_database_name)
|
||||
except graph_database.GraphDatabaseQueryException as gdb_error:
|
||||
logger.error(f"Error deleting Provider databases: {gdb_error}")
|
||||
raise
|
||||
|
||||
# Get all provider related data and delete them in batches
|
||||
with rls_transaction(tenant_id):
|
||||
instance = Provider.all_objects.get(pk=pk)
|
||||
deletion_summary = {}
|
||||
deletion_steps = [
|
||||
("Scan Summaries", ScanSummary.all_objects.filter(scan__provider=instance)),
|
||||
("Findings", Finding.all_objects.filter(scan__provider=instance)),
|
||||
("Resources", Resource.all_objects.filter(provider=instance)),
|
||||
("Scans", Scan.all_objects.filter(provider=instance)),
|
||||
("AttackPathsScans", AttackPathsScan.all_objects.filter(provider=instance)),
|
||||
]
|
||||
|
||||
deletion_summary = {}
|
||||
for step_name, queryset in deletion_steps:
|
||||
try:
|
||||
_, step_summary = batch_delete(tenant_id, queryset)
|
||||
@@ -48,6 +69,7 @@ def delete_provider(tenant_id: str, pk: str):
|
||||
except DatabaseError as db_error:
|
||||
logger.error(f"Error deleting Provider: {db_error}")
|
||||
raise
|
||||
|
||||
return deletion_summary
|
||||
|
||||
|
||||
|
||||
@@ -51,6 +51,9 @@ from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_azure import (
|
||||
AzureMitreAttack,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_gcp import GCPMitreAttack
|
||||
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_alibaba import (
|
||||
ProwlerThreatScoreAlibaba,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_aws import (
|
||||
ProwlerThreatScoreAWS,
|
||||
)
|
||||
@@ -131,6 +134,10 @@ COMPLIANCE_CLASS_MAP = {
|
||||
],
|
||||
"alibabacloud": [
|
||||
(lambda name: name.startswith("cis_"), AlibabaCloudCIS),
|
||||
(
|
||||
lambda name: name == "prowler_threatscore_alibabacloud",
|
||||
ProwlerThreatScoreAlibaba,
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,9 @@ from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.aws.lib.s3.s3 import S3
|
||||
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub
|
||||
from prowler.providers.common.models import Connection
|
||||
from prowler.providers.aws.lib.security_hub.exceptions.exceptions import (
|
||||
SecurityHubNoEnabledRegionsError,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@@ -222,8 +225,9 @@ def get_security_hub_client_from_integration(
|
||||
)
|
||||
return True, security_hub
|
||||
else:
|
||||
# Reset regions information if connection fails
|
||||
# Reset regions information if connection fails and integration is not connected
|
||||
with rls_transaction(tenant_id, using=MainRouter.default_db):
|
||||
integration.connected = False
|
||||
integration.configuration["regions"] = {}
|
||||
integration.save()
|
||||
|
||||
@@ -330,15 +334,18 @@ def upload_security_hub_integration(
|
||||
)
|
||||
|
||||
if not connected:
|
||||
logger.error(
|
||||
f"Security Hub connection failed for integration {integration.id}: "
|
||||
f"{security_hub.error}"
|
||||
)
|
||||
with rls_transaction(
|
||||
tenant_id, using=MainRouter.default_db
|
||||
if isinstance(
|
||||
security_hub.error,
|
||||
SecurityHubNoEnabledRegionsError,
|
||||
):
|
||||
integration.connected = False
|
||||
integration.save()
|
||||
logger.warning(
|
||||
f"Security Hub integration {integration.id} has no enabled regions"
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"Security Hub connection failed for integration {integration.id}: "
|
||||
f"{security_hub.error}"
|
||||
)
|
||||
break # Skip this integration
|
||||
|
||||
security_hub_client = security_hub
|
||||
@@ -409,22 +416,16 @@ def upload_security_hub_integration(
|
||||
logger.warning(
|
||||
f"Failed to archive previous findings: {str(archive_error)}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Security Hub integration {integration.id} failed: {str(e)}"
|
||||
)
|
||||
continue
|
||||
|
||||
result = integration_executions == len(integrations)
|
||||
if result:
|
||||
logger.info(
|
||||
f"All Security Hub integrations completed successfully for provider {provider_id}"
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"Some Security Hub integrations failed for provider {provider_id}"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -11,6 +11,41 @@ from api.models import LighthouseProviderConfiguration, LighthouseProviderModels
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# OpenAI model prefixes to exclude from Lighthouse model selection.
|
||||
# These models don't support text chat completions and tool calling.
|
||||
EXCLUDED_OPENAI_MODEL_PREFIXES = (
|
||||
"dall-e", # Image generation
|
||||
"whisper", # Audio transcription
|
||||
"tts-", # Text-to-speech (tts-1, tts-1-hd, etc.)
|
||||
"sora", # Text-to-video (sora-2, sora-2-pro, etc.)
|
||||
"text-embedding", # Embeddings
|
||||
"embedding", # Embeddings (alternative naming)
|
||||
"text-moderation", # Content moderation
|
||||
"omni-moderation", # Content moderation
|
||||
"text-davinci", # Legacy completion models
|
||||
"text-curie", # Legacy completion models
|
||||
"text-babbage", # Legacy completion models
|
||||
"text-ada", # Legacy completion models
|
||||
"davinci", # Legacy completion models
|
||||
"curie", # Legacy completion models
|
||||
"babbage", # Legacy completion models
|
||||
"ada", # Legacy completion models
|
||||
"computer-use", # Computer control agent
|
||||
"gpt-image", # Image generation
|
||||
"gpt-audio", # Audio models
|
||||
"gpt-realtime", # Realtime voice API
|
||||
)
|
||||
|
||||
# OpenAI model substrings to exclude (patterns that can appear anywhere in model ID).
|
||||
# These patterns identify non-chat model variants.
|
||||
EXCLUDED_OPENAI_MODEL_SUBSTRINGS = (
|
||||
"-audio-", # Audio preview models (gpt-4o-audio-preview, etc.)
|
||||
"-realtime-", # Realtime preview models (gpt-4o-realtime-preview, etc.)
|
||||
"-transcribe", # Transcription models (gpt-4o-transcribe, etc.)
|
||||
"-tts", # TTS models (gpt-4o-mini-tts)
|
||||
"-instruct", # Legacy instruct models (gpt-3.5-turbo-instruct, etc.)
|
||||
)
|
||||
|
||||
|
||||
def _extract_error_message(e: Exception) -> str:
|
||||
"""
|
||||
@@ -283,20 +318,41 @@ def _fetch_openai_models(api_key: str) -> Dict[str, str]:
|
||||
"""
|
||||
Fetch available models from OpenAI API.
|
||||
|
||||
Filters out models that don't support text input/output and tool calling,
|
||||
such as image generation (DALL-E), audio transcription (Whisper),
|
||||
text-to-speech (TTS), embeddings, and moderation models.
|
||||
|
||||
Args:
|
||||
api_key: OpenAI API key for authentication.
|
||||
|
||||
Returns:
|
||||
Dict mapping model_id to model_name. For OpenAI, both are the same
|
||||
as the API doesn't provide separate display names.
|
||||
as the API doesn't provide separate display names. Only includes
|
||||
models that support text input, text output or tool calling.
|
||||
|
||||
Raises:
|
||||
Exception: If the API call fails.
|
||||
"""
|
||||
client = openai.OpenAI(api_key=api_key)
|
||||
models = client.models.list()
|
||||
# OpenAI uses model.id for both ID and display name
|
||||
return {m.id: m.id for m in getattr(models, "data", [])}
|
||||
|
||||
# Filter models to only include those supporting chat completions + tool calling
|
||||
filtered_models = {}
|
||||
for model in getattr(models, "data", []):
|
||||
model_id = model.id
|
||||
|
||||
# Skip if model ID starts with excluded prefixes
|
||||
if model_id.startswith(EXCLUDED_OPENAI_MODEL_PREFIXES):
|
||||
continue
|
||||
|
||||
# Skip if model ID contains excluded substrings
|
||||
if any(substring in model_id for substring in EXCLUDED_OPENAI_MODEL_SUBSTRINGS):
|
||||
continue
|
||||
|
||||
# Include model (supports chat completions + tool calling)
|
||||
filtered_models[model_id] = model_id
|
||||
|
||||
return filtered_models
|
||||
|
||||
|
||||
def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, str]:
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""
|
||||
Shared SQL queries for tasks.
|
||||
|
||||
This module centralizes raw SQL queries used across multiple task modules
|
||||
to ensure consistency and maintainability.
|
||||
"""
|
||||
|
||||
# =============================================================================
|
||||
# COMPLIANCE SCORE QUERIES
|
||||
# =============================================================================
|
||||
|
||||
# Upsert provider compliance scores from a scan's compliance requirements.
|
||||
# Uses FAIL-dominant aggregation: FAIL > MANUAL > PASS
|
||||
# Parameters: [tenant_id, scan_id]
|
||||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL = """
|
||||
INSERT INTO provider_compliance_scores
|
||||
(id, tenant_id, provider_id, scan_id, compliance_id, requirement_id,
|
||||
requirement_status, scan_completed_at)
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
agg.tenant_id,
|
||||
agg.provider_id,
|
||||
agg.scan_id,
|
||||
agg.compliance_id,
|
||||
agg.requirement_id,
|
||||
agg.requirement_status,
|
||||
agg.completed_at
|
||||
FROM (
|
||||
SELECT DISTINCT ON (cro.compliance_id, cro.requirement_id)
|
||||
cro.tenant_id,
|
||||
s.provider_id,
|
||||
cro.scan_id,
|
||||
cro.compliance_id,
|
||||
cro.requirement_id,
|
||||
(CASE
|
||||
WHEN bool_or(cro.requirement_status = 'FAIL')
|
||||
OVER (PARTITION BY cro.compliance_id, cro.requirement_id) THEN 'FAIL'
|
||||
WHEN bool_or(cro.requirement_status = 'MANUAL')
|
||||
OVER (PARTITION BY cro.compliance_id, cro.requirement_id) THEN 'MANUAL'
|
||||
ELSE 'PASS'
|
||||
END)::status as requirement_status,
|
||||
s.completed_at
|
||||
FROM compliance_requirements_overviews cro
|
||||
JOIN scans s ON s.id = cro.scan_id
|
||||
WHERE cro.tenant_id = %s AND cro.scan_id = %s
|
||||
ORDER BY cro.compliance_id, cro.requirement_id
|
||||
) agg
|
||||
ON CONFLICT (tenant_id, provider_id, compliance_id, requirement_id)
|
||||
DO UPDATE SET
|
||||
requirement_status = EXCLUDED.requirement_status,
|
||||
scan_id = EXCLUDED.scan_id,
|
||||
scan_completed_at = EXCLUDED.scan_completed_at
|
||||
WHERE EXCLUDED.scan_completed_at > provider_compliance_scores.scan_completed_at
|
||||
"""
|
||||
|
||||
# Upsert tenant compliance summary for specific compliance IDs.
|
||||
# Aggregates across all providers with FAIL-dominant logic at requirement level.
|
||||
# Parameters: [tenant_id, tenant_id, compliance_ids_array]
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL = """
|
||||
INSERT INTO tenant_compliance_summaries
|
||||
(id, tenant_id, compliance_id,
|
||||
requirements_passed, requirements_failed, requirements_manual,
|
||||
total_requirements, updated_at)
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
%s as tenant_id,
|
||||
compliance_id,
|
||||
COUNT(*) FILTER (WHERE req_status = 'PASS') as requirements_passed,
|
||||
COUNT(*) FILTER (WHERE req_status = 'FAIL') as requirements_failed,
|
||||
COUNT(*) FILTER (WHERE req_status = 'MANUAL') as requirements_manual,
|
||||
COUNT(*) as total_requirements,
|
||||
NOW() as updated_at
|
||||
FROM (
|
||||
SELECT
|
||||
compliance_id,
|
||||
requirement_id,
|
||||
CASE
|
||||
WHEN bool_or(requirement_status = 'FAIL') THEN 'FAIL'
|
||||
WHEN bool_or(requirement_status = 'MANUAL') THEN 'MANUAL'
|
||||
ELSE 'PASS'
|
||||
END as req_status
|
||||
FROM provider_compliance_scores
|
||||
WHERE tenant_id = %s AND compliance_id = ANY(%s)
|
||||
GROUP BY compliance_id, requirement_id
|
||||
) req_agg
|
||||
GROUP BY compliance_id
|
||||
ON CONFLICT (tenant_id, compliance_id)
|
||||
DO UPDATE SET
|
||||
requirements_passed = EXCLUDED.requirements_passed,
|
||||
requirements_failed = EXCLUDED.requirements_failed,
|
||||
requirements_manual = EXCLUDED.requirements_manual,
|
||||
total_requirements = EXCLUDED.total_requirements,
|
||||
updated_at = NOW()
|
||||
"""
|
||||
|
||||
# Upsert tenant compliance summary for ALL compliance IDs in tenant.
|
||||
# Used by backfill when recalculating entire tenant summary.
|
||||
# Parameters: [tenant_id, tenant_id]
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_ALL_SQL = """
|
||||
INSERT INTO tenant_compliance_summaries
|
||||
(id, tenant_id, compliance_id,
|
||||
requirements_passed, requirements_failed, requirements_manual,
|
||||
total_requirements, updated_at)
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
%s as tenant_id,
|
||||
compliance_id,
|
||||
COUNT(*) FILTER (WHERE req_status = 'PASS') as requirements_passed,
|
||||
COUNT(*) FILTER (WHERE req_status = 'FAIL') as requirements_failed,
|
||||
COUNT(*) FILTER (WHERE req_status = 'MANUAL') as requirements_manual,
|
||||
COUNT(*) as total_requirements,
|
||||
NOW() as updated_at
|
||||
FROM (
|
||||
SELECT
|
||||
compliance_id,
|
||||
requirement_id,
|
||||
CASE
|
||||
WHEN bool_or(requirement_status = 'FAIL') THEN 'FAIL'
|
||||
WHEN bool_or(requirement_status = 'MANUAL') THEN 'MANUAL'
|
||||
ELSE 'PASS'
|
||||
END as req_status
|
||||
FROM provider_compliance_scores
|
||||
WHERE tenant_id = %s
|
||||
GROUP BY compliance_id, requirement_id
|
||||
) req_agg
|
||||
GROUP BY compliance_id
|
||||
ON CONFLICT (tenant_id, compliance_id)
|
||||
DO UPDATE SET
|
||||
requirements_passed = EXCLUDED.requirements_passed,
|
||||
requirements_failed = EXCLUDED.requirements_failed,
|
||||
requirements_manual = EXCLUDED.requirements_manual,
|
||||
total_requirements = EXCLUDED.total_requirements,
|
||||
updated_at = NOW()
|
||||
"""
|
||||
@@ -243,15 +243,28 @@ def _safe_getattr(obj, attr: str, default: str = "N/A") -> str:
|
||||
|
||||
|
||||
def _create_info_table_style() -> TableStyle:
|
||||
"""Create a reusable table style for information/metadata tables."""
|
||||
"""Create a reusable table style for information/metadata tables.
|
||||
|
||||
ReportLab TableStyle coordinate system:
|
||||
- Format: (COMMAND, (start_col, start_row), (end_col, end_row), value)
|
||||
- Coordinates use (column, row) format, starting at (0, 0) for top-left cell
|
||||
- Negative indices work like Python slicing: -1 means "last row/column"
|
||||
- (0, 0) to (0, -1) = entire first column (all rows)
|
||||
- (0, 0) to (-1, 0) = entire first row (all columns)
|
||||
- (0, 0) to (-1, -1) = entire table
|
||||
- Styles are applied in order; later rules override earlier ones
|
||||
"""
|
||||
return TableStyle(
|
||||
[
|
||||
# Column 0 (labels): blue background with white text
|
||||
("BACKGROUND", (0, 0), (0, -1), COLOR_BLUE),
|
||||
("TEXTCOLOR", (0, 0), (0, -1), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (0, -1), "FiraCode"),
|
||||
# Column 1 (values): light blue background with gray text
|
||||
("BACKGROUND", (1, 0), (1, -1), COLOR_BG_BLUE),
|
||||
("TEXTCOLOR", (1, 0), (1, -1), COLOR_GRAY),
|
||||
("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"),
|
||||
# Apply to entire table
|
||||
("ALIGN", (0, 0), (-1, -1), "LEFT"),
|
||||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 11),
|
||||
@@ -265,19 +278,30 @@ def _create_info_table_style() -> TableStyle:
|
||||
|
||||
|
||||
def _create_header_table_style(header_color: colors.Color = None) -> TableStyle:
|
||||
"""Create a reusable table style for tables with headers."""
|
||||
"""Create a reusable table style for tables with headers.
|
||||
|
||||
ReportLab TableStyle coordinate system:
|
||||
- Format: (COMMAND, (start_col, start_row), (end_col, end_row), value)
|
||||
- (0, 0) to (-1, 0) = entire first row (header row)
|
||||
- (1, 1) to (-1, -1) = all data cells (excludes header row and first column)
|
||||
- See _create_info_table_style() for full coordinate system documentation
|
||||
"""
|
||||
if header_color is None:
|
||||
header_color = COLOR_BLUE
|
||||
|
||||
return TableStyle(
|
||||
[
|
||||
# Header row (row 0): colored background with white text
|
||||
("BACKGROUND", (0, 0), (-1, 0), header_color),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
|
||||
("FONTSIZE", (0, 0), (-1, 0), 10),
|
||||
# Apply to entire table
|
||||
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
# Data cells (excluding header): smaller font
|
||||
("FONTSIZE", (1, 1), (-1, -1), 9),
|
||||
# Apply to entire table
|
||||
("GRID", (0, 0), (-1, -1), 1, COLOR_GRID_GRAY),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), PADDING_MEDIUM),
|
||||
@@ -288,18 +312,30 @@ def _create_header_table_style(header_color: colors.Color = None) -> TableStyle:
|
||||
|
||||
|
||||
def _create_findings_table_style() -> TableStyle:
|
||||
"""Create a reusable table style for findings tables."""
|
||||
"""Create a reusable table style for findings tables.
|
||||
|
||||
ReportLab TableStyle coordinate system:
|
||||
- Format: (COMMAND, (start_col, start_row), (end_col, end_row), value)
|
||||
- (0, 0) to (-1, 0) = entire first row (header row)
|
||||
- (0, 0) to (0, 0) = only the top-left cell
|
||||
- See _create_info_table_style() for full coordinate system documentation
|
||||
"""
|
||||
return TableStyle(
|
||||
[
|
||||
# Header row (row 0): colored background with white text
|
||||
("BACKGROUND", (0, 0), (-1, 0), COLOR_BLUE),
|
||||
("TEXTCOLOR", (0, 0), (-1, 0), COLOR_WHITE),
|
||||
("FONTNAME", (0, 0), (-1, 0), "FiraCode"),
|
||||
# Only top-left cell centered (for index/number column)
|
||||
("ALIGN", (0, 0), (0, 0), "CENTER"),
|
||||
# Apply to entire table
|
||||
("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 9),
|
||||
("GRID", (0, 0), (-1, -1), 0.1, COLOR_BORDER_GRAY),
|
||||
# Remove padding only from top-left cell
|
||||
("LEFTPADDING", (0, 0), (0, 0), 0),
|
||||
("RIGHTPADDING", (0, 0), (0, 0), 0),
|
||||
# Apply to entire table
|
||||
("TOPPADDING", (0, 0), (-1, -1), PADDING_SMALL),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), PADDING_SMALL),
|
||||
]
|
||||
@@ -1103,11 +1139,15 @@ def generate_threatscore_report(
|
||||
elements.append(Spacer(1, 0.5 * inch))
|
||||
|
||||
# Add compliance information table
|
||||
provider_alias = provider_obj.alias or "N/A"
|
||||
info_data = [
|
||||
["Framework:", compliance_framework],
|
||||
["ID:", compliance_id],
|
||||
["Name:", Paragraph(compliance_name, normal_center)],
|
||||
["Version:", compliance_version],
|
||||
["Provider:", provider_type.upper()],
|
||||
["Account ID:", provider_obj.uid],
|
||||
["Alias:", provider_alias],
|
||||
["Scan ID:", scan_id],
|
||||
["Description:", Paragraph(compliance_description, normal_center)],
|
||||
]
|
||||
@@ -2059,12 +2099,15 @@ def generate_ens_report(
|
||||
elements.append(Spacer(1, 0.5 * inch))
|
||||
|
||||
# Add compliance information table
|
||||
provider_alias = provider_obj.alias or "N/A"
|
||||
info_data = [
|
||||
["Framework:", compliance_framework],
|
||||
["ID:", compliance_id],
|
||||
["Nombre:", Paragraph(compliance_name, normal_center)],
|
||||
["Versión:", compliance_version],
|
||||
["Proveedor:", provider_type.upper()],
|
||||
["Account ID:", provider_obj.uid],
|
||||
["Alias:", provider_alias],
|
||||
["Scan ID:", scan_id],
|
||||
["Descripción:", Paragraph(compliance_description, normal_center)],
|
||||
]
|
||||
@@ -2072,12 +2115,12 @@ def generate_ens_report(
|
||||
info_table.setStyle(
|
||||
TableStyle(
|
||||
[
|
||||
("BACKGROUND", (0, 0), (0, 6), colors.Color(0.2, 0.4, 0.6)),
|
||||
("TEXTCOLOR", (0, 0), (0, 6), colors.white),
|
||||
("FONTNAME", (0, 0), (0, 6), "FiraCode"),
|
||||
("BACKGROUND", (1, 0), (1, 6), colors.Color(0.95, 0.97, 1.0)),
|
||||
("TEXTCOLOR", (1, 0), (1, 6), colors.Color(0.2, 0.2, 0.2)),
|
||||
("FONTNAME", (1, 0), (1, 6), "PlusJakartaSans"),
|
||||
("BACKGROUND", (0, 0), (0, -1), colors.Color(0.2, 0.4, 0.6)),
|
||||
("TEXTCOLOR", (0, 0), (0, -1), colors.white),
|
||||
("FONTNAME", (0, 0), (0, -1), "FiraCode"),
|
||||
("BACKGROUND", (1, 0), (1, -1), colors.Color(0.95, 0.97, 1.0)),
|
||||
("TEXTCOLOR", (1, 0), (1, -1), colors.Color(0.2, 0.2, 0.2)),
|
||||
("FONTNAME", (1, 0), (1, -1), "PlusJakartaSans"),
|
||||
("ALIGN", (0, 0), (-1, -1), "LEFT"),
|
||||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||||
("FONTSIZE", (0, 0), (-1, -1), 11),
|
||||
@@ -2997,11 +3040,14 @@ def generate_nis2_report(
|
||||
elements.append(Spacer(1, 0.3 * inch))
|
||||
|
||||
# Compliance metadata table
|
||||
provider_alias = provider_obj.alias or "N/A"
|
||||
metadata_data = [
|
||||
["Framework:", compliance_framework],
|
||||
["Name:", Paragraph(compliance_name, normal_center)],
|
||||
["Version:", compliance_version or "N/A"],
|
||||
["Provider:", provider_type.upper()],
|
||||
["Account ID:", provider_obj.uid],
|
||||
["Alias:", provider_alias],
|
||||
["Scan ID:", scan_id],
|
||||
["Description:", Paragraph(compliance_description, normal_center)],
|
||||
]
|
||||
@@ -3485,6 +3531,7 @@ def generate_compliance_reports(
|
||||
"gcp",
|
||||
"m365",
|
||||
"kubernetes",
|
||||
"alibabacloud",
|
||||
]:
|
||||
logger.info(
|
||||
f"Provider {provider_id} ({provider_type}) is not supported for ThreatScore report"
|
||||
|
||||
@@ -8,11 +8,16 @@ from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import sentry_sdk
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.env import env
|
||||
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
|
||||
from django.db import IntegrityError, OperationalError
|
||||
from django.db.models import Case, Count, IntegerField, Prefetch, Q, Sum, When
|
||||
from tasks.jobs.queries import (
|
||||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL,
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
|
||||
)
|
||||
from tasks.utils import CustomEncoder
|
||||
|
||||
from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
|
||||
@@ -39,6 +44,8 @@ from api.models import (
|
||||
ResourceScanSummary,
|
||||
ResourceTag,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
)
|
||||
@@ -77,7 +84,6 @@ FINDINGS_MICRO_BATCH_SIZE = env.int("DJANGO_FINDINGS_MICRO_BATCH_SIZE", default=
|
||||
# Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres
|
||||
SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=500)
|
||||
|
||||
|
||||
ATTACK_SURFACE_PROVIDER_COMPATIBILITY = {
|
||||
"internet-exposed": None, # Compatible with all providers
|
||||
"secrets": None, # Compatible with all providers
|
||||
@@ -88,6 +94,84 @@ ATTACK_SURFACE_PROVIDER_COMPATIBILITY = {
|
||||
_ATTACK_SURFACE_MAPPING_CACHE: dict[str, dict] = {}
|
||||
|
||||
|
||||
def aggregate_category_counts(
|
||||
categories: list[str],
|
||||
severity: str,
|
||||
status: str,
|
||||
delta: str | None,
|
||||
muted: bool,
|
||||
cache: dict[tuple[str, str], dict[str, int]],
|
||||
) -> None:
|
||||
"""
|
||||
Increment category counters in-place for a finding.
|
||||
|
||||
Args:
|
||||
categories: List of categories from finding metadata.
|
||||
severity: Severity level (e.g., "high", "medium").
|
||||
status: Finding status as string ("FAIL", "PASS").
|
||||
delta: Delta value as string ("new", "changed") or None.
|
||||
muted: Whether the finding is muted.
|
||||
cache: Dict {(category, severity): {"total", "failed", "new_failed"}} to update.
|
||||
"""
|
||||
is_failed = status == "FAIL" and not muted
|
||||
is_new_failed = is_failed and delta == "new"
|
||||
|
||||
for cat in categories:
|
||||
key = (cat, severity)
|
||||
if key not in cache:
|
||||
cache[key] = {"total": 0, "failed": 0, "new_failed": 0}
|
||||
if not muted:
|
||||
cache[key]["total"] += 1
|
||||
if is_failed:
|
||||
cache[key]["failed"] += 1
|
||||
if is_new_failed:
|
||||
cache[key]["new_failed"] += 1
|
||||
|
||||
|
||||
def aggregate_resource_group_counts(
|
||||
resource_group: str | None,
|
||||
severity: str,
|
||||
status: str,
|
||||
delta: str | None,
|
||||
muted: bool,
|
||||
resource_uid: str,
|
||||
cache: dict[tuple[str, str], dict[str, int]],
|
||||
group_resources_cache: dict[str, set],
|
||||
) -> None:
|
||||
"""
|
||||
Increment resource group counters in-place for a finding.
|
||||
|
||||
Args:
|
||||
resource_group: Resource group from check metadata (e.g., "database", "compute").
|
||||
severity: Severity level (e.g., "high", "medium").
|
||||
status: Finding status as string ("FAIL", "PASS").
|
||||
delta: Delta value as string ("new", "changed") or None.
|
||||
muted: Whether the finding is muted.
|
||||
resource_uid: Unique identifier for the resource to count distinct resources.
|
||||
cache: Dict {(resource_group, severity): {"total", "failed", "new_failed"}} to update.
|
||||
group_resources_cache: Dict {resource_group: set(resource_uids)} for group-level resource tracking.
|
||||
"""
|
||||
if not resource_group:
|
||||
return
|
||||
|
||||
is_failed = status == "FAIL" and not muted
|
||||
is_new_failed = is_failed and delta == "new"
|
||||
|
||||
key = (resource_group, severity)
|
||||
if key not in cache:
|
||||
cache[key] = {"total": 0, "failed": 0, "new_failed": 0}
|
||||
if not muted:
|
||||
cache[key]["total"] += 1
|
||||
if is_failed:
|
||||
cache[key]["failed"] += 1
|
||||
if is_new_failed:
|
||||
cache[key]["new_failed"] += 1
|
||||
|
||||
# Track resources at GROUP level (not per-severity) to avoid over-counting
|
||||
if resource_uid and not muted:
|
||||
group_resources_cache.setdefault(resource_group, set()).add(resource_uid)
|
||||
|
||||
|
||||
def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict:
|
||||
global _ATTACK_SURFACE_MAPPING_CACHE
|
||||
|
||||
@@ -398,6 +482,9 @@ def _process_finding_micro_batch(
|
||||
unique_resources: set,
|
||||
scan_resource_cache: set,
|
||||
mute_rules_cache: dict,
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]],
|
||||
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]],
|
||||
group_resources_cache: dict[str, set],
|
||||
) -> None:
|
||||
"""
|
||||
Process a micro-batch of findings and persist them using bulk operations.
|
||||
@@ -418,6 +505,9 @@ def _process_finding_micro_batch(
|
||||
unique_resources: Set tracking (uid, region) pairs seen in the scan.
|
||||
scan_resource_cache: Set of tuples used to create `ResourceScanSummary` rows.
|
||||
mute_rules_cache: Map of finding UID -> mute reason gathered before the scan.
|
||||
scan_categories_cache: Dict tracking category counts {(category, severity): {"total", "failed", "new_failed"}}.
|
||||
scan_resource_groups_cache: Dict tracking resource group counts {(resource_group, severity): {"total", "failed", "new_failed"}}.
|
||||
group_resources_cache: Dict tracking unique resources per group {resource_group: set(resource_uids)}.
|
||||
"""
|
||||
# Accumulate objects for bulk operations
|
||||
findings_to_create = []
|
||||
@@ -458,6 +548,8 @@ def _process_finding_micro_batch(
|
||||
with rls_transaction(tenant_id):
|
||||
resource_uid = finding.resource_uid
|
||||
if resource_uid not in resource_cache:
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
resource_instance, _ = Resource.objects.get_or_create(
|
||||
tenant_id=tenant_id,
|
||||
provider=provider_instance,
|
||||
@@ -467,6 +559,7 @@ def _process_finding_micro_batch(
|
||||
"service": finding.service_name,
|
||||
"type": finding.resource_type,
|
||||
"name": finding.resource_name,
|
||||
"groups": [group] if group else None,
|
||||
},
|
||||
)
|
||||
resource_cache[resource_uid] = resource_instance
|
||||
@@ -487,6 +580,8 @@ def _process_finding_micro_batch(
|
||||
|
||||
# Track resource field changes (defer save)
|
||||
updated = False
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
if finding.region and resource_instance.region != finding.region:
|
||||
resource_instance.region = finding.region
|
||||
updated = True
|
||||
@@ -507,6 +602,11 @@ def _process_finding_micro_batch(
|
||||
if resource_instance.partition != finding.partition:
|
||||
resource_instance.partition = finding.partition
|
||||
updated = True
|
||||
if group and (
|
||||
not resource_instance.groups or group not in resource_instance.groups
|
||||
):
|
||||
resource_instance.groups = (resource_instance.groups or []) + [group]
|
||||
updated = True
|
||||
|
||||
if updated:
|
||||
dirty_resources[resource_uid] = resource_instance
|
||||
@@ -573,11 +673,12 @@ def _process_finding_micro_batch(
|
||||
resource_failed_findings_cache[resource_uid] += 1
|
||||
|
||||
# Create finding object (don't save yet)
|
||||
check_metadata = finding.get_metadata()
|
||||
finding_instance = Finding(
|
||||
tenant_id=tenant_id,
|
||||
uid=finding_uid,
|
||||
delta=delta,
|
||||
check_metadata=finding.get_metadata(),
|
||||
check_metadata=check_metadata,
|
||||
status=status,
|
||||
status_extended=finding.status_extended,
|
||||
severity=finding.severity,
|
||||
@@ -590,6 +691,8 @@ def _process_finding_micro_batch(
|
||||
muted_at=datetime.now(tz=timezone.utc) if is_muted else None,
|
||||
muted_reason=muted_reason,
|
||||
compliance=finding.compliance,
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
resource_groups=check_metadata.get("resourcegroup") or None,
|
||||
)
|
||||
findings_to_create.append(finding_instance)
|
||||
resource_denormalized_data.append((finding_instance, resource_instance))
|
||||
@@ -604,6 +707,28 @@ def _process_finding_micro_batch(
|
||||
)
|
||||
)
|
||||
|
||||
# Track categories with counts for ScanCategorySummary by (category, severity)
|
||||
aggregate_category_counts(
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
cache=scan_categories_cache,
|
||||
)
|
||||
|
||||
# Track resource groups with counts for ScanGroupSummary
|
||||
aggregate_resource_group_counts(
|
||||
resource_group=check_metadata.get("resourcegroup") or None,
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
resource_uid=resource_instance.uid if resource_instance else "",
|
||||
cache=scan_resource_groups_cache,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Bulk operations within single transaction
|
||||
with rls_transaction(tenant_id):
|
||||
# Bulk create findings
|
||||
@@ -661,7 +786,15 @@ def _process_finding_micro_batch(
|
||||
tenant_id=tenant_id,
|
||||
model=Resource,
|
||||
objects=list(dirty_resources.values()),
|
||||
fields=["metadata", "details", "partition", "region", "service", "type"],
|
||||
fields=[
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
"region",
|
||||
"service",
|
||||
"type",
|
||||
"groups",
|
||||
],
|
||||
batch_size=1000,
|
||||
)
|
||||
|
||||
@@ -703,6 +836,9 @@ def perform_prowler_scan(
|
||||
exception = None
|
||||
unique_resources = set()
|
||||
scan_resource_cache: set[tuple[str, str, str, str]] = set()
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
scan_resource_groups_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
group_resources_cache: dict[str, set] = {}
|
||||
start_time = time.time()
|
||||
exc = None
|
||||
|
||||
@@ -792,6 +928,9 @@ def perform_prowler_scan(
|
||||
unique_resources=unique_resources,
|
||||
scan_resource_cache=scan_resource_cache,
|
||||
mute_rules_cache=mute_rules_cache,
|
||||
scan_categories_cache=scan_categories_cache,
|
||||
scan_resource_groups_cache=scan_resource_groups_cache,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Update scan progress
|
||||
@@ -851,13 +990,65 @@ def perform_prowler_scan(
|
||||
resource_scan_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
except Exception as filter_exception:
|
||||
import sentry_sdk
|
||||
|
||||
sentry_sdk.capture_exception(filter_exception)
|
||||
logger.error(
|
||||
f"Error storing filter values for scan {scan_id}: {filter_exception}"
|
||||
)
|
||||
|
||||
try:
|
||||
if scan_categories_cache:
|
||||
category_summaries = [
|
||||
ScanCategorySummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
category=category,
|
||||
severity=severity,
|
||||
total_findings=counts["total"],
|
||||
failed_findings=counts["failed"],
|
||||
new_failed_findings=counts["new_failed"],
|
||||
)
|
||||
for (category, severity), counts in scan_categories_cache.items()
|
||||
]
|
||||
with rls_transaction(tenant_id):
|
||||
ScanCategorySummary.objects.bulk_create(
|
||||
category_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
except Exception as cat_exception:
|
||||
sentry_sdk.capture_exception(cat_exception)
|
||||
logger.error(f"Error storing categories for scan {scan_id}: {cat_exception}")
|
||||
|
||||
try:
|
||||
if scan_resource_groups_cache:
|
||||
# Compute group-level resource counts (same value for all severity rows in a group)
|
||||
group_resource_counts = {
|
||||
grp: len(uids) for grp, uids in group_resources_cache.items()
|
||||
}
|
||||
resource_group_summaries = [
|
||||
ScanGroupSummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
resource_group=grp,
|
||||
severity=severity,
|
||||
total_findings=counts["total"],
|
||||
failed_findings=counts["failed"],
|
||||
new_failed_findings=counts["new_failed"],
|
||||
resources_count=group_resource_counts.get(grp, 0),
|
||||
)
|
||||
for (
|
||||
grp,
|
||||
severity,
|
||||
), counts in scan_resource_groups_cache.items()
|
||||
]
|
||||
with rls_transaction(tenant_id):
|
||||
ScanGroupSummary.objects.bulk_create(
|
||||
resource_group_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
except Exception as rg_exception:
|
||||
sentry_sdk.capture_exception(rg_exception)
|
||||
logger.error(
|
||||
f"Error storing resource groups for scan {scan_id}: {rg_exception}"
|
||||
)
|
||||
|
||||
serializer = ScanTaskSerializer(instance=scan_instance)
|
||||
return serializer.data
|
||||
|
||||
@@ -1418,3 +1609,140 @@ def aggregate_daily_severity(tenant_id: str, scan_id: str):
|
||||
"date": str(scan_date),
|
||||
"severity_data": severity_data,
|
||||
}
|
||||
|
||||
|
||||
def update_provider_compliance_scores(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Update ProviderComplianceScore with requirement statuses from a completed scan.
|
||||
|
||||
Uses atomic SQL upsert with ON CONFLICT for concurrency safety. Only updates
|
||||
if the new scan is more recent than existing data. Also cleans up stale
|
||||
requirements that no longer exist in the new scan.
|
||||
|
||||
Reads from primary DB (not replica) to avoid replication lag issues since
|
||||
this runs immediately after create_compliance_requirements_task.
|
||||
|
||||
Args:
|
||||
tenant_id: Tenant that owns the scan.
|
||||
scan_id: Scan UUID whose compliance data should be materialized.
|
||||
|
||||
Returns:
|
||||
dict: Statistics about the upsert operation.
|
||||
"""
|
||||
with rls_transaction(tenant_id):
|
||||
scan = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id=scan_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
.select_related("provider")
|
||||
.first()
|
||||
)
|
||||
|
||||
if not scan:
|
||||
logger.warning(
|
||||
f"Scan {scan_id} not found or not completed for compliance score update"
|
||||
)
|
||||
return {"status": "skipped", "reason": "scan not completed"}
|
||||
|
||||
if not scan.completed_at:
|
||||
logger.warning(f"Scan {scan_id} has no completed_at timestamp")
|
||||
return {"status": "skipped", "reason": "no completed_at"}
|
||||
|
||||
provider_id = str(scan.provider_id)
|
||||
scan_completed_at = scan.completed_at
|
||||
|
||||
delete_stale_sql = """
|
||||
DELETE FROM provider_compliance_scores pcs
|
||||
WHERE pcs.tenant_id = %s
|
||||
AND pcs.provider_id = %s
|
||||
AND pcs.scan_completed_at < %s
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM compliance_requirements_overviews cro
|
||||
WHERE cro.tenant_id = pcs.tenant_id
|
||||
AND cro.scan_id = %s
|
||||
AND cro.compliance_id = pcs.compliance_id
|
||||
AND cro.requirement_id = pcs.requirement_id
|
||||
)
|
||||
RETURNING compliance_id
|
||||
"""
|
||||
|
||||
compliance_ids_sql = """
|
||||
SELECT DISTINCT compliance_id
|
||||
FROM compliance_requirements_overviews
|
||||
WHERE tenant_id = %s AND scan_id = %s
|
||||
"""
|
||||
|
||||
try:
|
||||
with psycopg_connection(MainRouter.default_db) as connection:
|
||||
connection.autocommit = False
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id])
|
||||
|
||||
# Update requirement-level scores per provider
|
||||
cursor.execute(
|
||||
COMPLIANCE_UPSERT_PROVIDER_SCORE_SQL, [tenant_id, scan_id]
|
||||
)
|
||||
upserted_count = cursor.rowcount
|
||||
|
||||
cursor.execute(compliance_ids_sql, [tenant_id, scan_id])
|
||||
scan_rows = cursor.fetchall()
|
||||
if not isinstance(scan_rows, (list, tuple)):
|
||||
scan_rows = []
|
||||
scan_compliance_ids = {row[0] for row in scan_rows}
|
||||
|
||||
cursor.execute(
|
||||
delete_stale_sql,
|
||||
[tenant_id, provider_id, scan_completed_at, scan_id],
|
||||
)
|
||||
deleted_rows = cursor.fetchall()
|
||||
if not isinstance(deleted_rows, (list, tuple)):
|
||||
deleted_rows = []
|
||||
deleted_ids = {row[0] for row in deleted_rows}
|
||||
stale_deleted = len(deleted_ids)
|
||||
|
||||
impacted_compliance_ids = sorted(scan_compliance_ids | deleted_ids)
|
||||
|
||||
if impacted_compliance_ids:
|
||||
# Advisory lock on tenant to prevent race conditions when
|
||||
# multiple scans complete simultaneously for the same tenant
|
||||
cursor.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtext(%s))", [tenant_id]
|
||||
)
|
||||
|
||||
# Recalculate tenant-level summary (FAIL-dominant across all providers)
|
||||
cursor.execute(
|
||||
COMPLIANCE_UPSERT_TENANT_SUMMARY_SQL,
|
||||
[tenant_id, tenant_id, impacted_compliance_ids],
|
||||
)
|
||||
tenant_summary_count = cursor.rowcount
|
||||
else:
|
||||
tenant_summary_count = 0
|
||||
|
||||
connection.commit()
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
f"Provider compliance scores updated for scan {scan_id}: "
|
||||
f"{upserted_count} upserted, {stale_deleted} stale deleted, "
|
||||
f"{tenant_summary_count} tenant summaries upserted"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"scan_id": str(scan_id),
|
||||
"provider_id": provider_id,
|
||||
"upserted": upserted_count,
|
||||
"stale_deleted": stale_deleted,
|
||||
"tenant_summary_count": tenant_summary_count,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error updating provider compliance scores for scan {scan_id}: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
+200
-52
@@ -8,10 +8,17 @@ from celery.utils.log import get_task_logger
|
||||
from config.celery import RLSTask
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from tasks.jobs.attack_paths import (
|
||||
attack_paths_scan,
|
||||
can_provider_run_attack_paths_scan,
|
||||
)
|
||||
from tasks.jobs.backfill import (
|
||||
backfill_compliance_summaries,
|
||||
backfill_daily_severity_summaries,
|
||||
backfill_provider_compliance_scores,
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
backfill_scan_resource_group_summaries,
|
||||
)
|
||||
from tasks.jobs.connection import (
|
||||
check_integration_connection,
|
||||
@@ -43,8 +50,13 @@ from tasks.jobs.scan import (
|
||||
aggregate_findings,
|
||||
create_compliance_requirements,
|
||||
perform_prowler_scan,
|
||||
update_provider_compliance_scores,
|
||||
)
|
||||
from tasks.utils import (
|
||||
_get_or_create_scheduled_scan,
|
||||
batched,
|
||||
get_next_execution_datetime,
|
||||
)
|
||||
from tasks.utils import batched, get_next_execution_datetime
|
||||
|
||||
from api.compliance import get_compliance_frameworks
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
@@ -60,6 +72,58 @@ from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def _cleanup_orphan_scheduled_scans(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
scheduler_task_id: int,
|
||||
) -> int:
|
||||
"""
|
||||
TEMPORARY WORKAROUND: Clean up orphan AVAILABLE scans.
|
||||
|
||||
Detects and removes AVAILABLE scans that were never used due to an
|
||||
issue during the first scheduled scan setup.
|
||||
|
||||
An AVAILABLE scan is considered orphan if there's also a SCHEDULED scan for
|
||||
the same provider with the same scheduler_task_id. This situation indicates
|
||||
that the first scan execution didn't find the AVAILABLE scan (because it
|
||||
wasn't committed yet, probably) and created a new one, leaving the AVAILABLE orphaned.
|
||||
|
||||
Args:
|
||||
tenant_id: The tenant ID.
|
||||
provider_id: The provider ID.
|
||||
scheduler_task_id: The PeriodicTask ID that triggers these scans.
|
||||
|
||||
Returns:
|
||||
Number of orphan scans deleted (0 if none found).
|
||||
"""
|
||||
orphan_available_scans = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=scheduler_task_id,
|
||||
)
|
||||
|
||||
scheduled_scan_exists = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=scheduler_task_id,
|
||||
).exists()
|
||||
|
||||
if scheduled_scan_exists and orphan_available_scans.exists():
|
||||
orphan_count = orphan_available_scans.count()
|
||||
logger.warning(
|
||||
f"[WORKAROUND] Found {orphan_count} orphan AVAILABLE scan(s) for "
|
||||
f"provider {provider_id} alongside a SCHEDULED scan. Cleaning up orphans..."
|
||||
)
|
||||
orphan_available_scans.delete()
|
||||
return orphan_count
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str):
|
||||
"""
|
||||
Helper function to perform tasks after a scan is completed.
|
||||
@@ -69,9 +133,10 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
|
||||
scan_id (str): The ID of the scan that was performed.
|
||||
provider_id (str): The primary key of the Provider instance that was scanned.
|
||||
"""
|
||||
create_compliance_requirements_task.apply_async(
|
||||
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
|
||||
)
|
||||
chain(
|
||||
create_compliance_requirements_task.si(tenant_id=tenant_id, scan_id=scan_id),
|
||||
update_provider_compliance_scores_task.si(tenant_id=tenant_id, scan_id=scan_id),
|
||||
).apply_async()
|
||||
aggregate_attack_surface_task.apply_async(
|
||||
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
|
||||
)
|
||||
@@ -96,6 +161,11 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
|
||||
),
|
||||
).apply_async()
|
||||
|
||||
if can_provider_run_attack_paths_scan(tenant_id, provider_id):
|
||||
perform_attack_paths_scan_task.apply_async(
|
||||
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
|
||||
)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="provider-connection-check")
|
||||
@set_tenant
|
||||
@@ -208,57 +278,52 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
periodic_task_instance = PeriodicTask.objects.get(
|
||||
name=f"scan-perform-scheduled-{provider_id}"
|
||||
)
|
||||
|
||||
executed_scan = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
task__task_runner_task__task_id=task_id,
|
||||
).order_by("completed_at")
|
||||
|
||||
if (
|
||||
executing_scan = (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.EXECUTING,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
scheduled_at__date=datetime.now(timezone.utc).date(),
|
||||
).exists()
|
||||
or executed_scan.exists()
|
||||
):
|
||||
# Duplicated task execution due to visibility timeout or scan is already running
|
||||
logger.warning(f"Duplicated scheduled scan for provider {provider_id}.")
|
||||
try:
|
||||
affected_scan = executed_scan.first()
|
||||
if not affected_scan:
|
||||
raise ValueError(
|
||||
"Error retrieving affected scan details after detecting duplicated scheduled "
|
||||
"scan."
|
||||
)
|
||||
# Return the affected scan details to avoid losing data
|
||||
serializer = ScanTaskSerializer(instance=affected_scan)
|
||||
except Exception as duplicated_scan_exception:
|
||||
logger.error(
|
||||
f"Duplicated scheduled scan for provider {provider_id}. Error retrieving affected scan details: "
|
||||
f"{str(duplicated_scan_exception)}"
|
||||
)
|
||||
raise duplicated_scan_exception
|
||||
return serializer.data
|
||||
)
|
||||
.order_by("-started_at")
|
||||
.first()
|
||||
)
|
||||
if executing_scan:
|
||||
logger.warning(
|
||||
f"Scheduled scan already executing for provider {provider_id}. Skipping."
|
||||
)
|
||||
return ScanTaskSerializer(instance=executing_scan).data
|
||||
|
||||
next_scan_datetime = get_next_execution_datetime(task_id, provider_id)
|
||||
scan_instance, _ = Scan.objects.get_or_create(
|
||||
executed_scan = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state__in=(StateChoices.SCHEDULED, StateChoices.AVAILABLE),
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
defaults={
|
||||
"state": StateChoices.SCHEDULED,
|
||||
"name": "Daily scheduled scan",
|
||||
"scheduled_at": next_scan_datetime - timedelta(days=1),
|
||||
},
|
||||
task__task_runner_task__task_id=task_id,
|
||||
).first()
|
||||
|
||||
if executed_scan:
|
||||
# Duplicated task execution due to visibility timeout
|
||||
logger.warning(f"Duplicated scheduled scan for provider {provider_id}.")
|
||||
return ScanTaskSerializer(instance=executed_scan).data
|
||||
|
||||
interval = periodic_task_instance.interval
|
||||
next_scan_datetime = get_next_execution_datetime(task_id, provider_id)
|
||||
current_scan_datetime = next_scan_datetime - timedelta(
|
||||
**{interval.period: interval.every}
|
||||
)
|
||||
|
||||
# TEMPORARY WORKAROUND: Clean up orphan scans from transaction isolation issue
|
||||
_cleanup_orphan_scheduled_scans(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
)
|
||||
|
||||
scan_instance = _get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
scheduled_at=current_scan_datetime,
|
||||
)
|
||||
scan_instance.task_id = task_id
|
||||
scan_instance.save()
|
||||
|
||||
@@ -268,18 +333,19 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
scan_id=str(scan_instance.id),
|
||||
provider_id=provider_id,
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
finally:
|
||||
with rls_transaction(tenant_id):
|
||||
Scan.objects.get_or_create(
|
||||
now = datetime.now(timezone.utc)
|
||||
if next_scan_datetime <= now:
|
||||
interval_delta = timedelta(**{interval.period: interval.every})
|
||||
while next_scan_datetime <= now:
|
||||
next_scan_datetime += interval_delta
|
||||
_get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
name="Daily scheduled scan",
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduled_at=next_scan_datetime,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
scheduled_at=next_scan_datetime,
|
||||
update_state=True,
|
||||
)
|
||||
|
||||
_perform_scan_complete_tasks(tenant_id, str(scan_instance.id), provider_id)
|
||||
@@ -293,6 +359,29 @@ def perform_scan_summary_task(tenant_id: str, scan_id: str):
|
||||
return aggregate_findings(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(
|
||||
base=RLSTask,
|
||||
bind=True,
|
||||
name="attack-paths-scan-perform",
|
||||
queue="attack-paths-scans",
|
||||
)
|
||||
def perform_attack_paths_scan_task(self, tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Execute an Attack Paths scan for the given provider within the current tenant RLS context.
|
||||
|
||||
Args:
|
||||
self: The task instance (automatically passed when bind=True).
|
||||
tenant_id (str): The tenant identifier for RLS context.
|
||||
scan_id (str): The Prowler scan identifier for obtaining the tenant and provider context.
|
||||
|
||||
Returns:
|
||||
Any: The result from `attack_paths_scan`, including any per-scan failure details.
|
||||
"""
|
||||
return attack_paths_scan(
|
||||
tenant_id=tenant_id, scan_id=scan_id, task_id=self.request.id
|
||||
)
|
||||
|
||||
|
||||
@shared_task(name="tenant-deletion", queue="deletion", autoretry_for=(Exception,))
|
||||
def delete_tenant_task(tenant_id: str):
|
||||
return delete_tenant(pk=tenant_id)
|
||||
@@ -534,6 +623,50 @@ def backfill_daily_severity_summaries_task(tenant_id: str, days: int = None):
|
||||
return backfill_daily_severity_summaries(tenant_id=tenant_id, days=days)
|
||||
|
||||
|
||||
@shared_task(name="backfill-scan-category-summaries", queue="backfill")
|
||||
@handle_provider_deletion
|
||||
def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanCategorySummary for a completed scan.
|
||||
|
||||
Aggregates unique categories from findings and creates a summary row.
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant identifier.
|
||||
scan_id (str): The scan identifier.
|
||||
"""
|
||||
return backfill_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="backfill-scan-resource-group-summaries", queue="backfill")
|
||||
@handle_provider_deletion
|
||||
def backfill_scan_resource_group_summaries_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanGroupSummary for a completed scan.
|
||||
|
||||
Aggregates unique resource groups from findings and creates a summary row.
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant identifier.
|
||||
scan_id (str): The scan identifier.
|
||||
"""
|
||||
return backfill_scan_resource_group_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="backfill-provider-compliance-scores", queue="backfill")
|
||||
def backfill_provider_compliance_scores_task(tenant_id: str):
|
||||
"""
|
||||
Backfill ProviderComplianceScore from latest completed scan per provider.
|
||||
|
||||
Used to populate the compliance watchlist materialized table for tenants
|
||||
that had scans before the feature was deployed.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID.
|
||||
"""
|
||||
return backfill_provider_compliance_scores(tenant_id=tenant_id)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="scan-compliance-overviews", queue="compliance")
|
||||
@handle_provider_deletion
|
||||
def create_compliance_requirements_task(tenant_id: str, scan_id: str):
|
||||
@@ -567,6 +700,21 @@ def aggregate_attack_surface_task(tenant_id: str, scan_id: str):
|
||||
return aggregate_attack_surface(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="scan-provider-compliance-scores", queue="compliance")
|
||||
def update_provider_compliance_scores_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Update provider compliance scores from a completed scan.
|
||||
|
||||
This task materializes compliance requirement statuses into ProviderComplianceScore
|
||||
for efficient watchlist queries. Uses atomic upsert with concurrency protection.
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant ID for which to update scores.
|
||||
scan_id (str): The ID of the scan whose data should be materialized.
|
||||
"""
|
||||
return update_provider_compliance_scores(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="scan-daily-severity", queue="overview")
|
||||
@handle_provider_deletion
|
||||
def aggregate_daily_severity_task(tenant_id: str, scan_id: str):
|
||||
|
||||
@@ -0,0 +1,708 @@
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pytest
|
||||
from tasks.jobs.attack_paths import prowler as prowler_module
|
||||
from tasks.jobs.attack_paths.scan import run as attack_paths_run
|
||||
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
Finding,
|
||||
Provider,
|
||||
Resource,
|
||||
ResourceFindingMapping,
|
||||
Scan,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
)
|
||||
from prowler.lib.check.models import Severity
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestAttackPathsRun:
|
||||
def test_run_success_flow(self, tenants_fixture, providers_fixture, scans_fixture):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
scan = scans_fixture[0]
|
||||
scan.provider = provider
|
||||
scan.save()
|
||||
|
||||
attack_paths_scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
scan=scan,
|
||||
state=StateChoices.SCHEDULED,
|
||||
)
|
||||
|
||||
mock_session = MagicMock()
|
||||
session_ctx = MagicMock()
|
||||
session_ctx.__enter__.return_value = mock_session
|
||||
session_ctx.__exit__.return_value = False
|
||||
ingestion_result = {"organizations": "warning"}
|
||||
ingestion_fn = MagicMock(return_value=ingestion_result)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
|
||||
return_value=MagicMock(_enabled_regions=["us-east-1"]),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_uri",
|
||||
return_value="bolt://neo4j",
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
|
||||
return_value="db-scan-id",
|
||||
) as mock_get_db_name,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.create_database"
|
||||
) as mock_create_db,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_session",
|
||||
return_value=session_ctx,
|
||||
) as mock_get_session,
|
||||
patch("tasks.jobs.attack_paths.scan.graph_database.clear_cache"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.cartography_create_indexes.run"
|
||||
) as mock_cartography_indexes,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.cartography_analysis.run"
|
||||
) as mock_cartography_analysis,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.cartography_ontology.run"
|
||||
) as mock_cartography_ontology,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.prowler.create_indexes"
|
||||
) as mock_prowler_indexes,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.prowler.analysis"
|
||||
) as mock_prowler_analysis,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan",
|
||||
return_value=attack_paths_scan,
|
||||
) as mock_retrieve_scan,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan"
|
||||
) as mock_starting,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress"
|
||||
) as mock_update_progress,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan"
|
||||
) as mock_finish,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
|
||||
return_value=ingestion_fn,
|
||||
) as mock_get_ingestion,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan._call_within_event_loop",
|
||||
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
|
||||
) as mock_event_loop,
|
||||
):
|
||||
result = attack_paths_run(str(tenant.id), str(scan.id), "task-123")
|
||||
|
||||
assert result == ingestion_result
|
||||
mock_retrieve_scan.assert_called_once_with(str(tenant.id), str(scan.id))
|
||||
mock_starting.assert_called_once()
|
||||
config = mock_starting.call_args[0][2]
|
||||
assert config.neo4j_database == "db-scan-id"
|
||||
|
||||
mock_create_db.assert_called_once_with("db-scan-id")
|
||||
mock_get_session.assert_called_once_with("db-scan-id")
|
||||
mock_cartography_indexes.assert_called_once_with(mock_session, config)
|
||||
mock_prowler_indexes.assert_called_once_with(mock_session)
|
||||
mock_cartography_analysis.assert_called_once_with(mock_session, config)
|
||||
mock_cartography_ontology.assert_called_once_with(mock_session, config)
|
||||
mock_prowler_analysis.assert_called_once_with(
|
||||
mock_session,
|
||||
provider,
|
||||
str(scan.id),
|
||||
config,
|
||||
)
|
||||
mock_get_ingestion.assert_called_once_with(provider.provider)
|
||||
mock_event_loop.assert_called_once()
|
||||
mock_update_progress.assert_any_call(attack_paths_scan, 1)
|
||||
mock_update_progress.assert_any_call(attack_paths_scan, 2)
|
||||
mock_update_progress.assert_any_call(attack_paths_scan, 95)
|
||||
mock_finish.assert_called_once_with(
|
||||
attack_paths_scan, StateChoices.COMPLETED, ingestion_result
|
||||
)
|
||||
mock_get_db_name.assert_called_once_with(attack_paths_scan.id)
|
||||
|
||||
def test_run_failure_marks_scan_failed(
|
||||
self, tenants_fixture, providers_fixture, scans_fixture
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
scan = scans_fixture[0]
|
||||
scan.provider = provider
|
||||
scan.save()
|
||||
|
||||
attack_paths_scan = AttackPathsScan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
scan=scan,
|
||||
state=StateChoices.SCHEDULED,
|
||||
)
|
||||
|
||||
mock_session = MagicMock()
|
||||
session_ctx = MagicMock()
|
||||
session_ctx.__enter__.return_value = mock_session
|
||||
session_ctx.__exit__.return_value = False
|
||||
ingestion_fn = MagicMock(side_effect=RuntimeError("ingestion boom"))
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
|
||||
return_value=MagicMock(_enabled_regions=["us-east-1"]),
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.scan.graph_database.get_uri"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_database_name",
|
||||
return_value="db-scan-id",
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.scan.graph_database.create_database"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.graph_database.get_session",
|
||||
return_value=session_ctx,
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.scan.cartography_create_indexes.run"),
|
||||
patch("tasks.jobs.attack_paths.scan.cartography_analysis.run"),
|
||||
patch("tasks.jobs.attack_paths.scan.prowler.create_indexes"),
|
||||
patch("tasks.jobs.attack_paths.scan.prowler.analysis"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan",
|
||||
return_value=attack_paths_scan,
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.scan.db_utils.starting_attack_paths_scan"),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.update_attack_paths_scan_progress"
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.finish_attack_paths_scan"
|
||||
) as mock_finish,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
|
||||
return_value=ingestion_fn,
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan._call_within_event_loop",
|
||||
side_effect=lambda fn, *a, **kw: fn(*a, **kw),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
|
||||
return_value="Cartography failed: ingestion boom",
|
||||
),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="ingestion boom"):
|
||||
attack_paths_run(str(tenant.id), str(scan.id), "task-456")
|
||||
|
||||
failure_args = mock_finish.call_args[0]
|
||||
assert failure_args[0] is attack_paths_scan
|
||||
assert failure_args[1] == StateChoices.FAILED
|
||||
assert failure_args[2] == {
|
||||
"global_cartography_error": "Cartography failed: ingestion boom"
|
||||
}
|
||||
|
||||
def test_run_returns_early_for_unsupported_provider(self, tenants_fixture):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = Provider.objects.create(
|
||||
provider=Provider.ProviderChoices.GCP,
|
||||
uid="gcp-account",
|
||||
alias="gcp",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
scan = Scan.objects.create(
|
||||
name="GCP Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.initialize_prowler_provider",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.get_cartography_ingestion_function",
|
||||
return_value=None,
|
||||
) as mock_get_ingestion,
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.scan.db_utils.retrieve_attack_paths_scan"
|
||||
) as mock_retrieve,
|
||||
):
|
||||
mock_retrieve.return_value = None
|
||||
result = attack_paths_run(str(tenant.id), str(scan.id), "task-789")
|
||||
|
||||
assert result == {
|
||||
"global_error": "Provider gcp is not supported for Attack Paths scans"
|
||||
}
|
||||
mock_get_ingestion.assert_called_once_with(provider.provider)
|
||||
mock_retrieve.assert_called_once_with(str(tenant.id), str(scan.id))
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestAttackPathsProwlerHelpers:
|
||||
def test_create_indexes_executes_all_statements(self):
|
||||
mock_session = MagicMock()
|
||||
with patch("tasks.jobs.attack_paths.prowler.run_write_query") as mock_run_write:
|
||||
prowler_module.create_indexes(mock_session)
|
||||
|
||||
assert mock_run_write.call_count == len(prowler_module.INDEX_STATEMENTS)
|
||||
mock_run_write.assert_has_calls(
|
||||
[call(mock_session, stmt) for stmt in prowler_module.INDEX_STATEMENTS]
|
||||
)
|
||||
|
||||
def test_load_findings_batches_requests(self, providers_fixture):
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
# Create a generator that yields two batches
|
||||
def findings_generator():
|
||||
yield [{"id": "1", "resource_uid": "r-1"}]
|
||||
yield [{"id": "2", "resource_uid": "r-2"}]
|
||||
|
||||
config = SimpleNamespace(update_tag=12345)
|
||||
mock_session = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.prowler.get_root_node_label",
|
||||
return_value="AWSAccount",
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.prowler.get_node_uid_field",
|
||||
return_value="arn",
|
||||
),
|
||||
):
|
||||
prowler_module.load_findings(
|
||||
mock_session, findings_generator(), provider, config
|
||||
)
|
||||
|
||||
assert mock_session.run.call_count == 2
|
||||
for call_args in mock_session.run.call_args_list:
|
||||
params = call_args.args[1]
|
||||
assert params["provider_uid"] == str(provider.uid)
|
||||
assert params["last_updated"] == config.update_tag
|
||||
assert "findings_data" in params
|
||||
|
||||
def test_cleanup_findings_runs_batches(self, providers_fixture):
|
||||
provider = providers_fixture[0]
|
||||
config = SimpleNamespace(update_tag=1024)
|
||||
mock_session = MagicMock()
|
||||
|
||||
first_batch = MagicMock()
|
||||
first_batch.single.return_value = {"deleted_findings_count": 3}
|
||||
second_batch = MagicMock()
|
||||
second_batch.single.return_value = {"deleted_findings_count": 0}
|
||||
mock_session.run.side_effect = [first_batch, second_batch]
|
||||
|
||||
prowler_module.cleanup_findings(mock_session, provider, config)
|
||||
|
||||
assert mock_session.run.call_count == 2
|
||||
params = mock_session.run.call_args.args[1]
|
||||
assert params["provider_uid"] == str(provider.uid)
|
||||
assert params["last_updated"] == config.update_tag
|
||||
|
||||
def test_get_provider_last_scan_findings_returns_latest_scan_data(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
resource = Resource.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
uid="resource-uid",
|
||||
name="Resource",
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="instance",
|
||||
)
|
||||
|
||||
older_scan = Scan.objects.create(
|
||||
name="Older",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
old_finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="older-finding",
|
||||
scan=older_scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.PASS,
|
||||
status_extended="ok",
|
||||
severity=Severity.low,
|
||||
impact=Severity.low,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-old",
|
||||
check_metadata={"checktitle": "Old"},
|
||||
first_seen_at=older_scan.inserted_at,
|
||||
)
|
||||
ResourceFindingMapping.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
resource=resource,
|
||||
finding=old_finding,
|
||||
)
|
||||
|
||||
latest_scan = Scan.objects.create(
|
||||
name="Latest",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="finding-uid",
|
||||
scan=latest_scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.FAIL,
|
||||
status_extended="failed",
|
||||
severity=Severity.high,
|
||||
impact=Severity.high,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-1",
|
||||
check_metadata={"checktitle": "Check title"},
|
||||
first_seen_at=latest_scan.inserted_at,
|
||||
)
|
||||
ResourceFindingMapping.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
resource=resource,
|
||||
finding=finding,
|
||||
)
|
||||
|
||||
latest_scan.refresh_from_db()
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.attack_paths.prowler.rls_transaction",
|
||||
new=lambda *args, **kwargs: nullcontext(),
|
||||
), patch(
|
||||
"tasks.jobs.attack_paths.prowler.READ_REPLICA_ALIAS",
|
||||
"default",
|
||||
):
|
||||
# Generator yields batches, collect all findings from all batches
|
||||
findings_batches = prowler_module.get_provider_last_scan_findings(
|
||||
provider,
|
||||
str(latest_scan.id),
|
||||
)
|
||||
findings_data = []
|
||||
for batch in findings_batches:
|
||||
findings_data.extend(batch)
|
||||
|
||||
assert len(findings_data) == 1
|
||||
finding_dict = findings_data[0]
|
||||
assert finding_dict["id"] == str(finding.id)
|
||||
assert finding_dict["resource_uid"] == resource.uid
|
||||
assert finding_dict["check_title"] == "Check title"
|
||||
assert finding_dict["scan_id"] == str(latest_scan.id)
|
||||
|
||||
def test_enrich_and_flatten_batch_single_resource(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
"""One finding + one resource = one output dict"""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
resource = Resource.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
uid="resource-uid-1",
|
||||
name="Resource 1",
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="instance",
|
||||
)
|
||||
|
||||
scan = Scan.objects.create(
|
||||
name="Test Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="finding-uid",
|
||||
scan=scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.FAIL,
|
||||
status_extended="failed",
|
||||
severity=Severity.high,
|
||||
impact=Severity.high,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-1",
|
||||
check_metadata={"checktitle": "Check title"},
|
||||
first_seen_at=scan.inserted_at,
|
||||
)
|
||||
ResourceFindingMapping.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
resource=resource,
|
||||
finding=finding,
|
||||
)
|
||||
|
||||
# Simulate the dict returned by .values()
|
||||
finding_dict = {
|
||||
"id": finding.id,
|
||||
"uid": finding.uid,
|
||||
"inserted_at": finding.inserted_at,
|
||||
"updated_at": finding.updated_at,
|
||||
"first_seen_at": finding.first_seen_at,
|
||||
"scan_id": scan.id,
|
||||
"delta": finding.delta,
|
||||
"status": finding.status,
|
||||
"status_extended": finding.status_extended,
|
||||
"severity": finding.severity,
|
||||
"check_id": finding.check_id,
|
||||
"check_metadata__checktitle": finding.check_metadata["checktitle"],
|
||||
"muted": finding.muted,
|
||||
"muted_reason": finding.muted_reason,
|
||||
}
|
||||
|
||||
# _enrich_and_flatten_batch queries ResourceFindingMapping directly
|
||||
# No RLS mock needed - test DB doesn't enforce RLS policies
|
||||
with patch(
|
||||
"tasks.jobs.attack_paths.prowler.READ_REPLICA_ALIAS",
|
||||
"default",
|
||||
):
|
||||
result = prowler_module._enrich_and_flatten_batch([finding_dict])
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["resource_uid"] == resource.uid
|
||||
assert result[0]["id"] == str(finding.id)
|
||||
assert result[0]["status"] == "FAIL"
|
||||
|
||||
def test_enrich_and_flatten_batch_multiple_resources(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
"""One finding + three resources = three output dicts"""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
resources = []
|
||||
for i in range(3):
|
||||
resource = Resource.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
uid=f"resource-uid-{i}",
|
||||
name=f"Resource {i}",
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="instance",
|
||||
)
|
||||
resources.append(resource)
|
||||
|
||||
scan = Scan.objects.create(
|
||||
name="Test Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="finding-uid",
|
||||
scan=scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.FAIL,
|
||||
status_extended="failed",
|
||||
severity=Severity.high,
|
||||
impact=Severity.high,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-1",
|
||||
check_metadata={"checktitle": "Check title"},
|
||||
first_seen_at=scan.inserted_at,
|
||||
)
|
||||
|
||||
# Map finding to all 3 resources
|
||||
for resource in resources:
|
||||
ResourceFindingMapping.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
resource=resource,
|
||||
finding=finding,
|
||||
)
|
||||
|
||||
finding_dict = {
|
||||
"id": finding.id,
|
||||
"uid": finding.uid,
|
||||
"inserted_at": finding.inserted_at,
|
||||
"updated_at": finding.updated_at,
|
||||
"first_seen_at": finding.first_seen_at,
|
||||
"scan_id": scan.id,
|
||||
"delta": finding.delta,
|
||||
"status": finding.status,
|
||||
"status_extended": finding.status_extended,
|
||||
"severity": finding.severity,
|
||||
"check_id": finding.check_id,
|
||||
"check_metadata__checktitle": finding.check_metadata["checktitle"],
|
||||
"muted": finding.muted,
|
||||
"muted_reason": finding.muted_reason,
|
||||
}
|
||||
|
||||
# _enrich_and_flatten_batch queries ResourceFindingMapping directly
|
||||
# No RLS mock needed - test DB doesn't enforce RLS policies
|
||||
with patch(
|
||||
"tasks.jobs.attack_paths.prowler.READ_REPLICA_ALIAS",
|
||||
"default",
|
||||
):
|
||||
result = prowler_module._enrich_and_flatten_batch([finding_dict])
|
||||
|
||||
assert len(result) == 3
|
||||
result_resource_uids = {r["resource_uid"] for r in result}
|
||||
assert result_resource_uids == {r.uid for r in resources}
|
||||
|
||||
# All should have same finding data
|
||||
for r in result:
|
||||
assert r["id"] == str(finding.id)
|
||||
assert r["status"] == "FAIL"
|
||||
|
||||
def test_enrich_and_flatten_batch_no_resources_skips(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
"""Finding without resources should be skipped"""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
scan = Scan.objects.create(
|
||||
name="Test Scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
uid="orphan-finding",
|
||||
scan=scan,
|
||||
delta=Finding.DeltaChoices.NEW,
|
||||
status=StatusChoices.FAIL,
|
||||
status_extended="failed",
|
||||
severity=Severity.high,
|
||||
impact=Severity.high,
|
||||
impact_extended="",
|
||||
raw_result={},
|
||||
check_id="check-1",
|
||||
check_metadata={"checktitle": "Check title"},
|
||||
first_seen_at=scan.inserted_at,
|
||||
)
|
||||
# Note: No ResourceFindingMapping created
|
||||
|
||||
finding_dict = {
|
||||
"id": finding.id,
|
||||
"uid": finding.uid,
|
||||
"inserted_at": finding.inserted_at,
|
||||
"updated_at": finding.updated_at,
|
||||
"first_seen_at": finding.first_seen_at,
|
||||
"scan_id": scan.id,
|
||||
"delta": finding.delta,
|
||||
"status": finding.status,
|
||||
"status_extended": finding.status_extended,
|
||||
"severity": finding.severity,
|
||||
"check_id": finding.check_id,
|
||||
"check_metadata__checktitle": finding.check_metadata["checktitle"],
|
||||
"muted": finding.muted,
|
||||
"muted_reason": finding.muted_reason,
|
||||
}
|
||||
|
||||
# Mock logger to verify no warning is emitted
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.prowler.READ_REPLICA_ALIAS",
|
||||
"default",
|
||||
),
|
||||
patch("tasks.jobs.attack_paths.prowler.logger") as mock_logger,
|
||||
):
|
||||
result = prowler_module._enrich_and_flatten_batch([finding_dict])
|
||||
|
||||
assert len(result) == 0
|
||||
mock_logger.warning.assert_not_called()
|
||||
|
||||
def test_generator_is_lazy(self, providers_fixture):
|
||||
"""Generator should not execute queries until iterated"""
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
scan_id = "some-scan-id"
|
||||
|
||||
with (
|
||||
patch("tasks.jobs.attack_paths.prowler.rls_transaction") as mock_rls,
|
||||
patch("tasks.jobs.attack_paths.prowler.Finding") as mock_finding,
|
||||
):
|
||||
# Create generator but don't iterate
|
||||
prowler_module.get_provider_last_scan_findings(provider, scan_id)
|
||||
|
||||
# Nothing should be called yet
|
||||
mock_rls.assert_not_called()
|
||||
mock_finding.objects.filter.assert_not_called()
|
||||
|
||||
def test_load_findings_empty_generator(self, providers_fixture):
|
||||
"""Empty generator should not call neo4j"""
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
|
||||
mock_session = MagicMock()
|
||||
config = SimpleNamespace(update_tag=12345)
|
||||
|
||||
def empty_gen():
|
||||
return
|
||||
yield # Make it a generator
|
||||
|
||||
with (
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.prowler.get_root_node_label",
|
||||
return_value="AWSAccount",
|
||||
),
|
||||
patch(
|
||||
"tasks.jobs.attack_paths.prowler.get_node_uid_field",
|
||||
return_value="arn",
|
||||
),
|
||||
):
|
||||
prowler_module.load_findings(mock_session, empty_gen(), provider, config)
|
||||
|
||||
mock_session.run.assert_not_called()
|
||||
@@ -1,17 +1,27 @@
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from tasks.jobs.backfill import (
|
||||
backfill_compliance_summaries,
|
||||
backfill_provider_compliance_scores,
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
backfill_scan_resource_group_summaries,
|
||||
)
|
||||
|
||||
from api.models import (
|
||||
ComplianceOverviewSummary,
|
||||
Finding,
|
||||
ResourceScanSummary,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
StateChoices,
|
||||
)
|
||||
from prowler.lib.check.models import Severity
|
||||
from prowler.lib.outputs.finding import Status
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
@@ -46,6 +56,45 @@ def get_not_completed_scans(providers_fixture):
|
||||
return scan_1, scan_2
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_categories_fixture(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource = resources_fixture[0]
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_with_categories",
|
||||
scan=scan,
|
||||
delta="new",
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="test_check",
|
||||
check_metadata={"CheckId": "test_check"},
|
||||
categories=["gen-ai", "security"],
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def scan_category_summary_fixture(scans_fixture):
|
||||
scan = scans_fixture[0]
|
||||
return ScanCategorySummary.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
scan=scan,
|
||||
category="existing-category",
|
||||
severity=Severity.critical,
|
||||
total_findings=1,
|
||||
failed_findings=0,
|
||||
new_failed_findings=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBackfillResourceScanSummaries:
|
||||
def test_already_backfilled(self, resource_scan_summary_data):
|
||||
@@ -172,3 +221,194 @@ class TestBackfillComplianceSummaries:
|
||||
assert summary.requirements_failed == expected_counts["requirements_failed"]
|
||||
assert summary.requirements_manual == expected_counts["requirements_manual"]
|
||||
assert summary.total_requirements == expected_counts["total_requirements"]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBackfillScanCategorySummaries:
|
||||
def test_already_backfilled(self, scan_category_summary_fixture):
|
||||
tenant_id = scan_category_summary_fixture.tenant_id
|
||||
scan_id = scan_category_summary_fixture.scan_id
|
||||
|
||||
result = backfill_scan_category_summaries(str(tenant_id), str(scan_id))
|
||||
|
||||
assert result == {"status": "already backfilled"}
|
||||
|
||||
def test_not_completed_scan(self, get_not_completed_scans):
|
||||
for scan in get_not_completed_scans:
|
||||
result = backfill_scan_category_summaries(str(scan.tenant_id), str(scan.id))
|
||||
assert result == {"status": "scan is not completed"}
|
||||
|
||||
def test_no_categories_to_backfill(self, scans_fixture):
|
||||
scan = scans_fixture[1] # Failed scan with no findings
|
||||
result = backfill_scan_category_summaries(str(scan.tenant_id), str(scan.id))
|
||||
assert result == {"status": "no categories to backfill"}
|
||||
|
||||
def test_successful_backfill(self, findings_with_categories_fixture):
|
||||
finding = findings_with_categories_fixture
|
||||
tenant_id = str(finding.tenant_id)
|
||||
scan_id = str(finding.scan_id)
|
||||
|
||||
result = backfill_scan_category_summaries(tenant_id, scan_id)
|
||||
|
||||
# 2 categories × 1 severity = 2 rows
|
||||
assert result == {"status": "backfilled", "categories_count": 2}
|
||||
|
||||
summaries = ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
)
|
||||
assert summaries.count() == 2
|
||||
categories = set(summaries.values_list("category", flat=True))
|
||||
assert categories == {"gen-ai", "security"}
|
||||
|
||||
for summary in summaries:
|
||||
assert summary.severity == Severity.critical
|
||||
assert summary.total_findings == 1
|
||||
assert summary.failed_findings == 1
|
||||
assert summary.new_failed_findings == 1
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_group_fixture(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource = resources_fixture[0]
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_with_group",
|
||||
scan=scan,
|
||||
delta="new",
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.high,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.high,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="test_check",
|
||||
check_metadata={"CheckId": "test_check"},
|
||||
resource_groups="ai_ml",
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def scan_resource_group_summary_fixture(scans_fixture):
|
||||
scan = scans_fixture[0]
|
||||
return ScanGroupSummary.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
scan=scan,
|
||||
resource_group="existing-group",
|
||||
severity=Severity.high,
|
||||
total_findings=1,
|
||||
failed_findings=0,
|
||||
new_failed_findings=0,
|
||||
resources_count=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBackfillScanGroupSummaries:
|
||||
def test_already_backfilled(self, scan_resource_group_summary_fixture):
|
||||
tenant_id = scan_resource_group_summary_fixture.tenant_id
|
||||
scan_id = scan_resource_group_summary_fixture.scan_id
|
||||
|
||||
result = backfill_scan_resource_group_summaries(str(tenant_id), str(scan_id))
|
||||
|
||||
assert result == {"status": "already backfilled"}
|
||||
|
||||
def test_not_completed_scan(self, get_not_completed_scans):
|
||||
for scan in get_not_completed_scans:
|
||||
result = backfill_scan_resource_group_summaries(
|
||||
str(scan.tenant_id), str(scan.id)
|
||||
)
|
||||
assert result == {"status": "scan is not completed"}
|
||||
|
||||
def test_no_resource_groups_to_backfill(self, scans_fixture):
|
||||
scan = scans_fixture[1] # Failed scan with no findings
|
||||
result = backfill_scan_resource_group_summaries(
|
||||
str(scan.tenant_id), str(scan.id)
|
||||
)
|
||||
assert result == {"status": "no resource groups to backfill"}
|
||||
|
||||
def test_successful_backfill(self, findings_with_group_fixture):
|
||||
finding = findings_with_group_fixture
|
||||
tenant_id = str(finding.tenant_id)
|
||||
scan_id = str(finding.scan_id)
|
||||
|
||||
result = backfill_scan_resource_group_summaries(tenant_id, scan_id)
|
||||
|
||||
# 1 resource group × 1 severity = 1 row
|
||||
assert result == {"status": "backfilled", "resource_groups_count": 1}
|
||||
|
||||
summaries = ScanGroupSummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
)
|
||||
assert summaries.count() == 1
|
||||
|
||||
summary = summaries.first()
|
||||
assert summary.resource_group == "ai_ml"
|
||||
assert summary.severity == Severity.high
|
||||
assert summary.total_findings == 1
|
||||
assert summary.failed_findings == 1
|
||||
assert summary.new_failed_findings == 1
|
||||
assert summary.resources_count == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBackfillProviderComplianceScores:
|
||||
def test_no_completed_scans(self, tenants_fixture):
|
||||
tenant = tenants_fixture[2]
|
||||
result = backfill_provider_compliance_scores(str(tenant.id))
|
||||
assert result == {"status": "no completed scans"}
|
||||
|
||||
def test_no_scans_to_process(self, tenants_fixture, scans_fixture):
|
||||
tenant = tenants_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
scan.completed_at = None
|
||||
scan.save()
|
||||
|
||||
result = backfill_provider_compliance_scores(str(tenant.id))
|
||||
assert result == {"status": "no completed scans"}
|
||||
|
||||
@patch("tasks.jobs.backfill.psycopg_connection")
|
||||
def test_successful_backfill_executes_sql_queries(
|
||||
self,
|
||||
mock_psycopg_connection,
|
||||
tenants_fixture,
|
||||
scans_fixture,
|
||||
settings,
|
||||
):
|
||||
"""Test successful backfill executes SQL queries and returns correct stats."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
tenant = tenants_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
|
||||
# Set completed_at to make the scan eligible for backfill
|
||||
scan.completed_at = datetime.now(timezone.utc)
|
||||
scan.save()
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
connection.autocommit = True
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
cursor.rowcount = 5
|
||||
|
||||
result = backfill_provider_compliance_scores(str(tenant.id))
|
||||
|
||||
assert result["status"] == "backfilled"
|
||||
assert result["providers_processed"] == 1
|
||||
assert result["providers_skipped"] == 0
|
||||
assert result["total_upserted"] == 5
|
||||
assert result["tenant_summary_count"] == 5
|
||||
|
||||
@@ -28,6 +28,7 @@ class TestScheduleProviderScan:
|
||||
"tenant_id": str(provider_instance.tenant_id),
|
||||
"provider_id": str(provider_instance.id),
|
||||
},
|
||||
countdown=5,
|
||||
)
|
||||
|
||||
task_name = f"scan-perform-scheduled-{provider_instance.id}"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user