mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-06-15 01:49:29 +00:00
Compare commits
237 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e07e45c8e5 | |||
| a37aea84e7 | |||
| 8d1d041092 | |||
| 6f018183cd | |||
| 8ce56b5ed6 | |||
| ad5095595c | |||
| 3fbe157d10 | |||
| 83d04753ef | |||
| de8e2219c2 | |||
| 2850c40dd5 | |||
| e213afd4e1 | |||
| deada62d66 | |||
| b8d9860a2f | |||
| be759216c4 | |||
| ca9211b5ed | |||
| 3cf7f7845e | |||
| 81e046ecf6 | |||
| 0d363e6100 | |||
| 0719e31b58 | |||
| 19ceb7db88 | |||
| 43875b6ae7 | |||
| 641dc78c3a | |||
| 57b9a2ea10 | |||
| 19e9a9965b | |||
| 3eb2595f6d | |||
| d776356d16 | |||
| 5118d0ecb4 | |||
| df8e465366 | |||
| f4a78d64f1 | |||
| e5cd25e60c | |||
| 7d963751aa | |||
| fa4371bbf6 | |||
| ff6fbcbf48 | |||
| 9bf3702d71 | |||
| ec32be2f1d | |||
| d93c7dcc4d | |||
| 4abead2787 | |||
| d1d03ba421 | |||
| bd47fe2072 | |||
| b395f52a00 | |||
| d14bf31844 | |||
| fcea8dba12 | |||
| 83dac0c59f | |||
| 0bdd1c3f35 | |||
| c6b4b9c94f | |||
| 1c241bb53c | |||
| d15dd53708 | |||
| 15eac061fc | |||
| 597364fb09 | |||
| 13ec7c13b9 | |||
| 89b3b5a81f | |||
| c58ca136f0 | |||
| 594188f7ed | |||
| b9bfdc1a5a | |||
| c83374d4ed | |||
| c1e1fb00c6 | |||
| cbc621cb43 | |||
| 433853493b | |||
| 5aa112d438 | |||
| 1b2c73d2e3 | |||
| 90e3fabc33 | |||
| d4b90abd10 | |||
| 251fc6d4e3 | |||
| dd85da703e | |||
| b549c8dbad | |||
| 79ac7cf6d4 | |||
| d292c6e58a | |||
| 8f361e7e8d | |||
| 3eb278cb9f | |||
| 2f7eec8bca | |||
| 00063c57de | |||
| 2341b5bc7d | |||
| 4015beff20 | |||
| ab475bafc3 | |||
| b4ce01afd4 | |||
| 2b4b23c719 | |||
| 4398b00801 | |||
| e0cf8bffd4 | |||
| 6761f0ffd0 | |||
| 51bbaeb403 | |||
| 6158c16108 | |||
| 0c2c5ea265 | |||
| 3b56166c34 | |||
| b5151a8ee5 | |||
| 0495267351 | |||
| eefe045c18 | |||
| d7d1b22c45 | |||
| 439dbe679b | |||
| 0e9ba4b116 | |||
| 89295f7e7d | |||
| 7cf7758851 | |||
| 06142094cd | |||
| 93f1c02f44 | |||
| e2f30e0987 | |||
| c80710adfc | |||
| 1410fe2ff1 | |||
| 284910d402 | |||
| 04f795bd49 | |||
| 8b5e00163e | |||
| 57d7f77c81 | |||
| 16b1052ff1 | |||
| 978e2c82af | |||
| 0c3ba0b737 | |||
| 4addfcc848 | |||
| 8588cc03f4 | |||
| 7507fea24b | |||
| 18f0fc693e | |||
| 606f505ba3 | |||
| bfce602859 | |||
| ba45b86a82 | |||
| d786bb4440 | |||
| 9424289416 | |||
| 3cbb6175a5 | |||
| 438deef3f8 | |||
| 1cdf4e65b2 | |||
| dbdd02ebd1 | |||
| d264f3daff | |||
| 01fe379b55 | |||
| 50286846e0 | |||
| 20ed8b3d2d | |||
| 45cc6e8b85 | |||
| 962c64eae5 | |||
| 7b56f0640f | |||
| 49c75cc418 | |||
| 56bca7c104 | |||
| faaa172b86 | |||
| 219ce0ba89 | |||
| 2170e5fe12 | |||
| e9efb12aa8 | |||
| 74d72dd56b | |||
| 06d1d214fd | |||
| 902bc9ad57 | |||
| 3616c0a8c0 | |||
| 7288585fec | |||
| 6400dc1059 | |||
| 379c1dc7dd | |||
| eb247360c3 | |||
| 7f12832808 | |||
| 9c387d5742 | |||
| 4a5801c519 | |||
| 85cb39af28 | |||
| c7abd77a1c | |||
| a622b9d965 | |||
| 8bd95a04ce | |||
| 340454ba68 | |||
| 6dff4bfd8b | |||
| 22c88e66a1 | |||
| 3b711f6143 | |||
| dbdce98cf2 | |||
| 53404dfa62 | |||
| c8872dd6ac | |||
| 26fd7d3adc | |||
| cb84bd0f94 | |||
| cb3f3ab35d | |||
| f58c1fddfb | |||
| c1bb51cf1a | |||
| a4e12a94f9 | |||
| 7b1915e489 | |||
| 56d092c87e | |||
| 29a1034658 | |||
| f5c2146d19 | |||
| 069f0d106c | |||
| 803ada7b16 | |||
| 5e033321e8 | |||
| 175d7f95f5 | |||
| 07e82bde56 | |||
| 4661e01c26 | |||
| dda0a2567d | |||
| 56ea498cca | |||
| f9e1e29631 | |||
| 3dadb264cc | |||
| 495aee015e | |||
| d3a000cbc4 | |||
| b2abdbeb60 | |||
| dc852b4595 | |||
| 1250f582a5 | |||
| bb43e924ee | |||
| 0225627a98 | |||
| 3097513525 | |||
| 6af9ff4b4b | |||
| 06fa57a949 | |||
| dc9e91ac4e | |||
| 59f8dfe5ae | |||
| 7e0c5540bb | |||
| 79ec53bfc5 | |||
| ed5f6b3af6 | |||
| 6e135abaa0 | |||
| 65b054f798 | |||
| 28d5b2bb6c | |||
| c8d9f37e70 | |||
| 9d7b9c3327 | |||
| 127b8d8e56 | |||
| 4e9dd46a5e | |||
| 880345bebe | |||
| 1259713fd6 | |||
| 26088868a2 | |||
| e58574e2a4 | |||
| a07e599cfc | |||
| e020b3f74b | |||
| 8e7e376e4f | |||
| a63a3d3f68 | |||
| 10838de636 | |||
| 5ebf455e04 | |||
| 0d59441c5f | |||
| 3b05a1430e | |||
| ea953fb256 | |||
| 2198e461c9 | |||
| 75abd8f54d | |||
| 2f184a493b | |||
| e2e06a78f9 | |||
| de5aba6d4d | |||
| 6e7266eacf | |||
| 58bb66ff27 | |||
| 46bfe02ee8 | |||
| cee9a9a755 | |||
| b11ba9b5cb | |||
| 789fc84e31 | |||
| 6426558b18 | |||
| 9a1ddedd94 | |||
| 0ae400d2b1 | |||
| ced122ac0d | |||
| dc7d2d5aeb | |||
| b6ba6c6e31 | |||
| 30312bbc03 | |||
| 94fe87b4a2 | |||
| 219bc12365 | |||
| 66394ab061 | |||
| 7348ed2179 | |||
| 0b94f2929d | |||
| c23e2502f3 | |||
| c418c59b53 | |||
| 3dc4ab5b83 | |||
| 148a6f341b | |||
| b5df26452a | |||
| 45792686aa | |||
| ee31e82707 | |||
| 0ba1226d88 |
@@ -15,6 +15,13 @@ AUTH_SECRET="N/c6mnaS5+SWq81+819OrzQZlmx1Vxtp/orjttJSmw8="
|
||||
# Google Tag Manager ID
|
||||
NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID=""
|
||||
|
||||
#### MCP Server ####
|
||||
PROWLER_MCP_VERSION=stable
|
||||
# For UI and MCP running on docker:
|
||||
PROWLER_MCP_SERVER_URL=http://mcp-server:8000/mcp
|
||||
# For UI running on host, MCP in docker:
|
||||
# PROWLER_MCP_SERVER_URL=http://localhost:8000/mcp
|
||||
|
||||
#### Code Review Configuration ####
|
||||
# Enable Claude Code standards validation on pre-push hook
|
||||
# Set to 'true' to validate changes against AGENTS.md standards via Claude Code
|
||||
@@ -112,7 +119,7 @@ NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.12.2
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.16.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -87,7 +87,7 @@ runs:
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
if: always()
|
||||
with:
|
||||
name: trivy-scan-report-${{ inputs.image-name }}
|
||||
name: trivy-scan-report-${{ inputs.image-name }}-${{ inputs.image-tag }}
|
||||
path: trivy-report.json
|
||||
retention-days: ${{ inputs.artifact-retention-days }}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"channel": "${{ env.SLACK_CHANNEL_ID }}",
|
||||
"ts": "${{ env.MESSAGE_TS }}",
|
||||
"attachments": [
|
||||
{
|
||||
"color": "${{ env.STATUS_COLOR }}",
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
name: 'API: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
current_api_version: ${{ steps.get_api_version.outputs.current_api_version }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Get current API version
|
||||
id: get_api_version
|
||||
run: |
|
||||
CURRENT_API_VERSION=$(grep -oP '^version = "\K[^"]+' api/pyproject.toml)
|
||||
echo "current_api_version=${CURRENT_API_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "Current API version: $CURRENT_API_VERSION"
|
||||
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next API minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}"
|
||||
|
||||
# API version follows Prowler minor + 1
|
||||
# For Prowler 5.17.0 -> API 1.18.0
|
||||
# For next master (Prowler 5.18.0) -> API 1.19.0
|
||||
NEXT_API_VERSION=1.$((MINOR_VERSION + 2)).0
|
||||
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_API_VERSION=${NEXT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0"
|
||||
echo "Current API version: $CURRENT_API_VERSION"
|
||||
echo "Next API minor version (for master): $NEXT_API_VERSION"
|
||||
|
||||
- name: Bump API versions in files for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_VERSION}\"|" api/src/backend/api/v1/views.py
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next API minor version to master
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}'
|
||||
branch: api-version-bump-to-v${{ env.NEXT_API_VERSION }}
|
||||
title: 'chore(api): Bump version to v${{ env.NEXT_API_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler API version to v${{ env.NEXT_API_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Calculate first API patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}"
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
# API version follows Prowler minor + 1
|
||||
# For Prowler 5.17.0 release -> version branch v5.17 should have API 1.18.1
|
||||
FIRST_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).1
|
||||
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "FIRST_API_PATCH_VERSION=${FIRST_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.0"
|
||||
echo "First API patch version (for ${VERSION_BRANCH}): $FIRST_API_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump API versions in files for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${FIRST_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${FIRST_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${FIRST_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for first API patch version to version branch
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}'
|
||||
branch: api-version-bump-to-v${{ env.FIRST_API_PATCH_VERSION }}
|
||||
title: 'chore(api): Bump version to v${{ env.FIRST_API_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler API version to v${{ env.FIRST_API_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next API patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }}
|
||||
CURRENT_API_VERSION="${{ needs.detect-release-type.outputs.current_api_version }}"
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
# Extract current API patch to increment it
|
||||
if [[ $CURRENT_API_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
API_PATCH=${BASH_REMATCH[3]}
|
||||
|
||||
# API version follows Prowler minor + 1
|
||||
# Keep same API minor (based on Prowler minor), increment patch
|
||||
NEXT_API_PATCH_VERSION=1.$((MINOR_VERSION + 1)).$((API_PATCH + 1))
|
||||
|
||||
echo "CURRENT_API_VERSION=${CURRENT_API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_API_PATCH_VERSION=${NEXT_API_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Prowler release version: ${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}"
|
||||
echo "Current API version: $CURRENT_API_VERSION"
|
||||
echo "Next API patch version: $NEXT_API_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
else
|
||||
echo "::error::Invalid API version format: $CURRENT_API_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Bump API versions in files for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s|spectacular_settings.VERSION = \"${CURRENT_API_VERSION}\"|spectacular_settings.VERSION = \"${NEXT_API_PATCH_VERSION}\"|" api/src/backend/api/v1/views.py
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next API patch version to version branch
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}'
|
||||
branch: api-version-bump-to-v${{ env.NEXT_API_PATCH_VERSION }}
|
||||
title: 'chore(api): Bump version to v${{ env.NEXT_API_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler API version to v${{ env.NEXT_API_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
|
||||
@@ -42,15 +42,15 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/api-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -7,10 +7,16 @@ on:
|
||||
paths:
|
||||
- 'api/**'
|
||||
- 'prowler/**'
|
||||
- '.github/workflows/api-build-lint-push-containers.yml'
|
||||
- '.github/workflows/api-container-build-push.yml'
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag (e.g., 5.14.0)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -22,7 +28,7 @@ concurrency:
|
||||
env:
|
||||
# Tags
|
||||
LATEST_TAG: latest
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }}
|
||||
STABLE_TAG: stable
|
||||
WORKING_DIRECTORY: ./api
|
||||
|
||||
@@ -42,8 +48,34 @@ jobs:
|
||||
id: set-short-sha
|
||||
run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
|
||||
|
||||
container-build-push:
|
||||
notify-release-started:
|
||||
if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
needs: setup
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
message-ts: ${{ steps.slack-notification.outputs.ts }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Notify container push started
|
||||
id: slack-notification
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: API
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
|
||||
|
||||
container-build-push:
|
||||
needs: [setup, notify-release-started]
|
||||
if: always() && needs.setup.result == 'success' && (needs.notify-release-started.result == 'success' || needs.notify-release-started.result == 'skipped')
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -61,7 +93,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
@@ -73,7 +105,8 @@ jobs:
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build and push API container for ${{ matrix.arch }}
|
||||
if: github.event_name == 'push' || github.event_name == 'release'
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
@@ -84,39 +117,10 @@ jobs:
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Notify container push started
|
||||
if: github.event_name == 'release'
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: API
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
|
||||
|
||||
- name: Notify container push completed
|
||||
if: github.event_name == 'release' && always()
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: API
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.container-push.outcome }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
needs: [setup, container-build-push]
|
||||
if: github.event_name == 'push' || github.event_name == 'release'
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
@@ -139,7 +143,7 @@ jobs:
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
|
||||
|
||||
- name: Create and push manifests for release event
|
||||
if: github.event_name == 'release'
|
||||
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} \
|
||||
@@ -159,6 +163,40 @@ jobs:
|
||||
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true
|
||||
echo "Cleanup completed"
|
||||
|
||||
notify-release-completed:
|
||||
if: always() && needs.notify-release-started.result == 'success' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
needs: [setup, notify-release-started, container-build-push, create-manifest]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Determine overall outcome
|
||||
id: outcome
|
||||
run: |
|
||||
if [[ "${{ needs.container-build-push.result }}" == "success" && "${{ needs.create-manifest.result }}" == "success" ]]; then
|
||||
echo "outcome=success" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "outcome=failure" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Notify container push completed
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
MESSAGE_TS: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
COMPONENT: API
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
if: github.event_name == 'push'
|
||||
needs: [setup, container-build-push]
|
||||
@@ -169,7 +207,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Trigger API deployment
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -20,6 +20,7 @@ env:
|
||||
|
||||
jobs:
|
||||
api-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
@@ -27,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
@@ -43,7 +44,17 @@ jobs:
|
||||
ignore: DL3013
|
||||
|
||||
api-container-build-and-scan:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -52,7 +63,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
@@ -68,22 +79,23 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build container
|
||||
- name: Build container for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: ${{ env.API_WORKING_DIR }}
|
||||
push: false
|
||||
load: true
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Scan container with Trivy
|
||||
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
|
||||
- name: Scan container with Trivy for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
name: 'Docs: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
current_docs_version: ${{ steps.get_docs_version.outputs.current_docs_version }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Get current documentation version
|
||||
id: get_docs_version
|
||||
run: |
|
||||
CURRENT_DOCS_VERSION=$(grep -oP 'PROWLER_UI_VERSION="\K[^"]+' docs/getting-started/installation/prowler-app.mdx)
|
||||
echo "current_docs_version=${CURRENT_DOCS_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "Current documentation version: $CURRENT_DOCS_VERSION"
|
||||
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}"
|
||||
|
||||
NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current documentation version: $CURRENT_DOCS_VERSION"
|
||||
echo "Current release version: $PROWLER_VERSION"
|
||||
echo "Next minor version: $NEXT_MINOR_VERSION"
|
||||
|
||||
- name: Bump versions in documentation for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to master
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
- All `*.mdx` files with `<VersionBadge>` components
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Calculate first patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}"
|
||||
|
||||
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "First patch version: $FIRST_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump versions in documentation for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to version branch
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}-branch
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }}
|
||||
CURRENT_DOCS_VERSION="${{ needs.detect-release-type.outputs.current_docs_version }}"
|
||||
|
||||
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "CURRENT_DOCS_VERSION=${CURRENT_DOCS_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current documentation version: $CURRENT_DOCS_VERSION"
|
||||
echo "Current release version: $PROWLER_VERSION"
|
||||
echo "Next patch version: $NEXT_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump versions in documentation for patch version
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Update prowler-app.mdx with current release version
|
||||
sed -i "s|PROWLER_UI_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_UI_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
sed -i "s|PROWLER_API_VERSION=\"${CURRENT_DOCS_VERSION}\"|PROWLER_API_VERSION=\"${PROWLER_VERSION}\"|" docs/getting-started/installation/prowler-app.mdx
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for documentation update to version branch
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
branch: docs-version-update-to-v${{ env.PROWLER_VERSION }}
|
||||
title: 'docs: Update version to v${{ env.PROWLER_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Update Prowler documentation version references to v${{ env.PROWLER_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `docs/getting-started/installation/prowler-app.mdx`: `PROWLER_UI_VERSION` and `PROWLER_API_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -23,11 +23,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Scan for secrets with TruffleHog
|
||||
uses: trufflesecurity/trufflehog@b84c3d14d189e16da175e2c27fa8136603783ffc # v3.90.12
|
||||
uses: trufflesecurity/trufflehog@aade3bff5594fe8808578dd4db3dfeae9bf2abdc # v3.91.1
|
||||
with:
|
||||
extra_args: '--results=verified,unknown'
|
||||
|
||||
@@ -10,6 +10,12 @@ on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag (e.g., 5.14.0)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -21,7 +27,7 @@ concurrency:
|
||||
env:
|
||||
# Tags
|
||||
LATEST_TAG: latest
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }}
|
||||
STABLE_TAG: stable
|
||||
WORKING_DIRECTORY: ./mcp_server
|
||||
|
||||
@@ -41,9 +47,35 @@ jobs:
|
||||
id: set-short-sha
|
||||
run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
|
||||
|
||||
container-build-push:
|
||||
notify-release-started:
|
||||
if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
needs: setup
|
||||
runs-on: ${{ matrix.runner }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
message-ts: ${{ steps.slack-notification.outputs.ts }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Notify container push started
|
||||
id: slack-notification
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: MCP
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
|
||||
|
||||
container-build-push:
|
||||
needs: [setup, notify-release-started]
|
||||
if: always() && needs.setup.result == 'success' && (needs.notify-release-started.result == 'success' || needs.notify-release-started.result == 'skipped')
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -59,7 +91,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
@@ -71,7 +103,8 @@ jobs:
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build and push MCP container for ${{ matrix.arch }}
|
||||
if: github.event_name == 'push' || github.event_name == 'release'
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
@@ -90,39 +123,10 @@ jobs:
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Notify container push started
|
||||
if: github.event_name == 'release'
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: MCP
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
|
||||
|
||||
- name: Notify container push completed
|
||||
if: github.event_name == 'release' && always()
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: MCP
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.container-push.outcome }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
needs: [setup, container-build-push]
|
||||
if: github.event_name == 'push' || github.event_name == 'release'
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
@@ -145,14 +149,14 @@ jobs:
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
|
||||
|
||||
- name: Create and push manifests for release event
|
||||
if: github.event_name == 'release'
|
||||
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} \
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
|
||||
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@main
|
||||
@@ -165,6 +169,40 @@ jobs:
|
||||
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true
|
||||
echo "Cleanup completed"
|
||||
|
||||
notify-release-completed:
|
||||
if: always() && needs.notify-release-started.result == 'success' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
needs: [setup, notify-release-started, container-build-push, create-manifest]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Determine overall outcome
|
||||
id: outcome
|
||||
run: |
|
||||
if [[ "${{ needs.container-build-push.result }}" == "success" && "${{ needs.create-manifest.result }}" == "success" ]]; then
|
||||
echo "outcome=success" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "outcome=failure" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Notify container push completed
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
MESSAGE_TS: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
COMPONENT: MCP
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
if: github.event_name == 'push'
|
||||
needs: [setup, container-build-push]
|
||||
@@ -175,7 +213,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Trigger MCP deployment
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -20,6 +20,7 @@ env:
|
||||
|
||||
jobs:
|
||||
mcp-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
@@ -27,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
@@ -42,7 +43,17 @@ jobs:
|
||||
dockerfile: mcp_server/Dockerfile
|
||||
|
||||
mcp-container-build-and-scan:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -51,7 +62,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for MCP changes
|
||||
id: check-changes
|
||||
@@ -66,22 +77,23 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build MCP container
|
||||
- name: Build MCP container for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: ${{ env.MCP_WORKING_DIR }}
|
||||
push: false
|
||||
load: true
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Scan MCP container with Trivy
|
||||
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
|
||||
- name: Scan MCP container with Trivy for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
name: "MCP: PyPI Release"
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- "published"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
||||
PYTHON_VERSION: "3.12"
|
||||
WORKING_DIRECTORY: ./mcp_server
|
||||
|
||||
jobs:
|
||||
validate-release:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
prowler_version: ${{ steps.parse-version.outputs.version }}
|
||||
major_version: ${{ steps.parse-version.outputs.major }}
|
||||
|
||||
steps:
|
||||
- name: Parse and validate version
|
||||
id: parse-version
|
||||
run: |
|
||||
PROWLER_VERSION="${{ env.RELEASE_TAG }}"
|
||||
echo "version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Extract major version
|
||||
MAJOR_VERSION="${PROWLER_VERSION%%.*}"
|
||||
echo "major=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Validate major version (only Prowler 3, 4, 5 supported)
|
||||
case ${MAJOR_VERSION} in
|
||||
3|4|5)
|
||||
echo "✓ Releasing Prowler MCP for tag ${PROWLER_VERSION}"
|
||||
;;
|
||||
*)
|
||||
echo "::error::Unsupported Prowler major version: ${MAJOR_VERSION}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
publish-prowler-mcp:
|
||||
needs: validate-release
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
environment:
|
||||
name: pypi-prowler-mcp
|
||||
url: https://pypi.org/project/prowler-mcp/
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Build prowler-mcp package
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: uv build
|
||||
|
||||
- name: Publish prowler-mcp package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
with:
|
||||
packages-dir: ${{ env.WORKING_DIRECTORY }}/dist/
|
||||
print-hash: true
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout PR head
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -13,7 +13,10 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
trigger-cloud-pull-request:
|
||||
if: github.event.pull_request.merged == true && github.repository == 'prowler-cloud/prowler'
|
||||
if: |
|
||||
github.event.pull_request.merged == true &&
|
||||
github.repository == 'prowler-cloud/prowler' &&
|
||||
!contains(github.event.pull_request.labels.*.name, 'skip-sync')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
@@ -26,7 +29,7 @@ jobs:
|
||||
echo "SHORT_SHA=${SHORT_SHA::7}" >> $GITHUB_ENV
|
||||
|
||||
- name: Trigger Cloud repository pull request
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -27,13 +27,13 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
@@ -88,59 +88,56 @@ jobs:
|
||||
|
||||
- name: Read changelog versions from release branch
|
||||
run: |
|
||||
# Function to extract the latest version from changelog
|
||||
extract_latest_version() {
|
||||
# Function to extract the version for a specific Prowler release from changelog
|
||||
# This looks for entries with "(Prowler X.Y.Z)" to find the released version
|
||||
extract_version_for_release() {
|
||||
local changelog_file="$1"
|
||||
local prowler_version="$2"
|
||||
if [ -f "$changelog_file" ]; then
|
||||
# Extract the first version entry (most recent) from changelog
|
||||
# Format: ## [version] (1.2.3) or ## [vversion] (v1.2.3)
|
||||
local version=$(grep -m 1 '^## \[' "$changelog_file" | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]')
|
||||
# Extract version that matches this Prowler release
|
||||
# Format: ## [version] (Prowler X.Y.Z) or ## [vversion] (Prowler vX.Y.Z)
|
||||
local version=$(grep '^## \[' "$changelog_file" | grep "(Prowler v\?${prowler_version})" | head -1 | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]')
|
||||
echo "$version"
|
||||
else
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
# Read actual versions from changelogs (source of truth)
|
||||
UI_VERSION=$(extract_latest_version "ui/CHANGELOG.md")
|
||||
API_VERSION=$(extract_latest_version "api/CHANGELOG.md")
|
||||
SDK_VERSION=$(extract_latest_version "prowler/CHANGELOG.md")
|
||||
MCP_VERSION=$(extract_latest_version "mcp_server/CHANGELOG.md")
|
||||
# Read versions from changelogs for this specific Prowler release
|
||||
SDK_VERSION=$(extract_version_for_release "prowler/CHANGELOG.md" "$PROWLER_VERSION")
|
||||
API_VERSION=$(extract_version_for_release "api/CHANGELOG.md" "$PROWLER_VERSION")
|
||||
UI_VERSION=$(extract_version_for_release "ui/CHANGELOG.md" "$PROWLER_VERSION")
|
||||
MCP_VERSION=$(extract_version_for_release "mcp_server/CHANGELOG.md" "$PROWLER_VERSION")
|
||||
|
||||
echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "SDK_VERSION=${SDK_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "MCP_VERSION=${MCP_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
if [ -n "$UI_VERSION" ]; then
|
||||
echo "Read UI version from changelog: $UI_VERSION"
|
||||
if [ -n "$SDK_VERSION" ]; then
|
||||
echo "✓ SDK version for Prowler $PROWLER_VERSION: $SDK_VERSION"
|
||||
else
|
||||
echo "Warning: No UI version found in ui/CHANGELOG.md"
|
||||
echo "ℹ No SDK version found for Prowler $PROWLER_VERSION in prowler/CHANGELOG.md"
|
||||
fi
|
||||
|
||||
if [ -n "$API_VERSION" ]; then
|
||||
echo "Read API version from changelog: $API_VERSION"
|
||||
echo "✓ API version for Prowler $PROWLER_VERSION: $API_VERSION"
|
||||
else
|
||||
echo "Warning: No API version found in api/CHANGELOG.md"
|
||||
echo "ℹ No API version found for Prowler $PROWLER_VERSION in api/CHANGELOG.md"
|
||||
fi
|
||||
|
||||
if [ -n "$SDK_VERSION" ]; then
|
||||
echo "Read SDK version from changelog: $SDK_VERSION"
|
||||
if [ -n "$UI_VERSION" ]; then
|
||||
echo "✓ UI version for Prowler $PROWLER_VERSION: $UI_VERSION"
|
||||
else
|
||||
echo "Warning: No SDK version found in prowler/CHANGELOG.md"
|
||||
echo "ℹ No UI version found for Prowler $PROWLER_VERSION in ui/CHANGELOG.md"
|
||||
fi
|
||||
|
||||
if [ -n "$MCP_VERSION" ]; then
|
||||
echo "Read MCP version from changelog: $MCP_VERSION"
|
||||
echo "✓ MCP version for Prowler $PROWLER_VERSION: $MCP_VERSION"
|
||||
else
|
||||
echo "Warning: No MCP version found in mcp_server/CHANGELOG.md"
|
||||
echo "ℹ No MCP version found for Prowler $PROWLER_VERSION in mcp_server/CHANGELOG.md"
|
||||
fi
|
||||
|
||||
echo "UI version: $UI_VERSION"
|
||||
echo "API version: $API_VERSION"
|
||||
echo "SDK version: $SDK_VERSION"
|
||||
echo "MCP version: $MCP_VERSION"
|
||||
|
||||
- name: Extract and combine changelog entries
|
||||
run: |
|
||||
set -e
|
||||
@@ -166,70 +163,54 @@ jobs:
|
||||
|
||||
# Remove --- separators
|
||||
sed -i '/^---$/d' "$output_file"
|
||||
|
||||
# Remove only trailing empty lines (not all empty lines)
|
||||
sed -i -e :a -e '/^\s*$/d;N;ba' "$output_file"
|
||||
}
|
||||
|
||||
# Calculate expected versions for this release
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
EXPECTED_UI_VERSION="1.${BASH_REMATCH[2]}.${BASH_REMATCH[3]}"
|
||||
EXPECTED_API_VERSION="1.$((${BASH_REMATCH[2]} + 1)).${BASH_REMATCH[3]}"
|
||||
|
||||
echo "Expected UI version for this release: $EXPECTED_UI_VERSION"
|
||||
echo "Expected API version for this release: $EXPECTED_API_VERSION"
|
||||
fi
|
||||
|
||||
# Determine if components have changes for this specific release
|
||||
# UI has changes if its current version matches what we expect for this release
|
||||
if [ -n "$UI_VERSION" ] && [ "$UI_VERSION" = "$EXPECTED_UI_VERSION" ]; then
|
||||
echo "HAS_UI_CHANGES=true" >> $GITHUB_ENV
|
||||
echo "✓ UI changes detected - version matches expected: $UI_VERSION"
|
||||
extract_changelog "ui/CHANGELOG.md" "$UI_VERSION" "ui_changelog.md"
|
||||
else
|
||||
echo "HAS_UI_CHANGES=false" >> $GITHUB_ENV
|
||||
echo "ℹ No UI changes for this release (current: $UI_VERSION, expected: $EXPECTED_UI_VERSION)"
|
||||
touch "ui_changelog.md"
|
||||
fi
|
||||
|
||||
# API has changes if its current version matches what we expect for this release
|
||||
if [ -n "$API_VERSION" ] && [ "$API_VERSION" = "$EXPECTED_API_VERSION" ]; then
|
||||
echo "HAS_API_CHANGES=true" >> $GITHUB_ENV
|
||||
echo "✓ API changes detected - version matches expected: $API_VERSION"
|
||||
extract_changelog "api/CHANGELOG.md" "$API_VERSION" "api_changelog.md"
|
||||
else
|
||||
echo "HAS_API_CHANGES=false" >> $GITHUB_ENV
|
||||
echo "ℹ No API changes for this release (current: $API_VERSION, expected: $EXPECTED_API_VERSION)"
|
||||
touch "api_changelog.md"
|
||||
fi
|
||||
|
||||
# SDK has changes if its current version matches the input version
|
||||
if [ -n "$SDK_VERSION" ] && [ "$SDK_VERSION" = "$PROWLER_VERSION" ]; then
|
||||
if [ -n "$SDK_VERSION" ]; then
|
||||
echo "HAS_SDK_CHANGES=true" >> $GITHUB_ENV
|
||||
echo "✓ SDK changes detected - version matches input: $SDK_VERSION"
|
||||
extract_changelog "prowler/CHANGELOG.md" "$PROWLER_VERSION" "prowler_changelog.md"
|
||||
HAS_SDK_CHANGES="true"
|
||||
echo "✓ SDK changes detected - version: $SDK_VERSION"
|
||||
extract_changelog "prowler/CHANGELOG.md" "$SDK_VERSION" "prowler_changelog.md"
|
||||
else
|
||||
echo "HAS_SDK_CHANGES=false" >> $GITHUB_ENV
|
||||
echo "ℹ No SDK changes for this release (current: $SDK_VERSION, input: $PROWLER_VERSION)"
|
||||
HAS_SDK_CHANGES="false"
|
||||
echo "ℹ No SDK changes for this release"
|
||||
touch "prowler_changelog.md"
|
||||
fi
|
||||
|
||||
# MCP has changes if the changelog references this Prowler version
|
||||
# Check if the changelog contains "(Prowler X.Y.Z)" or "(Prowler UNRELEASED)"
|
||||
if [ -f "mcp_server/CHANGELOG.md" ]; then
|
||||
MCP_PROWLER_REF=$(grep -m 1 "^## \[.*\] (Prowler" mcp_server/CHANGELOG.md | sed -E 's/.*\(Prowler ([^)]+)\).*/\1/' | tr -d '[:space:]')
|
||||
if [ "$MCP_PROWLER_REF" = "$PROWLER_VERSION" ] || [ "$MCP_PROWLER_REF" = "UNRELEASED" ]; then
|
||||
echo "HAS_MCP_CHANGES=true" >> $GITHUB_ENV
|
||||
echo "✓ MCP changes detected - Prowler reference: $MCP_PROWLER_REF (version: $MCP_VERSION)"
|
||||
extract_changelog "mcp_server/CHANGELOG.md" "$MCP_VERSION" "mcp_changelog.md"
|
||||
else
|
||||
echo "HAS_MCP_CHANGES=false" >> $GITHUB_ENV
|
||||
echo "ℹ No MCP changes for this release (Prowler reference: $MCP_PROWLER_REF, input: $PROWLER_VERSION)"
|
||||
touch "mcp_changelog.md"
|
||||
fi
|
||||
if [ -n "$API_VERSION" ]; then
|
||||
echo "HAS_API_CHANGES=true" >> $GITHUB_ENV
|
||||
HAS_API_CHANGES="true"
|
||||
echo "✓ API changes detected - version: $API_VERSION"
|
||||
extract_changelog "api/CHANGELOG.md" "$API_VERSION" "api_changelog.md"
|
||||
else
|
||||
echo "HAS_API_CHANGES=false" >> $GITHUB_ENV
|
||||
HAS_API_CHANGES="false"
|
||||
echo "ℹ No API changes for this release"
|
||||
touch "api_changelog.md"
|
||||
fi
|
||||
|
||||
if [ -n "$UI_VERSION" ]; then
|
||||
echo "HAS_UI_CHANGES=true" >> $GITHUB_ENV
|
||||
HAS_UI_CHANGES="true"
|
||||
echo "✓ UI changes detected - version: $UI_VERSION"
|
||||
extract_changelog "ui/CHANGELOG.md" "$UI_VERSION" "ui_changelog.md"
|
||||
else
|
||||
echo "HAS_UI_CHANGES=false" >> $GITHUB_ENV
|
||||
HAS_UI_CHANGES="false"
|
||||
echo "ℹ No UI changes for this release"
|
||||
touch "ui_changelog.md"
|
||||
fi
|
||||
|
||||
if [ -n "$MCP_VERSION" ]; then
|
||||
echo "HAS_MCP_CHANGES=true" >> $GITHUB_ENV
|
||||
HAS_MCP_CHANGES="true"
|
||||
echo "✓ MCP changes detected - version: $MCP_VERSION"
|
||||
extract_changelog "mcp_server/CHANGELOG.md" "$MCP_VERSION" "mcp_changelog.md"
|
||||
else
|
||||
echo "HAS_MCP_CHANGES=false" >> $GITHUB_ENV
|
||||
echo "ℹ No MCP changelog found"
|
||||
HAS_MCP_CHANGES="false"
|
||||
echo "ℹ No MCP changes for this release"
|
||||
touch "mcp_changelog.md"
|
||||
fi
|
||||
|
||||
@@ -325,6 +306,17 @@ jobs:
|
||||
fi
|
||||
echo "✓ api/src/backend/api/v1/views.py version: $CURRENT_API_VERSION"
|
||||
|
||||
- name: Verify API version in api/src/backend/api/specs/v1.yaml
|
||||
if: ${{ env.HAS_API_CHANGES == 'true' }}
|
||||
run: |
|
||||
CURRENT_API_VERSION=$(grep '^ version: ' api/src/backend/api/specs/v1.yaml | sed -E 's/ version: ([0-9]+\.[0-9]+\.[0-9]+)/\1/' | tr -d '[:space:]')
|
||||
API_VERSION_TRIMMED=$(echo "$API_VERSION" | tr -d '[:space:]')
|
||||
if [ "$CURRENT_API_VERSION" != "$API_VERSION_TRIMMED" ]; then
|
||||
echo "ERROR: API version mismatch in api/src/backend/api/specs/v1.yaml (expected: '$API_VERSION_TRIMMED', found: '$CURRENT_API_VERSION')"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ api/src/backend/api/specs/v1.yaml version: $CURRENT_API_VERSION"
|
||||
|
||||
- name: Update API prowler dependency for minor release
|
||||
if: ${{ env.PATCH_VERSION == '0' }}
|
||||
run: |
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next minor version
|
||||
run: |
|
||||
@@ -86,7 +86,6 @@ jobs:
|
||||
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_MINOR_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_MINOR_VERSION}\"|" prowler/config/config.py
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
@@ -100,7 +99,7 @@ jobs:
|
||||
commit-message: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
branch: version-bump-to-v${{ env.NEXT_MINOR_VERSION }}
|
||||
title: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
labels: no-changelog
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
@@ -111,7 +110,7 @@ jobs:
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
@@ -135,7 +134,6 @@ jobs:
|
||||
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${FIRST_PATCH_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${FIRST_PATCH_VERSION}\"|" prowler/config/config.py
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
@@ -149,7 +147,7 @@ jobs:
|
||||
commit-message: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
branch: version-bump-to-v${{ env.FIRST_PATCH_VERSION }}
|
||||
title: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
labels: no-changelog
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
@@ -169,7 +167,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next patch version
|
||||
run: |
|
||||
@@ -193,7 +191,6 @@ jobs:
|
||||
|
||||
sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_PATCH_VERSION}\"|" pyproject.toml
|
||||
sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_PATCH_VERSION}\"|" prowler/config/config.py
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
@@ -207,7 +204,7 @@ jobs:
|
||||
commit-message: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
branch: version-bump-to-v${{ env.NEXT_PATCH_VERSION }}
|
||||
title: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
labels: no-changelog
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'poetry'
|
||||
|
||||
@@ -49,15 +49,15 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/sdk-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -16,6 +16,12 @@ on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag (e.g., 5.14.0)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -44,36 +50,21 @@ env:
|
||||
AWS_REGION: us-east-1
|
||||
|
||||
jobs:
|
||||
container-build-push:
|
||||
setup:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
prowler_version: ${{ steps.get-prowler-version.outputs.prowler_version }}
|
||||
prowler_version_major: ${{ steps.get-prowler-version.outputs.prowler_version_major }}
|
||||
latest_tag: ${{ steps.get-prowler-version.outputs.latest_tag }}
|
||||
stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }}
|
||||
env:
|
||||
POETRY_VIRTUALENVS_CREATE: 'false'
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
@@ -87,32 +78,24 @@ jobs:
|
||||
run: |
|
||||
PROWLER_VERSION="$(poetry version -s 2>/dev/null)"
|
||||
echo "prowler_version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "PROWLER_VERSION=${PROWLER_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
# Extract major version
|
||||
PROWLER_VERSION_MAJOR="${PROWLER_VERSION%%.*}"
|
||||
echo "prowler_version_major=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_OUTPUT}"
|
||||
echo "PROWLER_VERSION_MAJOR=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_ENV}"
|
||||
|
||||
# Set version-specific tags
|
||||
case ${PROWLER_VERSION_MAJOR} in
|
||||
3)
|
||||
echo "LATEST_TAG=v3-latest" >> "${GITHUB_ENV}"
|
||||
echo "STABLE_TAG=v3-stable" >> "${GITHUB_ENV}"
|
||||
echo "latest_tag=v3-latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=v3-stable" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Prowler v3 detected - tags: v3-latest, v3-stable"
|
||||
;;
|
||||
4)
|
||||
echo "LATEST_TAG=v4-latest" >> "${GITHUB_ENV}"
|
||||
echo "STABLE_TAG=v4-stable" >> "${GITHUB_ENV}"
|
||||
echo "latest_tag=v4-latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=v4-stable" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Prowler v4 detected - tags: v4-latest, v4-stable"
|
||||
;;
|
||||
5)
|
||||
echo "LATEST_TAG=latest" >> "${GITHUB_ENV}"
|
||||
echo "STABLE_TAG=stable" >> "${GITHUB_ENV}"
|
||||
echo "latest_tag=latest" >> "${GITHUB_OUTPUT}"
|
||||
echo "stable_tag=stable" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Prowler v5 detected - tags: latest, stable"
|
||||
@@ -123,6 +106,53 @@ jobs:
|
||||
;;
|
||||
esac
|
||||
|
||||
notify-release-started:
|
||||
if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
needs: setup
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
message-ts: ${{ steps.slack-notification.outputs.ts }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Notify container push started
|
||||
id: slack-notification
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: SDK
|
||||
RELEASE_TAG: ${{ needs.setup.outputs.prowler_version }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
|
||||
|
||||
container-build-push:
|
||||
needs: [setup, notify-release-started]
|
||||
if: always() && needs.setup.result == 'success' && (needs.notify-release-started.result == 'success' || needs.notify-release-started.result == 'skipped')
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
@@ -142,7 +172,8 @@ jobs:
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build and push SDK container for ${{ matrix.arch }}
|
||||
if: github.event_name == 'push' || github.event_name == 'release'
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: .
|
||||
@@ -150,43 +181,14 @@ jobs:
|
||||
push: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: |
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }}-${{ matrix.arch }}
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Notify container push started
|
||||
if: github.event_name == 'release'
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: SDK
|
||||
RELEASE_TAG: ${{ env.PROWLER_VERSION }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
|
||||
|
||||
- name: Notify container push completed
|
||||
if: github.event_name == 'release' && always()
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: SDK
|
||||
RELEASE_TAG: ${{ env.PROWLER_VERSION }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.container-push.outcome }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
needs: [container-build-push]
|
||||
if: github.event_name == 'push' || github.event_name == 'release'
|
||||
needs: [setup, container-build-push]
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
@@ -196,6 +198,15 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
|
||||
env:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
@@ -203,24 +214,24 @@ jobs:
|
||||
if: github.event_name == 'push'
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \
|
||||
-t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \
|
||||
-t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64 \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }} \
|
||||
-t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }} \
|
||||
-t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }} \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-amd64 \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-arm64
|
||||
|
||||
- name: Create and push manifests for release event
|
||||
if: github.event_name == 'release'
|
||||
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.prowler_version }} \
|
||||
-t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.stable_tag }} \
|
||||
-t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.prowler_version }} \
|
||||
-t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.stable_tag }} \
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.prowler_version }} \
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.stable_tag }} \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64 \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64
|
||||
-t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.setup.outputs.prowler_version }} \
|
||||
-t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.setup.outputs.stable_tag }} \
|
||||
-t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.setup.outputs.prowler_version }} \
|
||||
-t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.setup.outputs.stable_tag }} \
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.prowler_version }} \
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.stable_tag }} \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-amd64 \
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-arm64
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
@@ -230,13 +241,47 @@ jobs:
|
||||
if: always()
|
||||
run: |
|
||||
echo "Cleaning up intermediate tags..."
|
||||
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64" || true
|
||||
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64" || true
|
||||
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-amd64" || true
|
||||
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.latest_tag }}-arm64" || true
|
||||
echo "Cleanup completed"
|
||||
|
||||
notify-release-completed:
|
||||
if: always() && needs.notify-release-started.result == 'success' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
needs: [setup, notify-release-started, container-build-push, create-manifest]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Determine overall outcome
|
||||
id: outcome
|
||||
run: |
|
||||
if [[ "${{ needs.container-build-push.result }}" == "success" && "${{ needs.create-manifest.result }}" == "success" ]]; then
|
||||
echo "outcome=success" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "outcome=failure" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Notify container push completed
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
MESSAGE_TS: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
COMPONENT: SDK
|
||||
RELEASE_TAG: ${{ needs.setup.outputs.prowler_version }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
dispatch-v3-deployment:
|
||||
if: needs.container-build-push.outputs.prowler_version_major == '3'
|
||||
needs: container-build-push
|
||||
if: needs.setup.outputs.prowler_version_major == '3'
|
||||
needs: [setup, container-build-push]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
@@ -249,7 +294,7 @@ jobs:
|
||||
|
||||
- name: Dispatch v3 deployment (latest)
|
||||
if: github.event_name == 'push'
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
|
||||
@@ -258,9 +303,9 @@ jobs:
|
||||
|
||||
- name: Dispatch v3 deployment (release)
|
||||
if: github.event_name == 'release'
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}
|
||||
event-type: dispatch
|
||||
client-payload: '{"version":"release","tag":"${{ needs.container-build-push.outputs.prowler_version }}"}'
|
||||
client-payload: '{"version":"release","tag":"${{ needs.setup.outputs.prowler_version }}"}'
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
@@ -44,7 +44,16 @@ jobs:
|
||||
|
||||
sdk-container-build-and-scan:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -53,7 +62,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
@@ -82,22 +91,23 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build SDK container
|
||||
- name: Build SDK container for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: .
|
||||
push: false
|
||||
load: true
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Scan SDK container with Trivy
|
||||
- name: Scan SDK container with Trivy for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -59,13 +59,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Install Poetry
|
||||
run: pipx install poetry==2.1.1
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'poetry'
|
||||
@@ -91,13 +91,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Install Poetry
|
||||
run: pipx install poetry==2.1.1
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'poetry'
|
||||
|
||||
@@ -25,12 +25,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: 'master'
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
run: pip install boto3
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
|
||||
with:
|
||||
aws-region: ${{ env.AWS_REGION }}
|
||||
role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }}
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
- name: Set up Python 3.12
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
cache: 'poetry'
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'poetry'
|
||||
@@ -82,9 +82,110 @@ jobs:
|
||||
./tests/**/aws/**
|
||||
./poetry.lock
|
||||
|
||||
- name: Resolve AWS services under test
|
||||
if: steps.changed-aws.outputs.any_changed == 'true'
|
||||
id: aws-services
|
||||
shell: bash
|
||||
run: |
|
||||
python3 <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
dependents = {
|
||||
"acm": ["elb"],
|
||||
"autoscaling": ["dynamodb"],
|
||||
"awslambda": ["ec2", "inspector2"],
|
||||
"backup": ["dynamodb", "ec2", "rds"],
|
||||
"cloudfront": ["shield"],
|
||||
"cloudtrail": ["awslambda", "cloudwatch"],
|
||||
"cloudwatch": ["bedrock"],
|
||||
"ec2": ["dlm", "dms", "elbv2", "emr", "inspector2", "rds", "redshift", "route53", "shield", "ssm"],
|
||||
"ecr": ["inspector2"],
|
||||
"elb": ["shield"],
|
||||
"elbv2": ["shield"],
|
||||
"globalaccelerator": ["shield"],
|
||||
"iam": ["bedrock", "cloudtrail", "cloudwatch", "codebuild"],
|
||||
"kafka": ["firehose"],
|
||||
"kinesis": ["firehose"],
|
||||
"kms": ["kafka"],
|
||||
"organizations": ["iam", "servicecatalog"],
|
||||
"route53": ["shield"],
|
||||
"s3": ["bedrock", "cloudfront", "cloudtrail", "macie"],
|
||||
"ssm": ["ec2"],
|
||||
"vpc": ["awslambda", "ec2", "efs", "elasticache", "neptune", "networkfirewall", "rds", "redshift", "workspaces"],
|
||||
"waf": ["elbv2"],
|
||||
"wafv2": ["cognito", "elbv2"],
|
||||
}
|
||||
|
||||
changed_raw = """${{ steps.changed-aws.outputs.all_changed_files }}"""
|
||||
# all_changed_files is space-separated, not newline-separated
|
||||
# Strip leading "./" if present for consistent path handling
|
||||
changed_files = [Path(f.lstrip("./")) for f in changed_raw.split() if f]
|
||||
|
||||
services = set()
|
||||
run_all = False
|
||||
|
||||
for path in changed_files:
|
||||
path_str = path.as_posix()
|
||||
parts = path.parts
|
||||
if path_str.startswith("prowler/providers/aws/services/"):
|
||||
if len(parts) > 4 and "." not in parts[4]:
|
||||
services.add(parts[4])
|
||||
else:
|
||||
run_all = True
|
||||
elif path_str.startswith("tests/providers/aws/services/"):
|
||||
if len(parts) > 4 and "." not in parts[4]:
|
||||
services.add(parts[4])
|
||||
else:
|
||||
run_all = True
|
||||
elif path_str.startswith("prowler/providers/aws/") or path_str.startswith("tests/providers/aws/"):
|
||||
run_all = True
|
||||
|
||||
# Expand with direct dependent services (one level only)
|
||||
# We only test services that directly depend on the changed services,
|
||||
# not transitive dependencies (services that depend on dependents)
|
||||
original_services = set(services)
|
||||
for svc in original_services:
|
||||
for dep in dependents.get(svc, []):
|
||||
services.add(dep)
|
||||
|
||||
if run_all or not services:
|
||||
run_all = True
|
||||
services = set()
|
||||
|
||||
service_paths = " ".join(sorted(f"tests/providers/aws/services/{svc}" for svc in services))
|
||||
|
||||
output_lines = [
|
||||
f"run_all={'true' if run_all else 'false'}",
|
||||
f"services={' '.join(sorted(services))}",
|
||||
f"service_paths={service_paths}",
|
||||
]
|
||||
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out:
|
||||
for line in output_lines:
|
||||
gh_out.write(line + "\n")
|
||||
|
||||
print(f"AWS changed files (filtered): {changed_raw or 'none'}")
|
||||
print(f"Run all AWS tests: {run_all}")
|
||||
if services:
|
||||
print(f"AWS service test paths: {service_paths}")
|
||||
else:
|
||||
print("AWS service test paths: none detected")
|
||||
PY
|
||||
|
||||
- name: Run AWS tests
|
||||
if: steps.changed-aws.outputs.any_changed == 'true'
|
||||
run: poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws
|
||||
run: |
|
||||
echo "AWS run_all=${{ steps.aws-services.outputs.run_all }}"
|
||||
echo "AWS service_paths='${{ steps.aws-services.outputs.service_paths }}'"
|
||||
|
||||
if [ "${{ steps.aws-services.outputs.run_all }}" = "true" ]; then
|
||||
poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws
|
||||
elif [ -z "${{ steps.aws-services.outputs.service_paths }}" ]; then
|
||||
echo "No AWS service paths detected; skipping AWS tests."
|
||||
else
|
||||
poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml ${{ steps.aws-services.outputs.service_paths }}
|
||||
fi
|
||||
|
||||
- name: Upload AWS coverage to Codecov
|
||||
if: steps.changed-aws.outputs.any_changed == 'true'
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
name: 'UI: Bump Version'
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ github.event.release.tag_name }}
|
||||
BASE_BRANCH: master
|
||||
|
||||
jobs:
|
||||
detect-release-type:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
is_minor: ${{ steps.detect.outputs.is_minor }}
|
||||
is_patch: ${{ steps.detect.outputs.is_patch }}
|
||||
major_version: ${{ steps.detect.outputs.major_version }}
|
||||
minor_version: ${{ steps.detect.outputs.minor_version }}
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
steps:
|
||||
- name: Detect release type and parse version
|
||||
id: detect
|
||||
run: |
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
MAJOR_VERSION=${BASH_REMATCH[1]}
|
||||
MINOR_VERSION=${BASH_REMATCH[2]}
|
||||
PATCH_VERSION=${BASH_REMATCH[3]}
|
||||
|
||||
echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
if (( MAJOR_VERSION != 5 )); then
|
||||
echo "::error::Releasing another Prowler major version, aborting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( PATCH_VERSION == 0 )); then
|
||||
echo "is_minor=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Minor release detected: $PROWLER_VERSION"
|
||||
else
|
||||
echo "is_minor=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "is_patch=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "✓ Patch release detected: $PROWLER_VERSION"
|
||||
fi
|
||||
else
|
||||
echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-minor-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_minor == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next minor version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0
|
||||
echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current version: $PROWLER_VERSION"
|
||||
echo "Next minor version: $NEXT_MINOR_VERSION"
|
||||
|
||||
- name: Bump UI version in .env for master
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next minor version to master
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: master
|
||||
commit-message: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
branch: ui-version-bump-to-v${{ env.NEXT_MINOR_VERSION }}
|
||||
title: 'chore(ui): Bump version to v${{ env.NEXT_MINOR_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler UI version to v${{ env.NEXT_MINOR_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
- name: Checkout version branch
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
- name: Calculate first patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
|
||||
FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "First patch version: $FIRST_PATCH_VERSION"
|
||||
echo "Version branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump UI version in .env for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for first patch version to version branch
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
branch: ui-version-bump-to-v${{ env.FIRST_PATCH_VERSION }}
|
||||
title: 'chore(ui): Bump version to v${{ env.FIRST_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler UI version to v${{ env.FIRST_PATCH_VERSION }} in version branch after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
|
||||
bump-patch-version:
|
||||
needs: detect-release-type
|
||||
if: needs.detect-release-type.outputs.is_patch == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Calculate next patch version
|
||||
run: |
|
||||
MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }}
|
||||
MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }}
|
||||
PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }}
|
||||
|
||||
NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1))
|
||||
VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION}
|
||||
|
||||
echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}"
|
||||
|
||||
echo "Current version: $PROWLER_VERSION"
|
||||
echo "Next patch version: $NEXT_PATCH_VERSION"
|
||||
echo "Target branch: $VERSION_BRANCH"
|
||||
|
||||
- name: Bump UI version in .env for version branch
|
||||
run: |
|
||||
set -e
|
||||
|
||||
sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env
|
||||
|
||||
echo "Files modified:"
|
||||
git --no-pager diff
|
||||
|
||||
- name: Create PR for next patch version to version branch
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
base: ${{ env.VERSION_BRANCH }}
|
||||
commit-message: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
branch: ui-version-bump-to-v${{ env.NEXT_PATCH_VERSION }}
|
||||
title: 'chore(ui): Bump version to v${{ env.NEXT_PATCH_VERSION }}'
|
||||
labels: no-changelog,skip-sync
|
||||
body: |
|
||||
### Description
|
||||
|
||||
Bump Prowler UI version to v${{ env.NEXT_PATCH_VERSION }} after releasing Prowler v${{ env.PROWLER_VERSION }}.
|
||||
|
||||
### Files Updated
|
||||
- `.env`: `NEXT_PUBLIC_PROWLER_RELEASE_VERSION`
|
||||
|
||||
### License
|
||||
|
||||
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
|
||||
@@ -45,15 +45,15 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/ui-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2
|
||||
uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -10,6 +10,12 @@ on:
|
||||
release:
|
||||
types:
|
||||
- 'published'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag (e.g., 5.14.0)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -21,7 +27,7 @@ concurrency:
|
||||
env:
|
||||
# Tags
|
||||
LATEST_TAG: latest
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }}
|
||||
STABLE_TAG: stable
|
||||
WORKING_DIRECTORY: ./ui
|
||||
|
||||
@@ -44,9 +50,35 @@ jobs:
|
||||
id: set-short-sha
|
||||
run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
|
||||
|
||||
container-build-push:
|
||||
notify-release-started:
|
||||
if: github.repository == 'prowler-cloud/prowler' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
needs: setup
|
||||
runs-on: ${{ matrix.runner }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
message-ts: ${{ steps.slack-notification.outputs.ts }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Notify container push started
|
||||
id: slack-notification
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: UI
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
|
||||
|
||||
container-build-push:
|
||||
needs: [setup, notify-release-started]
|
||||
if: always() && needs.setup.result == 'success' && (needs.notify-release-started.result == 'success' || needs.notify-release-started.result == 'skipped')
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -63,7 +95,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
@@ -75,12 +107,13 @@ jobs:
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build and push UI container for ${{ matrix.arch }}
|
||||
if: github.event_name == 'push'
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
build-args: |
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ needs.setup.outputs.short-sha }}
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ (github.event_name == 'release' || github.event_name == 'workflow_dispatch') && format('v{0}', env.RELEASE_TAG) || needs.setup.outputs.short-sha }}
|
||||
NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }}
|
||||
push: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
@@ -89,53 +122,10 @@ jobs:
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
|
||||
- name: Build and push UI container for ${{ matrix.arch }}
|
||||
if: github.event_name == 'push' || github.event_name == 'release'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
build-args: |
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ github.event_name == 'release' && format('v{0}', env.RELEASE_TAG) || needs.setup.outputs.short_sha }}
|
||||
NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Notify container push started
|
||||
if: github.event_name == 'release'
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: UI
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-started.json"
|
||||
|
||||
- name: Notify container push completed
|
||||
if: github.event_name == 'release' && always()
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
COMPONENT: UI
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.container-push.outcome }}
|
||||
|
||||
# Create and push multi-architecture manifest
|
||||
create-manifest:
|
||||
needs: [setup, container-build-push]
|
||||
if: github.event_name == 'push' || github.event_name == 'release'
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
@@ -158,7 +148,7 @@ jobs:
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64
|
||||
|
||||
- name: Create and push manifests for release event
|
||||
if: github.event_name == 'release'
|
||||
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} \
|
||||
@@ -178,6 +168,40 @@ jobs:
|
||||
regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true
|
||||
echo "Cleanup completed"
|
||||
|
||||
notify-release-completed:
|
||||
if: always() && needs.notify-release-started.result == 'success' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
||||
needs: [setup, notify-release-started, container-build-push, create-manifest]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Determine overall outcome
|
||||
id: outcome
|
||||
run: |
|
||||
if [[ "${{ needs.container-build-push.result }}" == "success" && "${{ needs.create-manifest.result }}" == "success" ]]; then
|
||||
echo "outcome=success" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "outcome=failure" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Notify container push completed
|
||||
uses: ./.github/actions/slack-notification
|
||||
env:
|
||||
SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }}
|
||||
MESSAGE_TS: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
COMPONENT: UI
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
with:
|
||||
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
if: github.event_name == 'push'
|
||||
needs: [setup, container-build-push]
|
||||
@@ -188,7 +212,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Trigger UI deployment
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -20,6 +20,7 @@ env:
|
||||
|
||||
jobs:
|
||||
ui-dockerfile-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
@@ -27,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
@@ -43,7 +44,17 @@ jobs:
|
||||
ignore: DL3018
|
||||
|
||||
ui-container-build-and-scan:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -52,7 +63,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for UI changes
|
||||
id: check-changes
|
||||
@@ -67,7 +78,7 @@ jobs:
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build UI container
|
||||
- name: Build UI container for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
@@ -75,17 +86,18 @@ jobs:
|
||||
target: prod
|
||||
push: false
|
||||
load: true
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
platforms: ${{ matrix.platform }}
|
||||
tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}-${{ matrix.arch }}
|
||||
cache-from: type=gha,scope=${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
||||
build-args: |
|
||||
NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX
|
||||
|
||||
- name: Scan UI container with Trivy
|
||||
if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true'
|
||||
- name: Scan UI container with Trivy for ${{ matrix.arch }}
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/trivy-scan
|
||||
with:
|
||||
image-name: ${{ env.IMAGE_NAME }}
|
||||
image-tag: ${{ github.sha }}
|
||||
image-tag: ${{ github.sha }}-${{ matrix.arch }}
|
||||
fail-on-critical: 'false'
|
||||
severity: 'CRITICAL'
|
||||
|
||||
@@ -10,6 +10,7 @@ on:
|
||||
- 'ui/**'
|
||||
|
||||
jobs:
|
||||
|
||||
e2e-tests:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
@@ -33,12 +34,50 @@ jobs:
|
||||
E2E_M365_SECRET_ID: ${{ secrets.E2E_M365_SECRET_ID }}
|
||||
E2E_M365_TENANT_ID: ${{ secrets.E2E_M365_TENANT_ID }}
|
||||
E2E_M365_CERTIFICATE_CONTENT: ${{ secrets.E2E_M365_CERTIFICATE_CONTENT }}
|
||||
E2E_NEW_PASSWORD: ${{ secrets.E2E_NEW_PASSWORD }}
|
||||
E2E_KUBERNETES_CONTEXT: 'kind-kind'
|
||||
E2E_KUBERNETES_KUBECONFIG_PATH: /home/runner/.kube/config
|
||||
E2E_GCP_BASE64_SERVICE_ACCOUNT_KEY: ${{ secrets.E2E_GCP_BASE64_SERVICE_ACCOUNT_KEY }}
|
||||
E2E_GCP_PROJECT_ID: ${{ secrets.E2E_GCP_PROJECT_ID }}
|
||||
E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }}
|
||||
E2E_GITHUB_BASE64_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_BASE64_APP_PRIVATE_KEY }}
|
||||
E2E_GITHUB_USERNAME: ${{ secrets.E2E_GITHUB_USERNAME }}
|
||||
E2E_GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.E2E_GITHUB_PERSONAL_ACCESS_TOKEN }}
|
||||
E2E_GITHUB_ORGANIZATION: ${{ secrets.E2E_GITHUB_ORGANIZATION }}
|
||||
E2E_GITHUB_ORGANIZATION_ACCESS_TOKEN: ${{ secrets.E2E_GITHUB_ORGANIZATION_ACCESS_TOKEN }}
|
||||
E2E_ORGANIZATION_ID: ${{ secrets.E2E_ORGANIZATION_ID }}
|
||||
E2E_OCI_TENANCY_ID: ${{ secrets.E2E_OCI_TENANCY_ID }}
|
||||
E2E_OCI_USER_ID: ${{ secrets.E2E_OCI_USER_ID }}
|
||||
E2E_OCI_FINGERPRINT: ${{ secrets.E2E_OCI_FINGERPRINT }}
|
||||
E2E_OCI_KEY_CONTENT: ${{ secrets.E2E_OCI_KEY_CONTENT }}
|
||||
E2E_OCI_REGION: ${{ secrets.E2E_OCI_REGION }}
|
||||
E2E_NEW_USER_PASSWORD: ${{ secrets.E2E_NEW_USER_PASSWORD }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
- name: Create k8s Kind Cluster
|
||||
uses: helm/kind-action@v1
|
||||
with:
|
||||
cluster_name: kind
|
||||
- name: Modify kubeconfig
|
||||
run: |
|
||||
# Modify the kubeconfig to use the kind cluster server to https://kind-control-plane:6443
|
||||
# from worker service into docker-compose.yml
|
||||
kubectl config set-cluster kind-kind --server=https://kind-control-plane:6443
|
||||
kubectl config view
|
||||
- name: Add network kind to docker compose
|
||||
run: |
|
||||
# Add the network kind to the docker compose to interconnect to kind cluster
|
||||
yq -i '.networks.kind.external = true' docker-compose.yml
|
||||
# Add network kind to worker service and default network too
|
||||
yq -i '.services.worker.networks = ["kind","default"]' docker-compose.yml
|
||||
- name: Fix API data directory permissions
|
||||
run: docker run --rm -v $(pwd)/_data/api:/data alpine chown -R 1000:1000 /data
|
||||
- name: Add AWS credentials for testing AWS SDK Default Adding Provider
|
||||
run: |
|
||||
echo "Adding AWS credentials for testing AWS SDK Default Adding Provider..."
|
||||
echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env
|
||||
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env
|
||||
- name: Start API services
|
||||
run: |
|
||||
# Override docker-compose image tag to use latest instead of stable
|
||||
@@ -78,29 +117,42 @@ jobs:
|
||||
uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
|
||||
with:
|
||||
node-version: '20.x'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: './ui/package-lock.json'
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
- name: Get pnpm store directory
|
||||
shell: bash
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
|
||||
- name: Setup pnpm cache
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: ${{ env.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pnpm-store-
|
||||
- name: Install UI dependencies
|
||||
working-directory: ./ui
|
||||
run: npm ci
|
||||
run: pnpm install --frozen-lockfile
|
||||
- name: Build UI application
|
||||
working-directory: ./ui
|
||||
run: npm run build
|
||||
run: pnpm run build
|
||||
- name: Cache Playwright browsers
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
id: playwright-cache
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-playwright-${{ hashFiles('ui/package-lock.json') }}
|
||||
key: ${{ runner.os }}-playwright-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-playwright-
|
||||
- name: Install Playwright browsers
|
||||
working-directory: ./ui
|
||||
if: steps.playwright-cache.outputs.cache-hit != 'true'
|
||||
run: npm run test:e2e:install
|
||||
run: pnpm run test:e2e:install
|
||||
- name: Run E2E tests
|
||||
working-directory: ./ui
|
||||
run: npm run test:e2e
|
||||
run: pnpm run test:e2e
|
||||
- name: Upload test reports
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
if: failure()
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Check for UI changes
|
||||
id: check-changes
|
||||
@@ -48,17 +48,36 @@ jobs:
|
||||
uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: 'npm'
|
||||
cache-dependency-path: './ui/package-lock.json'
|
||||
|
||||
- name: Setup pnpm
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- name: Get pnpm store directory
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
shell: bash
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
|
||||
|
||||
- name: Setup pnpm cache
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: ${{ env.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pnpm-store-
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: npm ci
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Run healthcheck
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: npm run healthcheck
|
||||
run: pnpm run healthcheck
|
||||
|
||||
- name: Build application
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
run: npm run build
|
||||
run: pnpm run build
|
||||
|
||||
@@ -150,9 +150,5 @@ _data/
|
||||
# Claude
|
||||
CLAUDE.md
|
||||
|
||||
# MCP Server
|
||||
mcp_server/prowler_mcp_server/prowler_app/server.py
|
||||
mcp_server/prowler_mcp_server/prowler_app/utils/schema.yaml
|
||||
|
||||
# Compliance report
|
||||
*.pdf
|
||||
|
||||
@@ -126,3 +126,12 @@ repos:
|
||||
entry: bash -c 'vulture --exclude "contrib,.venv,api/src/backend/api/tests/,api/src/backend/conftest.py,api/src/backend/tasks/tests/" --min-confidence 100 .'
|
||||
language: system
|
||||
files: '.*\.py'
|
||||
|
||||
- id: ui-checks
|
||||
name: UI - Husky Pre-commit
|
||||
description: "Run UI pre-commit checks (Claude Code validation + healthcheck)"
|
||||
entry: bash -c 'cd ui && .husky/pre-commit'
|
||||
language: system
|
||||
files: '^ui/.*\.(ts|tsx|js|jsx|json|css)$'
|
||||
pass_filenames: false
|
||||
verbose: true
|
||||
|
||||
@@ -12,6 +12,7 @@ ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
wget libicu72 libunwind8 libssl3 libcurl4 ca-certificates apt-transport-https gnupg \
|
||||
build-essential pkg-config libzstd-dev zlib1g-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install PowerShell
|
||||
|
||||
@@ -47,12 +47,12 @@ help: ## Show this help.
|
||||
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
|
||||
|
||||
##@ Build no cache
|
||||
build-no-cache-dev:
|
||||
docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat
|
||||
build-no-cache-dev:
|
||||
docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat mcp-server
|
||||
|
||||
##@ Development Environment
|
||||
run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, and workers
|
||||
docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat
|
||||
run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, MCP, and workers
|
||||
docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat mcp-server
|
||||
|
||||
##@ Development Environment
|
||||
build-and-run-api-dev: build-no-cache-dev run-api-dev
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<b><i>Prowler</b> is the Open Cloud Security platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
|
||||
</p>
|
||||
<p align="center">
|
||||
<b>Learn more at <a href="https://prowler.com">prowler.com</i></b>
|
||||
<b>Secure ANY cloud at AI Speed at <a href="https://prowler.com">prowler.com</i></b>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -23,6 +23,7 @@
|
||||
<a href="https://hub.docker.com/r/toniblyx/prowler"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/toniblyx/prowler"></a>
|
||||
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height=19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://github.com/prowler-cloud/prowler/releases"><img alt="Version" src="https://img.shields.io/github/v/release/prowler-cloud/prowler"></a>
|
||||
@@ -35,28 +36,32 @@
|
||||
</p>
|
||||
<hr>
|
||||
<p align="center">
|
||||
<img align="center" src="/docs/img/prowler-cli-quick.gif" width="100%" height="100%">
|
||||
<img align="center" src="/docs/img/prowler-cloud.gif" width="100%" height="100%">
|
||||
</p>
|
||||
|
||||
# Description
|
||||
|
||||
**Prowler** is an open-source security tool designed to assess and enforce security best practices across AWS, Azure, Google Cloud, and Kubernetes. It supports tasks such as security audits, incident response, continuous monitoring, system hardening, forensic readiness, and remediation processes.
|
||||
**Prowler** is the world’s most widely used _open-source cloud security platform_ that automates security and compliance across **any cloud environment**. With hundreds of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size.
|
||||
|
||||
Prowler includes hundreds of built-in controls to ensure compliance with standards and frameworks, including:
|
||||
|
||||
- **Industry Standards:** CIS, NIST 800, NIST CSF, and CISA
|
||||
- **Regulatory Compliance and Governance:** RBI, FedRAMP, and PCI-DSS
|
||||
- **Prowler ThreatScore:** Weighted risk prioritization scoring that helps you focus on the most critical security findings first
|
||||
- **Industry Standards:** CIS, NIST 800, NIST CSF, CISA, and MITRE ATT&CK
|
||||
- **Regulatory Compliance and Governance:** RBI, FedRAMP, PCI-DSS, and NIS2
|
||||
- **Frameworks for Sensitive Data and Privacy:** GDPR, HIPAA, and FFIEC
|
||||
- **Frameworks for Organizational Governance and Quality Control:** SOC2 and GXP
|
||||
- **AWS-Specific Frameworks:** AWS Foundational Technical Review (FTR) and AWS Well-Architected Framework (Security Pillar)
|
||||
- **National Security Standards:** ENS (Spanish National Security Scheme)
|
||||
- **Frameworks for Organizational Governance and Quality Control:** SOC2, GXP, and ISO 27001
|
||||
- **Cloud-Specific Frameworks:** AWS Foundational Technical Review (FTR), AWS Well-Architected Framework, and BSI C5
|
||||
- **National Security Standards:** ENS (Spanish National Security Scheme) and KISA ISMS-P (Korean)
|
||||
- **Custom Security Frameworks:** Tailored to your needs
|
||||
|
||||
## Prowler App
|
||||
## Prowler App / Prowler Cloud
|
||||
|
||||
Prowler App is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments.
|
||||
Prowler App / [Prowler Cloud](https://cloud.prowler.com/) is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments.
|
||||
|
||||

|
||||

|
||||

|
||||
|
||||
|
||||
>For more details, refer to the [Prowler App Documentation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-app-installation)
|
||||
|
||||
@@ -82,15 +87,16 @@ prowler dashboard
|
||||
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|
||||
|---|---|---|---|---|---|---|
|
||||
| AWS | 576 | 82 | 39 | 10 | Official | UI, API, CLI |
|
||||
| GCP | 79 | 13 | 13 | 3 | Official | UI, API, CLI |
|
||||
| Azure | 162 | 19 | 13 | 4 | Official | UI, API, CLI |
|
||||
| Kubernetes | 83 | 7 | 5 | 7 | Official | UI, API, CLI |
|
||||
| GitHub | 17 | 2 | 1 | 0 | Official | Stable | UI, API, CLI |
|
||||
| AWS | 584 | 85 | 40 | 17 | Official | UI, API, CLI |
|
||||
| GCP | 89 | 17 | 14 | 5 | Official | UI, API, CLI |
|
||||
| Azure | 169 | 22 | 15 | 8 | Official | UI, API, CLI |
|
||||
| Kubernetes | 84 | 7 | 6 | 9 | Official | UI, API, CLI |
|
||||
| GitHub | 20 | 2 | 1 | 2 | Official | UI, API, CLI |
|
||||
| M365 | 70 | 7 | 3 | 2 | Official | UI, API, CLI |
|
||||
| OCI | 51 | 13 | 1 | 10 | Official | UI, API, CLI |
|
||||
| OCI | 52 | 15 | 1 | 12 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 63 | 10 | 1 | 9 | Official | CLI |
|
||||
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | CLI, API |
|
||||
| MongoDB Atlas | 10 | 4 | 0 | 3 | Official | UI, API, CLI |
|
||||
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
|
||||
@@ -153,7 +159,7 @@ You can find more information in the [Troubleshooting](./docs/troubleshooting.md
|
||||
|
||||
* `git` installed.
|
||||
* `poetry` v2 installed: [poetry installation](https://python-poetry.org/docs/#installation).
|
||||
* `npm` installed: [npm installation](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm).
|
||||
* `pnpm` installed: [pnpm installation](https://pnpm.io/installation).
|
||||
* `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
|
||||
**Commands to run the API**
|
||||
@@ -209,9 +215,9 @@ python -m celery -A config.celery beat -l info --scheduler django_celery_beat.sc
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
cd prowler/ui
|
||||
npm install
|
||||
npm run build
|
||||
npm start
|
||||
pnpm install
|
||||
pnpm run build
|
||||
pnpm start
|
||||
```
|
||||
|
||||
> Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
|
||||
@@ -271,11 +277,12 @@ python prowler-cli.py -v
|
||||
# ✏️ High level architecture
|
||||
|
||||
## Prowler App
|
||||
**Prowler App** is composed of three key components:
|
||||
**Prowler App** is composed of four key components:
|
||||
|
||||
- **Prowler UI**: A web-based interface, built with Next.js, providing a user-friendly experience for executing Prowler scans and visualizing results.
|
||||
- **Prowler API**: A backend service, developed with Django REST Framework, responsible for running Prowler scans and storing the generated results.
|
||||
- **Prowler SDK**: A Python SDK designed to extend the functionality of the Prowler CLI for advanced capabilities.
|
||||
- **Prowler MCP Server**: A Model Context Protocol server that provides AI tools for Lighthouse, the AI-powered security assistant. This is a critical dependency for Lighthouse functionality.
|
||||
|
||||

|
||||
|
||||
|
||||
+91
-14
@@ -2,7 +2,79 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.15.0] (Prowler UNRELEASED)
|
||||
## [1.18.0] (Prowler UNRELEASED)
|
||||
|
||||
### Added
|
||||
- Support AlibabaCloud provider [(#9485)](https://github.com/prowler-cloud/prowler/pull/9485)
|
||||
|
||||
---
|
||||
|
||||
## [1.17.1] (Prowler v5.16.1)
|
||||
|
||||
### Changed
|
||||
- Security Hub integration error when no regions [(#9635)](https://github.com/prowler-cloud/prowler/pull/9635)
|
||||
|
||||
### Fixed
|
||||
- Orphan scheduled scans caused by transaction isolation during provider creation [(#9633)](https://github.com/prowler-cloud/prowler/pull/9633)
|
||||
|
||||
---
|
||||
|
||||
## [1.17.0] (Prowler v5.16.0)
|
||||
|
||||
### Added
|
||||
- New endpoint to retrieve and overview of the categories based on finding severities [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
|
||||
- Endpoints `GET /findings` and `GET /findings/latests` can now use the category filter [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
|
||||
- Account id, alias and provider name to PDF reporting table [(#9574)](https://github.com/prowler-cloud/prowler/pull/9574)
|
||||
|
||||
### Changed
|
||||
- Endpoint `GET /overviews/attack-surfaces` no longer returns the related check IDs [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529)
|
||||
- OpenAI provider to only load chat-compatible models with tool calling support [(#9523)](https://github.com/prowler-cloud/prowler/pull/9523)
|
||||
- Increased execution delay for the first scheduled scan tasks to 5 seconds[(#9558)](https://github.com/prowler-cloud/prowler/pull/9558)
|
||||
|
||||
### Fixed
|
||||
- Made `scan_id` a required filter in the compliance overview endpoint [(#9560)](https://github.com/prowler-cloud/prowler/pull/9560)
|
||||
- Reduced unnecessary UPDATE resources operations by only saving when tag mappings change, lowering write load during scans [(#9569)](https://github.com/prowler-cloud/prowler/pull/9569)
|
||||
|
||||
---
|
||||
|
||||
## [1.16.1] (Prowler v5.15.1)
|
||||
|
||||
### Fixed
|
||||
- Race condition in scheduled scan creation by adding countdown to task [(#9516)](https://github.com/prowler-cloud/prowler/pull/9516)
|
||||
|
||||
## [1.16.0] (Prowler v5.15.0)
|
||||
|
||||
### Added
|
||||
- New endpoint to retrieve an overview of the attack surfaces [(#9309)](https://github.com/prowler-cloud/prowler/pull/9309)
|
||||
- New endpoint `GET /api/v1/overviews/findings_severity/timeseries` to retrieve daily aggregated findings by severity level [(#9363)](https://github.com/prowler-cloud/prowler/pull/9363)
|
||||
- Lighthouse AI support for Amazon Bedrock API key [(#9343)](https://github.com/prowler-cloud/prowler/pull/9343)
|
||||
- Exception handler for provider deletions during scans [(#9414)](https://github.com/prowler-cloud/prowler/pull/9414)
|
||||
- Support to use admin credentials through the read replica database [(#9440)](https://github.com/prowler-cloud/prowler/pull/9440)
|
||||
|
||||
### Changed
|
||||
- Error messages from Lighthouse celery tasks [(#9165)](https://github.com/prowler-cloud/prowler/pull/9165)
|
||||
- Restore the compliance overview endpoint's mandatory filters [(#9338)](https://github.com/prowler-cloud/prowler/pull/9338)
|
||||
|
||||
---
|
||||
|
||||
## [1.15.2] (Prowler v5.14.2)
|
||||
|
||||
### Fixed
|
||||
- Unique constraint violation during compliance overviews task [(#9436)](https://github.com/prowler-cloud/prowler/pull/9436)
|
||||
- Division by zero error in ENS PDF report when all requirements are manual [(#9443)](https://github.com/prowler-cloud/prowler/pull/9443)
|
||||
|
||||
---
|
||||
|
||||
## [1.15.1] (Prowler v5.14.1)
|
||||
|
||||
### Fixed
|
||||
- Fix typo in PDF reporting [(#9345)](https://github.com/prowler-cloud/prowler/pull/9345)
|
||||
- Fix IaC provider initialization failure when mutelist processor is configured [(#9331)](https://github.com/prowler-cloud/prowler/pull/9331)
|
||||
- Match logic for ThreatScore when counting findings [(#9348)](https://github.com/prowler-cloud/prowler/pull/9348)
|
||||
|
||||
---
|
||||
|
||||
## [1.15.0] (Prowler v5.14.0)
|
||||
|
||||
### Added
|
||||
- IaC (Infrastructure as Code) provider support for remote repositories [(#8751)](https://github.com/prowler-cloud/prowler/pull/8751)
|
||||
@@ -14,28 +86,33 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Support muting findings based on simple rules with custom reason [(#9051)](https://github.com/prowler-cloud/prowler/pull/9051)
|
||||
- Support C5 compliance framework for the GCP provider [(#9097)](https://github.com/prowler-cloud/prowler/pull/9097)
|
||||
- Support for Amazon Bedrock and OpenAI compatible providers in Lighthouse AI [(#8957)](https://github.com/prowler-cloud/prowler/pull/8957)
|
||||
- Support PDF reporting for ENS compliance framework [(#9158)](https://github.com/prowler-cloud/prowler/pull/9158)
|
||||
- Support PDF reporting for NIS2 compliance framework [(#9170)](https://github.com/prowler-cloud/prowler/pull/9170)
|
||||
- Tenant-wide ThreatScore overview aggregation and snapshot persistence with backfill support [(#9148)](https://github.com/prowler-cloud/prowler/pull/9148)
|
||||
- Added `metadata`, `details`, and `partition` attributes to `/resources` endpoint & `details`, and `partition` to `/findings` endpoint [(#9098)](https://github.com/prowler-cloud/prowler/pull/9098)
|
||||
- Support for MongoDB Atlas provider [(#9167)](https://github.com/prowler-cloud/prowler/pull/9167)
|
||||
- Support Prowler ThreatScore for the K8S provider [(#9235)](https://github.com/prowler-cloud/prowler/pull/9235)
|
||||
- Enhanced compliance overview endpoint with provider filtering and latest scan aggregation [(#9244)](https://github.com/prowler-cloud/prowler/pull/9244)
|
||||
- New endpoint `GET /api/v1/overview/regions` to retrieve aggregated findings data by region [(#9273)](https://github.com/prowler-cloud/prowler/pull/9273)
|
||||
|
||||
### Changed
|
||||
- Optimized database write queries for scan related tasks [(#9190)](https://github.com/prowler-cloud/prowler/pull/9190)
|
||||
- Date filters are now optional for `GET /api/v1/overviews/services` endpoint; returns latest scan data by default [(#9248)](https://github.com/prowler-cloud/prowler/pull/9248)
|
||||
|
||||
### Fixed
|
||||
- Scans no longer fail when findings have UIDs exceeding 300 characters; such findings are now skipped with detailed logging [(#9246)](https://github.com/prowler-cloud/prowler/pull/9246)
|
||||
- Updated unique constraint for `Provider` model to exclude soft-deleted entries, resolving duplicate errors when re-deleting providers [(#9054)](https://github.com/prowler-cloud/prowler/pull/9054)
|
||||
- Removed compliance generation for providers without compliance frameworks [(#9208)](https://github.com/prowler-cloud/prowler/pull/9208)
|
||||
- Refresh output report timestamps for each scan [(#9272)](https://github.com/prowler-cloud/prowler/pull/9272)
|
||||
- Severity overview endpoint now ignores muted findings as expected [(#9283)](https://github.com/prowler-cloud/prowler/pull/9283)
|
||||
- Fixed discrepancy between ThreatScore PDF report values and database calculations [(#9296)](https://github.com/prowler-cloud/prowler/pull/9296)
|
||||
|
||||
### Security
|
||||
- Django updated to the latest 5.1 security release, 5.1.14, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/113) and [denial-of-service vulnerability](https://github.com/prowler-cloud/prowler/security/dependabot/114) [(#9176)](https://github.com/prowler-cloud/prowler/pull/9176)
|
||||
|
||||
---
|
||||
|
||||
## [1.14.2] (Prowler 5.13.2)
|
||||
|
||||
### Fixed
|
||||
- Update unique constraint for `Provider` model to exclude soft-deleted entries, resolving duplicate errors when re-deleting providers [(#9054)](https://github.com/prowler-cloud/prowler/pull/9054)
|
||||
- Remove compliance generation for providers without compliance frameworks [(#9208)](https://github.com/prowler-cloud/prowler/pull/9208)
|
||||
|
||||
## [1.14.1] (Prowler 5.13.1)
|
||||
## [1.14.1] (Prowler v5.13.1)
|
||||
|
||||
### Fixed
|
||||
- `/api/v1/overviews/providers` collapses data by provider type so the UI receives a single aggregated record per cloud family even when multiple accounts exist [(#9053)](https://github.com/prowler-cloud/prowler/pull/9053)
|
||||
@@ -44,7 +121,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
---
|
||||
|
||||
## [1.14.0] (Prowler 5.13.0)
|
||||
## [1.14.0] (Prowler v5.13.0)
|
||||
|
||||
### Added
|
||||
- Default JWT keys are generated and stored if they are missing from configuration [(#8655)](https://github.com/prowler-cloud/prowler/pull/8655)
|
||||
@@ -68,14 +145,14 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
---
|
||||
|
||||
## [1.13.2] (Prowler 5.12.3)
|
||||
## [1.13.2] (Prowler v5.12.3)
|
||||
|
||||
### Fixed
|
||||
- 500 error when deleting user [(#8731)](https://github.com/prowler-cloud/prowler/pull/8731)
|
||||
|
||||
---
|
||||
|
||||
## [1.13.1] (Prowler 5.12.2)
|
||||
## [1.13.1] (Prowler v5.12.2)
|
||||
|
||||
### Changed
|
||||
- Renamed compliance overview task queue to `compliance` [(#8755)](https://github.com/prowler-cloud/prowler/pull/8755)
|
||||
@@ -85,7 +162,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
---
|
||||
|
||||
## [1.13.0] (Prowler 5.12.0)
|
||||
## [1.13.0] (Prowler v5.12.0)
|
||||
|
||||
### Added
|
||||
- Integration with JIRA, enabling sending findings to a JIRA project [(#8622)](https://github.com/prowler-cloud/prowler/pull/8622), [(#8637)](https://github.com/prowler-cloud/prowler/pull/8637)
|
||||
@@ -94,7 +171,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
---
|
||||
|
||||
## [1.12.0] (Prowler 5.11.0)
|
||||
## [1.12.0] (Prowler v5.11.0)
|
||||
|
||||
### Added
|
||||
- Lighthouse support for OpenAI GPT-5 [(#8527)](https://github.com/prowler-cloud/prowler/pull/8527)
|
||||
@@ -106,7 +183,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
---
|
||||
|
||||
## [1.11.0] (Prowler 5.10.0)
|
||||
## [1.11.0] (Prowler v5.10.0)
|
||||
|
||||
### Added
|
||||
- Github provider support [(#8271)](https://github.com/prowler-cloud/prowler/pull/8271)
|
||||
|
||||
Generated
+3176
-1842
File diff suppressed because it is too large
Load Diff
+3
-2
@@ -35,7 +35,8 @@ dependencies = [
|
||||
"markdown (>=3.9,<4.0)",
|
||||
"drf-simple-apikey (==2.2.1)",
|
||||
"matplotlib (>=3.10.6,<4.0.0)",
|
||||
"reportlab (>=4.4.4,<5.0.0)"
|
||||
"reportlab (>=4.4.4,<5.0.0)",
|
||||
"gevent (>=25.9.1,<26.0.0)"
|
||||
]
|
||||
description = "Prowler's API (Django/DRF)"
|
||||
license = "Apache-2.0"
|
||||
@@ -43,7 +44,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.15.0"
|
||||
version = "1.18.0"
|
||||
|
||||
[project.scripts]
|
||||
celery = "src.backend.config.settings.celery"
|
||||
|
||||
@@ -26,6 +26,7 @@ class MainRouter:
|
||||
default_db = "default"
|
||||
admin_db = "admin"
|
||||
replica_db = "replica"
|
||||
admin_replica_db = "admin_replica"
|
||||
|
||||
def db_for_read(self, model, **hints): # noqa: F841
|
||||
model_table_name = model._meta.db_table
|
||||
@@ -49,7 +50,12 @@ class MainRouter:
|
||||
|
||||
def allow_relation(self, obj1, obj2, **hints): # noqa: F841
|
||||
# Allow relations when both objects originate from allowed connectors
|
||||
allowed_dbs = {self.default_db, self.admin_db, self.replica_db}
|
||||
allowed_dbs = {
|
||||
self.default_db,
|
||||
self.admin_db,
|
||||
self.replica_db,
|
||||
self.admin_replica_db,
|
||||
}
|
||||
if {obj1._state.db, obj2._state.db} <= allowed_dbs:
|
||||
return True
|
||||
return None
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
import uuid
|
||||
from functools import wraps
|
||||
|
||||
from django.db import connection, transaction
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.db import IntegrityError, connection, transaction
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
|
||||
from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY, rls_transaction
|
||||
from api.exceptions import ProviderDeletedException
|
||||
from api.models import Provider, Scan
|
||||
|
||||
|
||||
def set_tenant(func=None, *, keep_tenant=False):
|
||||
@@ -66,3 +70,49 @@ def set_tenant(func=None, *, keep_tenant=False):
|
||||
return decorator
|
||||
else:
|
||||
return decorator(func)
|
||||
|
||||
|
||||
def handle_provider_deletion(func):
|
||||
"""
|
||||
Decorator that raises ProviderDeletedException if provider was deleted during execution.
|
||||
|
||||
Catches ObjectDoesNotExist and IntegrityError, checks if provider still exists,
|
||||
and raises ProviderDeletedException if not. Otherwise, re-raises original exception.
|
||||
|
||||
Requires tenant_id and provider_id in kwargs.
|
||||
|
||||
Example:
|
||||
@shared_task
|
||||
@handle_provider_deletion
|
||||
def scan_task(scan_id, tenant_id, provider_id):
|
||||
...
|
||||
"""
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except (ObjectDoesNotExist, IntegrityError):
|
||||
tenant_id = kwargs.get("tenant_id")
|
||||
provider_id = kwargs.get("provider_id")
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
if provider_id is None:
|
||||
scan_id = kwargs.get("scan_id")
|
||||
if scan_id is None:
|
||||
raise AssertionError(
|
||||
"This task does not have provider or scan in the kwargs"
|
||||
)
|
||||
scan = Scan.objects.filter(pk=scan_id).first()
|
||||
if scan is None:
|
||||
raise ProviderDeletedException(
|
||||
f"Provider for scan '{scan_id}' was deleted during the scan"
|
||||
) from None
|
||||
provider_id = str(scan.provider_id)
|
||||
if not Provider.objects.filter(pk=provider_id).exists():
|
||||
raise ProviderDeletedException(
|
||||
f"Provider '{provider_id}' was deleted during the scan"
|
||||
) from None
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
|
||||
@@ -66,6 +66,10 @@ class ProviderConnectionError(Exception):
|
||||
"""Base exception for provider connection errors."""
|
||||
|
||||
|
||||
class ProviderDeletedException(Exception):
|
||||
"""Raised when a provider has been deleted during scan/task execution."""
|
||||
|
||||
|
||||
def custom_exception_handler(exc, context):
|
||||
if isinstance(exc, django_validation_error):
|
||||
if hasattr(exc, "error_dict"):
|
||||
|
||||
+115
-25
@@ -23,7 +23,9 @@ from api.db_utils import (
|
||||
StatusEnumField,
|
||||
)
|
||||
from api.models import (
|
||||
AttackSurfaceOverview,
|
||||
ComplianceRequirementOverview,
|
||||
DailySeveritySummary,
|
||||
Finding,
|
||||
Integration,
|
||||
Invitation,
|
||||
@@ -41,6 +43,7 @@ from api.models import (
|
||||
ResourceTag,
|
||||
Role,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanSummary,
|
||||
SeverityChoices,
|
||||
StateChoices,
|
||||
@@ -155,6 +158,9 @@ class CommonFindingFilters(FilterSet):
|
||||
field_name="resources__type", lookup_expr="icontains"
|
||||
)
|
||||
|
||||
category = CharFilter(method="filter_category")
|
||||
category__in = CharInFilter(field_name="categories", lookup_expr="overlap")
|
||||
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# resource_tag_key = CharFilter(field_name="resources__tags__key")
|
||||
# resource_tag_key__in = CharInFilter(
|
||||
@@ -186,6 +192,9 @@ class CommonFindingFilters(FilterSet):
|
||||
def filter_resource_type(self, queryset, name, value):
|
||||
return queryset.filter(resource_types__contains=[value])
|
||||
|
||||
def filter_category(self, queryset, name, value):
|
||||
return queryset.filter(categories__contains=[value])
|
||||
|
||||
def filter_resource_tag(self, queryset, name, value):
|
||||
overall_query = Q()
|
||||
for key_value_pair in value:
|
||||
@@ -760,7 +769,7 @@ class RoleFilter(FilterSet):
|
||||
|
||||
class ComplianceOverviewFilter(FilterSet):
|
||||
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
|
||||
scan_id = UUIDFilter(field_name="scan_id")
|
||||
scan_id = UUIDFilter(field_name="scan_id", required=True)
|
||||
region = CharFilter(field_name="region")
|
||||
|
||||
class Meta:
|
||||
@@ -794,6 +803,68 @@ class ScanSummaryFilter(FilterSet):
|
||||
}
|
||||
|
||||
|
||||
class DailySeveritySummaryFilter(FilterSet):
|
||||
"""Filter for findings_severity/timeseries endpoint."""
|
||||
|
||||
MAX_DATE_RANGE_DAYS = 365
|
||||
|
||||
provider_id = UUIDFilter(field_name="provider_id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="provider_id", lookup_expr="in")
|
||||
provider_type = ChoiceFilter(
|
||||
field_name="provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_type__in = ChoiceInFilter(
|
||||
field_name="provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
date_from = DateFilter(method="filter_noop")
|
||||
date_to = DateFilter(method="filter_noop")
|
||||
|
||||
class Meta:
|
||||
model = DailySeveritySummary
|
||||
fields = ["provider_id"]
|
||||
|
||||
def filter_noop(self, queryset, name, value):
|
||||
return queryset
|
||||
|
||||
def filter_queryset(self, queryset):
|
||||
if not self.data.get("date_from"):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "This query parameter is required.",
|
||||
"status": "400",
|
||||
"source": {"pointer": "filter[date_from]"},
|
||||
"code": "required",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
today = date.today()
|
||||
date_from = self.form.cleaned_data.get("date_from")
|
||||
date_to = min(self.form.cleaned_data.get("date_to") or today, today)
|
||||
|
||||
if (date_to - date_from).days > self.MAX_DATE_RANGE_DAYS:
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": f"Date range cannot exceed {self.MAX_DATE_RANGE_DAYS} days.",
|
||||
"status": "400",
|
||||
"source": {"pointer": "filter[date_from]"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# View access
|
||||
self.request._date_from = date_from
|
||||
self.request._date_to = date_to
|
||||
|
||||
# Apply date filter (only lte for fill-forward logic)
|
||||
queryset = queryset.filter(date__lte=date_to)
|
||||
|
||||
return super().filter_queryset(queryset)
|
||||
|
||||
|
||||
class ScanSummarySeverityFilter(ScanSummaryFilter):
|
||||
"""Filter for findings_severity ScanSummary endpoint - includes status filters"""
|
||||
|
||||
@@ -812,7 +883,8 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
|
||||
elif value == OverviewStatusChoices.PASS:
|
||||
return queryset.annotate(status_count=F("_pass"))
|
||||
else:
|
||||
return queryset.annotate(status_count=F("total"))
|
||||
# Exclude muted findings by default
|
||||
return queryset.annotate(status_count=F("_pass") + F("fail"))
|
||||
|
||||
def filter_status_in(self, queryset, name, value):
|
||||
# Validate the status values
|
||||
@@ -821,7 +893,7 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
|
||||
if status_val not in valid_statuses:
|
||||
raise ValidationError(f"Invalid status value: {status_val}")
|
||||
|
||||
# If all statuses or no valid statuses, use total
|
||||
# If all statuses or no valid statuses, exclude muted findings (pass + fail)
|
||||
if (
|
||||
set(value)
|
||||
>= {
|
||||
@@ -830,7 +902,7 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
|
||||
}
|
||||
or not value
|
||||
):
|
||||
return queryset.annotate(status_count=F("total"))
|
||||
return queryset.annotate(status_count=F("_pass") + F("fail"))
|
||||
|
||||
# Build the sum expression based on status values
|
||||
sum_expression = None
|
||||
@@ -848,7 +920,7 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
|
||||
sum_expression = sum_expression + field_expr
|
||||
|
||||
if sum_expression is None:
|
||||
return queryset.annotate(status_count=F("total"))
|
||||
return queryset.annotate(status_count=F("_pass") + F("fail"))
|
||||
|
||||
return queryset.annotate(status_count=sum_expression)
|
||||
|
||||
@@ -860,26 +932,6 @@ class ScanSummarySeverityFilter(ScanSummaryFilter):
|
||||
}
|
||||
|
||||
|
||||
class ServiceOverviewFilter(ScanSummaryFilter):
|
||||
def is_valid(self):
|
||||
# Check if at least one of the inserted_at filters is present
|
||||
inserted_at_filters = [
|
||||
self.data.get("inserted_at"),
|
||||
self.data.get("inserted_at__gte"),
|
||||
self.data.get("inserted_at__lte"),
|
||||
]
|
||||
if not any(inserted_at_filters):
|
||||
raise ValidationError(
|
||||
{
|
||||
"inserted_at": [
|
||||
"At least one of filter[inserted_at], filter[inserted_at__gte], or "
|
||||
"filter[inserted_at__lte] is required."
|
||||
]
|
||||
}
|
||||
)
|
||||
return super().is_valid()
|
||||
|
||||
|
||||
class IntegrationFilter(FilterSet):
|
||||
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
|
||||
integration_type = ChoiceFilter(choices=Integration.IntegrationChoices.choices)
|
||||
@@ -1032,3 +1084,41 @@ class ThreatScoreSnapshotFilter(FilterSet):
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
"overall_score": ["exact", "gte", "lte"],
|
||||
}
|
||||
|
||||
|
||||
class AttackSurfaceOverviewFilter(FilterSet):
|
||||
"""Filter for attack surface overview aggregations by provider."""
|
||||
|
||||
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
|
||||
provider_type = ChoiceFilter(
|
||||
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_type__in = ChoiceInFilter(
|
||||
field_name="scan__provider__provider",
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = AttackSurfaceOverview
|
||||
fields = {}
|
||||
|
||||
|
||||
class CategoryOverviewFilter(FilterSet):
|
||||
provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact")
|
||||
provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in")
|
||||
provider_type = ChoiceFilter(
|
||||
field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices
|
||||
)
|
||||
provider_type__in = ChoiceInFilter(
|
||||
field_name="scan__provider__provider",
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
category = CharFilter(field_name="category", lookup_expr="exact")
|
||||
category__in = CharInFilter(field_name="category", lookup_expr="in")
|
||||
|
||||
class Meta:
|
||||
model = ScanCategorySummary
|
||||
fields = {}
|
||||
|
||||
@@ -22,7 +22,7 @@ class Migration(migrations.Migration):
|
||||
("kubernetes", "Kubernetes"),
|
||||
("m365", "M365"),
|
||||
("github", "GitHub"),
|
||||
("oci", "Oracle Cloud Infrastructure"),
|
||||
("oraclecloud", "Oracle Cloud Infrastructure"),
|
||||
("iac", "IaC"),
|
||||
],
|
||||
default="aws",
|
||||
|
||||
@@ -29,4 +29,8 @@ class Migration(migrations.Migration):
|
||||
default="aws",
|
||||
),
|
||||
),
|
||||
migrations.RunSQL(
|
||||
"ALTER TYPE provider ADD VALUE IF NOT EXISTS 'mongodbatlas';",
|
||||
reverse_sql=migrations.RunSQL.noop,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -22,4 +22,8 @@ class Migration(migrations.Migration):
|
||||
model_name="compliancerequirementoverview",
|
||||
name="cro_scan_comp_req_idx",
|
||||
),
|
||||
RemoveIndexConcurrently(
|
||||
model_name="compliancerequirementoverview",
|
||||
name="cro_scan_comp_req_reg_idx",
|
||||
),
|
||||
]
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# Generated by Django 5.1.14 on 2025-11-19 13:03
|
||||
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0059_compliance_overview_summary"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="AttackSurfaceOverview",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("inserted_at", models.DateTimeField(auto_now_add=True)),
|
||||
(
|
||||
"attack_surface_type",
|
||||
models.CharField(
|
||||
choices=[
|
||||
("internet-exposed", "Internet Exposed"),
|
||||
("secrets", "Exposed Secrets"),
|
||||
("privilege-escalation", "Privilege Escalation"),
|
||||
("ec2-imdsv1", "EC2 IMDSv1 Enabled"),
|
||||
],
|
||||
max_length=50,
|
||||
),
|
||||
),
|
||||
("total_findings", models.IntegerField(default=0)),
|
||||
("failed_findings", models.IntegerField(default=0)),
|
||||
("muted_failed_findings", models.IntegerField(default=0)),
|
||||
],
|
||||
options={
|
||||
"db_table": "attack_surface_overviews",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="attacksurfaceoverview",
|
||||
name="scan",
|
||||
field=models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="attack_surface_overviews",
|
||||
related_query_name="attack_surface_overview",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="attacksurfaceoverview",
|
||||
name="tenant",
|
||||
field=models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="attacksurfaceoverview",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "scan_id"], name="attack_surf_tenant_scan_idx"
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="attacksurfaceoverview",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "attack_surface_type"),
|
||||
name="unique_attack_surface_per_scan",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="attacksurfaceoverview",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_attacksurfaceoverview",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,96 @@
|
||||
# Generated by Django 5.1.14 on 2025-12-03 13:38
|
||||
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0060_attack_surface_overview"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="DailySeveritySummary",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("date", models.DateField()),
|
||||
("critical", models.IntegerField(default=0)),
|
||||
("high", models.IntegerField(default=0)),
|
||||
("medium", models.IntegerField(default=0)),
|
||||
("low", models.IntegerField(default=0)),
|
||||
("informational", models.IntegerField(default=0)),
|
||||
("muted", models.IntegerField(default=0)),
|
||||
(
|
||||
"provider",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="daily_severity_summaries",
|
||||
related_query_name="daily_severity_summary",
|
||||
to="api.provider",
|
||||
),
|
||||
),
|
||||
(
|
||||
"scan",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="daily_severity_summaries",
|
||||
related_query_name="daily_severity_summary",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to="api.tenant",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "daily_severity_summaries",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="dailyseveritysummary",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "id"],
|
||||
name="dss_tenant_id_idx",
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="dailyseveritysummary",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "provider_id"],
|
||||
name="dss_tenant_provider_idx",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="dailyseveritysummary",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "provider", "date"),
|
||||
name="unique_daily_severity_summary",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="dailyseveritysummary",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_dailyseveritysummary",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Generated by Django 5.1.14 on 2025-12-10
|
||||
|
||||
from django.db import migrations
|
||||
from tasks.tasks import backfill_daily_severity_summaries_task
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.rls import Tenant
|
||||
|
||||
|
||||
def trigger_backfill_task(apps, schema_editor):
|
||||
"""
|
||||
Trigger the backfill task for all tenants.
|
||||
|
||||
This dispatches backfill_daily_severity_summaries_task for each tenant
|
||||
in the system to populate DailySeveritySummary records from historical scans.
|
||||
"""
|
||||
tenant_ids = Tenant.objects.using(MainRouter.admin_db).values_list("id", flat=True)
|
||||
|
||||
for tenant_id in tenant_ids:
|
||||
backfill_daily_severity_summaries_task.delay(tenant_id=str(tenant_id), days=90)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0061_daily_severity_summary"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(trigger_backfill_task, migrations.RunPython.noop),
|
||||
]
|
||||
@@ -0,0 +1,111 @@
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.db_utils
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0062_backfill_daily_severity_summaries"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="ScanCategorySummary",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to="api.tenant",
|
||||
),
|
||||
),
|
||||
(
|
||||
"inserted_at",
|
||||
models.DateTimeField(auto_now_add=True),
|
||||
),
|
||||
(
|
||||
"scan",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="category_summaries",
|
||||
related_query_name="category_summary",
|
||||
to="api.scan",
|
||||
),
|
||||
),
|
||||
(
|
||||
"category",
|
||||
models.CharField(max_length=100),
|
||||
),
|
||||
(
|
||||
"severity",
|
||||
api.db_utils.SeverityEnumField(
|
||||
choices=[
|
||||
("critical", "Critical"),
|
||||
("high", "High"),
|
||||
("medium", "Medium"),
|
||||
("low", "Low"),
|
||||
("informational", "Informational"),
|
||||
],
|
||||
),
|
||||
),
|
||||
(
|
||||
"total_findings",
|
||||
models.IntegerField(
|
||||
default=0, help_text="Non-muted findings (PASS + FAIL)"
|
||||
),
|
||||
),
|
||||
(
|
||||
"failed_findings",
|
||||
models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL findings (subset of total_findings)",
|
||||
),
|
||||
),
|
||||
(
|
||||
"new_failed_findings",
|
||||
models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "scan_category_summaries",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="scancategorysummary",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "scan"], name="scs_tenant_scan_idx"
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scancategorysummary",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "category", "severity"),
|
||||
name="unique_category_severity_per_scan",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="scancategorysummary",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_scancategorysummary",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
import django.contrib.postgres.fields
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0063_scan_category_summary"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="finding",
|
||||
name="categories",
|
||||
field=django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=100),
|
||||
blank=True,
|
||||
null=True,
|
||||
size=None,
|
||||
help_text="Categories from check metadata for efficient filtering",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,37 @@
|
||||
# Generated by Django migration for Alibaba Cloud provider support
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
import api.db_utils
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0064_finding_categories"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="provider",
|
||||
name="provider",
|
||||
field=api.db_utils.ProviderEnumField(
|
||||
choices=[
|
||||
("aws", "AWS"),
|
||||
("azure", "Azure"),
|
||||
("gcp", "GCP"),
|
||||
("kubernetes", "Kubernetes"),
|
||||
("m365", "M365"),
|
||||
("github", "GitHub"),
|
||||
("mongodbatlas", "MongoDB Atlas"),
|
||||
("iac", "IaC"),
|
||||
("oraclecloud", "Oracle Cloud Infrastructure"),
|
||||
("alibabacloud", "Alibaba Cloud"),
|
||||
],
|
||||
default="aws",
|
||||
),
|
||||
),
|
||||
migrations.RunSQL(
|
||||
"ALTER TYPE provider ADD VALUE IF NOT EXISTS 'alibabacloud';",
|
||||
reverse_sql=migrations.RunSQL.noop,
|
||||
),
|
||||
]
|
||||
@@ -287,6 +287,7 @@ class Provider(RowLevelSecurityProtectedModel):
|
||||
MONGODBATLAS = "mongodbatlas", _("MongoDB Atlas")
|
||||
IAC = "iac", _("IaC")
|
||||
ORACLECLOUD = "oraclecloud", _("Oracle Cloud Infrastructure")
|
||||
ALIBABACLOUD = "alibabacloud", _("Alibaba Cloud")
|
||||
|
||||
@staticmethod
|
||||
def validate_aws_uid(value):
|
||||
@@ -391,6 +392,15 @@ class Provider(RowLevelSecurityProtectedModel):
|
||||
pointer="/data/attributes/uid",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def validate_alibabacloud_uid(value):
|
||||
if not re.match(r"^\d{16}$", value):
|
||||
raise ModelValidationError(
|
||||
detail="Alibaba Cloud account ID must be exactly 16 digits.",
|
||||
code="alibabacloud-uid",
|
||||
pointer="/data/attributes/uid",
|
||||
)
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
updated_at = models.DateTimeField(auto_now=True, editable=False)
|
||||
@@ -716,14 +726,19 @@ class Resource(RowLevelSecurityProtectedModel):
|
||||
self.clear_tags()
|
||||
return
|
||||
|
||||
# Add new relationships with the tenant_id field
|
||||
# Add new relationships with the tenant_id field; avoid touching the
|
||||
# Resource row unless a mapping is actually created to prevent noisy
|
||||
# updates during scans.
|
||||
mapping_created = False
|
||||
for tag in tags:
|
||||
ResourceTagMapping.objects.update_or_create(
|
||||
_, created = ResourceTagMapping.objects.update_or_create(
|
||||
tag=tag, resource=self, tenant_id=self.tenant_id
|
||||
)
|
||||
mapping_created = mapping_created or created
|
||||
|
||||
# Save the instance
|
||||
self.save()
|
||||
if mapping_created:
|
||||
# Only bump updated_at when the tag set truly changed
|
||||
self.save(update_fields=["updated_at"])
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "resources"
|
||||
@@ -868,6 +883,14 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
|
||||
null=True,
|
||||
)
|
||||
|
||||
# Check metadata denormalization
|
||||
categories = ArrayField(
|
||||
models.CharField(max_length=100),
|
||||
blank=True,
|
||||
null=True,
|
||||
help_text="Categories from check metadata for efficient filtering",
|
||||
)
|
||||
|
||||
# Relationships
|
||||
scan = models.ForeignKey(to=Scan, related_name="findings", on_delete=models.CASCADE)
|
||||
|
||||
@@ -1500,6 +1523,65 @@ class ScanSummary(RowLevelSecurityProtectedModel):
|
||||
resource_name = "scan-summaries"
|
||||
|
||||
|
||||
class DailySeveritySummary(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Pre-aggregated daily severity counts per provider.
|
||||
Used by findings_severity/timeseries endpoint for efficient queries.
|
||||
"""
|
||||
|
||||
objects = ActiveProviderManager()
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
date = models.DateField()
|
||||
|
||||
provider = models.ForeignKey(
|
||||
Provider,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="daily_severity_summaries",
|
||||
related_query_name="daily_severity_summary",
|
||||
)
|
||||
scan = models.ForeignKey(
|
||||
Scan,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="daily_severity_summaries",
|
||||
related_query_name="daily_severity_summary",
|
||||
)
|
||||
|
||||
# Aggregated fail counts by severity
|
||||
critical = models.IntegerField(default=0)
|
||||
high = models.IntegerField(default=0)
|
||||
medium = models.IntegerField(default=0)
|
||||
low = models.IntegerField(default=0)
|
||||
informational = models.IntegerField(default=0)
|
||||
muted = models.IntegerField(default=0)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "daily_severity_summaries"
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "provider", "date"),
|
||||
name="unique_daily_severity_summary",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
indexes = [
|
||||
models.Index(
|
||||
fields=["tenant_id", "id"],
|
||||
name="dss_tenant_id_idx",
|
||||
),
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_id"],
|
||||
name="dss_tenant_provider_idx",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class Integration(RowLevelSecurityProtectedModel):
|
||||
class IntegrationChoices(models.TextChoices):
|
||||
AMAZON_S3 = "amazon_s3", _("Amazon S3")
|
||||
@@ -1892,6 +1974,64 @@ class ResourceScanSummary(RowLevelSecurityProtectedModel):
|
||||
]
|
||||
|
||||
|
||||
class ScanCategorySummary(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Pre-aggregated category metrics per scan by severity.
|
||||
|
||||
Stores one row per (category, severity) combination per scan for efficient
|
||||
overview queries. Categories come from check_metadata.categories.
|
||||
|
||||
Count relationships (each is a subset of the previous):
|
||||
- total_findings >= failed_findings >= new_failed_findings
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
|
||||
scan = models.ForeignKey(
|
||||
Scan,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="category_summaries",
|
||||
related_query_name="category_summary",
|
||||
)
|
||||
|
||||
category = models.CharField(max_length=100)
|
||||
severity = SeverityEnumField(choices=SeverityChoices)
|
||||
|
||||
total_findings = models.IntegerField(
|
||||
default=0, help_text="Non-muted findings (PASS + FAIL)"
|
||||
)
|
||||
failed_findings = models.IntegerField(
|
||||
default=0, help_text="Non-muted FAIL findings (subset of total_findings)"
|
||||
)
|
||||
new_failed_findings = models.IntegerField(
|
||||
default=0,
|
||||
help_text="Non-muted FAIL with delta='new' (subset of failed_findings)",
|
||||
)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "scan_category_summaries"
|
||||
|
||||
indexes = [
|
||||
models.Index(fields=["tenant_id", "scan"], name="scs_tenant_scan_idx"),
|
||||
]
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "category", "severity"),
|
||||
name="unique_category_severity_per_scan",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "scan-category-summaries"
|
||||
|
||||
|
||||
class LighthouseConfiguration(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Stores configuration and API keys for LLM services.
|
||||
@@ -2282,9 +2422,6 @@ class ThreatScoreSnapshot(RowLevelSecurityProtectedModel):
|
||||
Snapshots are created automatically after each ThreatScore report generation.
|
||||
"""
|
||||
|
||||
objects = models.Manager()
|
||||
all_objects = models.Manager()
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
|
||||
@@ -2408,3 +2545,63 @@ class ThreatScoreSnapshot(RowLevelSecurityProtectedModel):
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "threatscore-snapshots"
|
||||
|
||||
|
||||
class AttackSurfaceOverview(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Pre-aggregated attack surface metrics per scan.
|
||||
|
||||
Stores counts for each attack surface type (internet-exposed, secrets,
|
||||
privilege-escalation, ec2-imdsv1) to enable fast overview queries.
|
||||
"""
|
||||
|
||||
class AttackSurfaceTypeChoices(models.TextChoices):
|
||||
INTERNET_EXPOSED = "internet-exposed", _("Internet Exposed")
|
||||
SECRETS = "secrets", _("Exposed Secrets")
|
||||
PRIVILEGE_ESCALATION = "privilege-escalation", _("Privilege Escalation")
|
||||
EC2_IMDSV1 = "ec2-imdsv1", _("EC2 IMDSv1 Enabled")
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
|
||||
scan = models.ForeignKey(
|
||||
Scan,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="attack_surface_overviews",
|
||||
related_query_name="attack_surface_overview",
|
||||
)
|
||||
|
||||
attack_surface_type = models.CharField(
|
||||
max_length=50,
|
||||
choices=AttackSurfaceTypeChoices.choices,
|
||||
)
|
||||
|
||||
# Finding counts
|
||||
total_findings = models.IntegerField(default=0) # All findings (PASS + FAIL)
|
||||
failed_findings = models.IntegerField(default=0) # Non-muted failed findings
|
||||
muted_failed_findings = models.IntegerField(default=0) # Muted failed findings
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "attack_surface_overviews"
|
||||
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "scan_id", "attack_surface_type"),
|
||||
name="unique_attack_surface_per_scan",
|
||||
),
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
]
|
||||
|
||||
indexes = [
|
||||
models.Index(
|
||||
fields=["tenant_id", "scan_id"],
|
||||
name="attack_surf_tenant_scan_idx",
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-surface-overviews"
|
||||
|
||||
@@ -65,11 +65,11 @@ def get_providers(role: Role) -> QuerySet[Provider]:
|
||||
A QuerySet of Provider objects filtered by the role's provider groups.
|
||||
If the role has no provider groups, returns an empty queryset.
|
||||
"""
|
||||
tenant = role.tenant
|
||||
tenant_id = role.tenant_id
|
||||
provider_groups = role.provider_groups.all()
|
||||
if not provider_groups.exists():
|
||||
return Provider.objects.none()
|
||||
|
||||
return Provider.objects.filter(
|
||||
tenant=tenant, provider_groups__in=provider_groups
|
||||
tenant_id=tenant_id, provider_groups__in=provider_groups
|
||||
).distinct()
|
||||
|
||||
+1477
-83
File diff suppressed because it is too large
Load Diff
@@ -2,9 +2,12 @@ import uuid
|
||||
from unittest.mock import call, patch
|
||||
|
||||
import pytest
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.db import IntegrityError
|
||||
|
||||
from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY
|
||||
from api.decorators import set_tenant
|
||||
from api.decorators import handle_provider_deletion, set_tenant
|
||||
from api.exceptions import ProviderDeletedException
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -34,3 +37,142 @@ class TestSetTenantDecorator:
|
||||
|
||||
with pytest.raises(KeyError):
|
||||
random_func("test_arg")
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestHandleProviderDeletionDecorator:
|
||||
def test_success_no_exception(self, tenants_fixture, providers_fixture):
|
||||
"""Decorated function runs normally when no exception is raised."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
return "success"
|
||||
|
||||
result = task_func(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
)
|
||||
assert result == "success"
|
||||
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
def test_provider_deleted_with_provider_id(
|
||||
self, mock_filter, mock_rls, tenants_fixture
|
||||
):
|
||||
"""Raises ProviderDeletedException when provider_id provided and provider deleted."""
|
||||
tenant = tenants_fixture[0]
|
||||
deleted_provider_id = str(uuid.uuid4())
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
mock_filter.return_value.exists.return_value = False
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise ObjectDoesNotExist("Some object not found")
|
||||
|
||||
with pytest.raises(ProviderDeletedException) as exc_info:
|
||||
task_func(tenant_id=str(tenant.id), provider_id=deleted_provider_id)
|
||||
|
||||
assert deleted_provider_id in str(exc_info.value)
|
||||
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
@patch("api.decorators.Scan.objects.filter")
|
||||
def test_provider_deleted_with_scan_id(
|
||||
self, mock_scan_filter, mock_provider_filter, mock_rls, tenants_fixture
|
||||
):
|
||||
"""Raises ProviderDeletedException when scan exists but provider deleted."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan_id = str(uuid.uuid4())
|
||||
provider_id = str(uuid.uuid4())
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
|
||||
mock_scan = type("MockScan", (), {"provider_id": provider_id})()
|
||||
mock_scan_filter.return_value.first.return_value = mock_scan
|
||||
mock_provider_filter.return_value.exists.return_value = False
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise ObjectDoesNotExist("Some object not found")
|
||||
|
||||
with pytest.raises(ProviderDeletedException) as exc_info:
|
||||
task_func(tenant_id=str(tenant.id), scan_id=scan_id)
|
||||
|
||||
assert provider_id in str(exc_info.value)
|
||||
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Scan.objects.filter")
|
||||
def test_scan_deleted_cascade(self, mock_scan_filter, mock_rls, tenants_fixture):
|
||||
"""Raises ProviderDeletedException when scan was deleted (CASCADE from provider)."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan_id = str(uuid.uuid4())
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
mock_scan_filter.return_value.first.return_value = None
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise ObjectDoesNotExist("Some object not found")
|
||||
|
||||
with pytest.raises(ProviderDeletedException) as exc_info:
|
||||
task_func(tenant_id=str(tenant.id), scan_id=scan_id)
|
||||
|
||||
assert scan_id in str(exc_info.value)
|
||||
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
def test_provider_exists_reraises_original(
|
||||
self, mock_filter, mock_rls, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Re-raises original exception when provider still exists."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
mock_filter.return_value.exists.return_value = True
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise ObjectDoesNotExist("Actual object missing")
|
||||
|
||||
with pytest.raises(ObjectDoesNotExist):
|
||||
task_func(tenant_id=str(tenant.id), provider_id=str(provider.id))
|
||||
|
||||
@patch("api.decorators.rls_transaction")
|
||||
@patch("api.decorators.Provider.objects.filter")
|
||||
def test_integrity_error_provider_deleted(
|
||||
self, mock_filter, mock_rls, tenants_fixture
|
||||
):
|
||||
"""Raises ProviderDeletedException on IntegrityError when provider deleted."""
|
||||
tenant = tenants_fixture[0]
|
||||
deleted_provider_id = str(uuid.uuid4())
|
||||
|
||||
mock_rls.return_value.__enter__ = lambda s: None
|
||||
mock_rls.return_value.__exit__ = lambda s, *args: None
|
||||
mock_filter.return_value.exists.return_value = False
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise IntegrityError("FK constraint violation")
|
||||
|
||||
with pytest.raises(ProviderDeletedException):
|
||||
task_func(tenant_id=str(tenant.id), provider_id=deleted_provider_id)
|
||||
|
||||
def test_missing_provider_and_scan_raises_assertion(self, tenants_fixture):
|
||||
"""Raises AssertionError when neither provider_id nor scan_id in kwargs."""
|
||||
|
||||
@handle_provider_deletion
|
||||
def task_func(**kwargs):
|
||||
raise ObjectDoesNotExist("Some object not found")
|
||||
|
||||
with pytest.raises(AssertionError) as exc_info:
|
||||
task_func(tenant_id=str(tenants_fixture[0].id))
|
||||
|
||||
assert "provider or scan" in str(exc_info.value)
|
||||
|
||||
@@ -16,11 +16,13 @@ from api.utils import (
|
||||
return_prowler_provider,
|
||||
validate_invitation,
|
||||
)
|
||||
from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
|
||||
from prowler.providers.azure.azure_provider import AzureProvider
|
||||
from prowler.providers.gcp.gcp_provider import GcpProvider
|
||||
from prowler.providers.github.github_provider import GithubProvider
|
||||
from prowler.providers.iac.iac_provider import IacProvider
|
||||
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
|
||||
from prowler.providers.m365.m365_provider import M365Provider
|
||||
from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider
|
||||
@@ -114,6 +116,8 @@ class TestReturnProwlerProvider:
|
||||
(Provider.ProviderChoices.GITHUB.value, GithubProvider),
|
||||
(Provider.ProviderChoices.MONGODBATLAS.value, MongodbatlasProvider),
|
||||
(Provider.ProviderChoices.ORACLECLOUD.value, OraclecloudProvider),
|
||||
(Provider.ProviderChoices.IAC.value, IacProvider),
|
||||
(Provider.ProviderChoices.ALIBABACLOUD.value, AlibabacloudProvider),
|
||||
],
|
||||
)
|
||||
def test_return_prowler_provider(self, provider_type, expected_provider):
|
||||
@@ -254,6 +258,72 @@ class TestGetProwlerProviderKwargs:
|
||||
expected_result = {**secret_dict, "mutelist_content": {"key": "value"}}
|
||||
assert result == expected_result
|
||||
|
||||
def test_get_prowler_provider_kwargs_iac_provider(self):
|
||||
"""Test that IaC provider gets correct kwargs with repository URL."""
|
||||
provider_uid = "https://github.com/org/repo"
|
||||
secret_dict = {"access_token": "test_token"}
|
||||
secret_mock = MagicMock()
|
||||
secret_mock.secret = secret_dict
|
||||
|
||||
provider = MagicMock()
|
||||
provider.provider = Provider.ProviderChoices.IAC.value
|
||||
provider.secret = secret_mock
|
||||
provider.uid = provider_uid
|
||||
|
||||
result = get_prowler_provider_kwargs(provider)
|
||||
|
||||
expected_result = {
|
||||
"scan_repository_url": provider_uid,
|
||||
"oauth_app_token": "test_token",
|
||||
}
|
||||
assert result == expected_result
|
||||
|
||||
def test_get_prowler_provider_kwargs_iac_provider_without_token(self):
|
||||
"""Test that IaC provider works without access token for public repos."""
|
||||
provider_uid = "https://github.com/org/public-repo"
|
||||
secret_dict = {}
|
||||
secret_mock = MagicMock()
|
||||
secret_mock.secret = secret_dict
|
||||
|
||||
provider = MagicMock()
|
||||
provider.provider = Provider.ProviderChoices.IAC.value
|
||||
provider.secret = secret_mock
|
||||
provider.uid = provider_uid
|
||||
|
||||
result = get_prowler_provider_kwargs(provider)
|
||||
|
||||
expected_result = {"scan_repository_url": provider_uid}
|
||||
assert result == expected_result
|
||||
|
||||
def test_get_prowler_provider_kwargs_iac_provider_ignores_mutelist(self):
|
||||
"""Test that IaC provider does NOT receive mutelist_content.
|
||||
|
||||
IaC provider uses Trivy's built-in mutelist logic, so it should not
|
||||
receive mutelist_content even when a mutelist processor is configured.
|
||||
"""
|
||||
provider_uid = "https://github.com/org/repo"
|
||||
secret_dict = {"access_token": "test_token"}
|
||||
secret_mock = MagicMock()
|
||||
secret_mock.secret = secret_dict
|
||||
|
||||
mutelist_processor = MagicMock()
|
||||
mutelist_processor.configuration = {"Mutelist": {"key": "value"}}
|
||||
|
||||
provider = MagicMock()
|
||||
provider.provider = Provider.ProviderChoices.IAC.value
|
||||
provider.secret = secret_mock
|
||||
provider.uid = provider_uid
|
||||
|
||||
result = get_prowler_provider_kwargs(provider, mutelist_processor)
|
||||
|
||||
# IaC provider should NOT have mutelist_content
|
||||
assert "mutelist_content" not in result
|
||||
expected_result = {
|
||||
"scan_repository_url": provider_uid,
|
||||
"oauth_app_token": "test_token",
|
||||
}
|
||||
assert result == expected_result
|
||||
|
||||
def test_get_prowler_provider_kwargs_unsupported_provider(self):
|
||||
# Setup
|
||||
provider_uid = "provider_uid"
|
||||
|
||||
+1562
-400
File diff suppressed because it is too large
Load Diff
@@ -11,6 +11,7 @@ from api.exceptions import InvitationTokenExpiredException
|
||||
from api.models import Integration, Invitation, Processor, Provider, Resource
|
||||
from api.v1.serializers import FindingMetadataSerializer
|
||||
from prowler.lib.outputs.jira.jira import Jira, JiraBasicAuthError
|
||||
from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.aws.lib.s3.s3 import S3
|
||||
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub
|
||||
@@ -63,8 +64,9 @@ def merge_dicts(default_dict: dict, replacement_dict: dict) -> dict:
|
||||
|
||||
def return_prowler_provider(
|
||||
provider: Provider,
|
||||
) -> [
|
||||
AwsProvider
|
||||
) -> (
|
||||
AlibabacloudProvider
|
||||
| AwsProvider
|
||||
| AzureProvider
|
||||
| GcpProvider
|
||||
| GithubProvider
|
||||
@@ -73,14 +75,14 @@ def return_prowler_provider(
|
||||
| M365Provider
|
||||
| MongodbatlasProvider
|
||||
| OraclecloudProvider
|
||||
]:
|
||||
):
|
||||
"""Return the Prowler provider class based on the given provider type.
|
||||
|
||||
Args:
|
||||
provider (Provider): The provider object containing the provider type and associated secrets.
|
||||
|
||||
Returns:
|
||||
AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: The corresponding provider class.
|
||||
AlibabacloudProvider | AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: The corresponding provider class.
|
||||
|
||||
Raises:
|
||||
ValueError: If the provider type specified in `provider.provider` is not supported.
|
||||
@@ -104,6 +106,8 @@ def return_prowler_provider(
|
||||
prowler_provider = IacProvider
|
||||
case Provider.ProviderChoices.ORACLECLOUD.value:
|
||||
prowler_provider = OraclecloudProvider
|
||||
case Provider.ProviderChoices.ALIBABACLOUD.value:
|
||||
prowler_provider = AlibabacloudProvider
|
||||
case _:
|
||||
raise ValueError(f"Provider type {provider.provider} not supported")
|
||||
return prowler_provider
|
||||
@@ -158,7 +162,8 @@ def get_prowler_provider_kwargs(
|
||||
|
||||
if mutelist_processor:
|
||||
mutelist_content = mutelist_processor.configuration.get("Mutelist", {})
|
||||
if mutelist_content:
|
||||
# IaC provider doesn't support mutelist (uses Trivy's built-in logic)
|
||||
if mutelist_content and provider.provider != Provider.ProviderChoices.IAC.value:
|
||||
prowler_provider_kwargs["mutelist_content"] = mutelist_content
|
||||
|
||||
return prowler_provider_kwargs
|
||||
@@ -168,7 +173,8 @@ def initialize_prowler_provider(
|
||||
provider: Provider,
|
||||
mutelist_processor: Processor | None = None,
|
||||
) -> (
|
||||
AwsProvider
|
||||
AlibabacloudProvider
|
||||
| AwsProvider
|
||||
| AzureProvider
|
||||
| GcpProvider
|
||||
| GithubProvider
|
||||
@@ -185,9 +191,8 @@ def initialize_prowler_provider(
|
||||
mutelist_processor (Processor): The mutelist processor object containing the mutelist configuration.
|
||||
|
||||
Returns:
|
||||
AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: An instance of the corresponding provider class
|
||||
(`AwsProvider`, `AzureProvider`, `GcpProvider`, `GithubProvider`, `IacProvider`, `KubernetesProvider`, `M365Provider`, `OraclecloudProvider` or `MongodbatlasProvider`) initialized with the
|
||||
provider's secrets.
|
||||
AlibabacloudProvider | AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | MongodbatlasProvider | OraclecloudProvider: An instance of the corresponding provider class
|
||||
initialized with the provider's secrets.
|
||||
"""
|
||||
prowler_provider = return_prowler_provider(provider)
|
||||
prowler_provider_kwargs = get_prowler_provider_kwargs(provider, mutelist_processor)
|
||||
@@ -381,10 +386,18 @@ def get_findings_metadata_no_aggregations(tenant_id: str, filtered_queryset):
|
||||
regions = sorted({region for region in aggregation["regions"] or [] if region})
|
||||
resource_types = sorted(set(aggregation["resource_types"] or []))
|
||||
|
||||
# Aggregate categories from findings
|
||||
categories_set = set()
|
||||
for categories_list in filtered_queryset.values_list("categories", flat=True):
|
||||
if categories_list:
|
||||
categories_set.update(categories_list)
|
||||
categories = sorted(categories_set)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"resource_types": resource_types,
|
||||
"categories": categories,
|
||||
}
|
||||
|
||||
serializer = FindingMetadataSerializer(data=result)
|
||||
|
||||
@@ -40,11 +40,16 @@ class BedrockCredentialsSerializer(serializers.Serializer):
|
||||
"""
|
||||
Serializer for AWS Bedrock credentials validation.
|
||||
|
||||
Validates long-term AWS credentials (AKIA) and region format.
|
||||
Supports two authentication methods:
|
||||
1. AWS access key + secret key
|
||||
2. Bedrock API key (bearer token)
|
||||
|
||||
In both cases, region is mandatory.
|
||||
"""
|
||||
|
||||
access_key_id = serializers.CharField()
|
||||
secret_access_key = serializers.CharField()
|
||||
access_key_id = serializers.CharField(required=False, allow_blank=False)
|
||||
secret_access_key = serializers.CharField(required=False, allow_blank=False)
|
||||
api_key = serializers.CharField(required=False, allow_blank=False)
|
||||
region = serializers.CharField()
|
||||
|
||||
def validate_access_key_id(self, value: str) -> str:
|
||||
@@ -65,6 +70,15 @@ class BedrockCredentialsSerializer(serializers.Serializer):
|
||||
)
|
||||
return value
|
||||
|
||||
def validate_api_key(self, value: str) -> str:
|
||||
"""
|
||||
Validate Bedrock API key (bearer token).
|
||||
"""
|
||||
pattern = r"^ABSKQmVkcm9ja0FQSUtleS[A-Za-z0-9+/=]{110}$"
|
||||
if not re.match(pattern, value or ""):
|
||||
raise serializers.ValidationError("Invalid Bedrock API key format.")
|
||||
return value
|
||||
|
||||
def validate_region(self, value: str) -> str:
|
||||
"""Validate AWS region format."""
|
||||
pattern = r"^[a-z]{2}-[a-z]+-\d+$"
|
||||
@@ -74,6 +88,50 @@ class BedrockCredentialsSerializer(serializers.Serializer):
|
||||
)
|
||||
return value
|
||||
|
||||
def validate(self, attrs):
|
||||
"""
|
||||
Enforce either:
|
||||
- access_key_id + secret_access_key + region
|
||||
OR
|
||||
- api_key + region
|
||||
"""
|
||||
access_key_id = attrs.get("access_key_id")
|
||||
secret_access_key = attrs.get("secret_access_key")
|
||||
api_key = attrs.get("api_key")
|
||||
region = attrs.get("region")
|
||||
|
||||
errors = {}
|
||||
|
||||
if not region:
|
||||
errors["region"] = ["Region is required."]
|
||||
|
||||
using_access_keys = bool(access_key_id or secret_access_key)
|
||||
using_api_key = api_key is not None and api_key != ""
|
||||
|
||||
if using_access_keys and using_api_key:
|
||||
errors["non_field_errors"] = [
|
||||
"Provide either access key + secret key OR api key, not both."
|
||||
]
|
||||
elif not using_access_keys and not using_api_key:
|
||||
errors["non_field_errors"] = [
|
||||
"You must provide either access key + secret key OR api key."
|
||||
]
|
||||
elif using_access_keys:
|
||||
# Both access_key_id and secret_access_key must be present together
|
||||
if not access_key_id:
|
||||
errors.setdefault("access_key_id", []).append(
|
||||
"AWS access key ID is required when using access key authentication."
|
||||
)
|
||||
if not secret_access_key:
|
||||
errors.setdefault("secret_access_key", []).append(
|
||||
"AWS secret access key is required when using access key authentication."
|
||||
)
|
||||
|
||||
if errors:
|
||||
raise serializers.ValidationError(errors)
|
||||
|
||||
return attrs
|
||||
|
||||
def to_internal_value(self, data):
|
||||
"""Check for unknown fields before DRF filters them out."""
|
||||
if not isinstance(data, dict):
|
||||
@@ -111,6 +169,15 @@ class BedrockCredentialsUpdateSerializer(BedrockCredentialsSerializer):
|
||||
for field in self.fields.values():
|
||||
field.required = False
|
||||
|
||||
def validate(self, attrs):
|
||||
"""
|
||||
For updates, this serializer only checks individual fields.
|
||||
It does NOT enforce the "either access keys OR api key" rule.
|
||||
That rule is applied later, after merging with existing stored
|
||||
credentials, in LighthouseProviderConfigUpdateSerializer.
|
||||
"""
|
||||
return attrs
|
||||
|
||||
|
||||
class OpenAICompatibleCredentialsSerializer(serializers.Serializer):
|
||||
"""
|
||||
@@ -168,27 +235,51 @@ class OpenAICompatibleCredentialsSerializer(serializers.Serializer):
|
||||
"required": ["api_key"],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"title": "AWS Bedrock Credentials",
|
||||
"properties": {
|
||||
"access_key_id": {
|
||||
"type": "string",
|
||||
"description": "AWS access key ID.",
|
||||
"pattern": "^AKIA[0-9A-Z]{16}$",
|
||||
"oneOf": [
|
||||
{
|
||||
"title": "IAM Access Key Pair",
|
||||
"type": "object",
|
||||
"description": "Authenticate with AWS access key and secret key. Recommended when you manage IAM users or roles.",
|
||||
"properties": {
|
||||
"access_key_id": {
|
||||
"type": "string",
|
||||
"description": "AWS access key ID.",
|
||||
"pattern": "^AKIA[0-9A-Z]{16}$",
|
||||
},
|
||||
"secret_access_key": {
|
||||
"type": "string",
|
||||
"description": "AWS secret access key.",
|
||||
"pattern": "^[A-Za-z0-9/+=]{40}$",
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
"description": "AWS region identifier where Bedrock is available. Examples: us-east-1, "
|
||||
"us-west-2, eu-west-1, ap-northeast-1.",
|
||||
"pattern": "^[a-z]{2}-[a-z]+-\\d+$",
|
||||
},
|
||||
},
|
||||
"required": ["access_key_id", "secret_access_key", "region"],
|
||||
},
|
||||
"secret_access_key": {
|
||||
"type": "string",
|
||||
"description": "AWS secret access key.",
|
||||
"pattern": "^[A-Za-z0-9/+=]{40}$",
|
||||
{
|
||||
"title": "Amazon Bedrock API Key",
|
||||
"type": "object",
|
||||
"description": "Authenticate with an Amazon Bedrock API key (bearer token). Region is still required.",
|
||||
"properties": {
|
||||
"api_key": {
|
||||
"type": "string",
|
||||
"description": "Amazon Bedrock API key (bearer token).",
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
"description": "AWS region identifier where Bedrock is available. Examples: us-east-1, "
|
||||
"us-west-2, eu-west-1, ap-northeast-1.",
|
||||
"pattern": "^[a-z]{2}-[a-z]+-\\d+$",
|
||||
},
|
||||
},
|
||||
"required": ["api_key", "region"],
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
"description": "AWS region identifier where Bedrock is available. Examples: us-east-1, "
|
||||
"us-west-2, eu-west-1, ap-northeast-1.",
|
||||
"pattern": "^[a-z]{2}-[a-z]+-\\d+$",
|
||||
},
|
||||
},
|
||||
"required": ["access_key_id", "secret_access_key", "region"],
|
||||
],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
|
||||
@@ -304,6 +304,48 @@ from rest_framework_json_api import serializers
|
||||
},
|
||||
"required": ["atlas_public_key", "atlas_private_key"],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"title": "Alibaba Cloud Static Credentials",
|
||||
"properties": {
|
||||
"access_key_id": {
|
||||
"type": "string",
|
||||
"description": "The Alibaba Cloud access key ID for authentication.",
|
||||
},
|
||||
"access_key_secret": {
|
||||
"type": "string",
|
||||
"description": "The Alibaba Cloud access key secret for authentication.",
|
||||
},
|
||||
"security_token": {
|
||||
"type": "string",
|
||||
"description": "The STS security token for temporary credentials (optional).",
|
||||
},
|
||||
},
|
||||
"required": ["access_key_id", "access_key_secret"],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"title": "Alibaba Cloud RAM Role Assumption",
|
||||
"properties": {
|
||||
"role_arn": {
|
||||
"type": "string",
|
||||
"description": "The ARN of the RAM role to assume (e.g., acs:ram::1234567890123456:role/ProwlerRole).",
|
||||
},
|
||||
"access_key_id": {
|
||||
"type": "string",
|
||||
"description": "The Alibaba Cloud access key ID of the RAM user that will assume the role.",
|
||||
},
|
||||
"access_key_secret": {
|
||||
"type": "string",
|
||||
"description": "The Alibaba Cloud access key secret of the RAM user that will assume the role.",
|
||||
},
|
||||
"role_session_name": {
|
||||
"type": "string",
|
||||
"description": "An identifier for the role session (optional, defaults to 'ProwlerSession').",
|
||||
},
|
||||
},
|
||||
"required": ["role_arn", "access_key_id", "access_key_secret"],
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
@@ -72,6 +72,42 @@ from api.v1.serializer_utils.processors import ProcessorConfigField
|
||||
from api.v1.serializer_utils.providers import ProviderSecretField
|
||||
from prowler.lib.mutelist.mutelist import Mutelist
|
||||
|
||||
# Base
|
||||
|
||||
|
||||
class BaseModelSerializerV1(serializers.ModelSerializer):
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
|
||||
class BaseSerializerV1(serializers.Serializer):
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
|
||||
class BaseWriteSerializer(BaseModelSerializerV1):
|
||||
def validate(self, data):
|
||||
if hasattr(self, "initial_data"):
|
||||
initial_data = set(self.initial_data.keys()) - {"id", "type"}
|
||||
unknown_keys = initial_data - set(self.fields.keys())
|
||||
if unknown_keys:
|
||||
raise ValidationError(f"Invalid fields: {unknown_keys}")
|
||||
return data
|
||||
|
||||
|
||||
class RLSSerializer(BaseModelSerializerV1):
|
||||
def create(self, validated_data):
|
||||
tenant_id = self.context.get("tenant_id")
|
||||
validated_data["tenant_id"] = tenant_id
|
||||
return super().create(validated_data)
|
||||
|
||||
|
||||
class StateEnumSerializerField(serializers.ChoiceField):
|
||||
def __init__(self, **kwargs):
|
||||
kwargs["choices"] = StateChoices.choices
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
# Tokens
|
||||
|
||||
|
||||
@@ -179,7 +215,7 @@ class TokenSocialLoginSerializer(BaseTokenSerializer):
|
||||
|
||||
|
||||
# TODO: Check if we can change the parent class to TokenRefreshSerializer from rest_framework_simplejwt.serializers
|
||||
class TokenRefreshSerializer(serializers.Serializer):
|
||||
class TokenRefreshSerializer(BaseSerializerV1):
|
||||
refresh = serializers.CharField()
|
||||
|
||||
# Output token
|
||||
@@ -213,7 +249,7 @@ class TokenRefreshSerializer(serializers.Serializer):
|
||||
raise ValidationError({"refresh": "Invalid or expired token"})
|
||||
|
||||
|
||||
class TokenSwitchTenantSerializer(serializers.Serializer):
|
||||
class TokenSwitchTenantSerializer(BaseSerializerV1):
|
||||
tenant_id = serializers.UUIDField(
|
||||
write_only=True, help_text="The tenant ID for which to request a new token."
|
||||
)
|
||||
@@ -237,41 +273,10 @@ class TokenSwitchTenantSerializer(serializers.Serializer):
|
||||
return generate_tokens(user, tenant_id)
|
||||
|
||||
|
||||
# Base
|
||||
|
||||
|
||||
class BaseSerializerV1(serializers.ModelSerializer):
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
|
||||
class BaseWriteSerializer(BaseSerializerV1):
|
||||
def validate(self, data):
|
||||
if hasattr(self, "initial_data"):
|
||||
initial_data = set(self.initial_data.keys()) - {"id", "type"}
|
||||
unknown_keys = initial_data - set(self.fields.keys())
|
||||
if unknown_keys:
|
||||
raise ValidationError(f"Invalid fields: {unknown_keys}")
|
||||
return data
|
||||
|
||||
|
||||
class RLSSerializer(BaseSerializerV1):
|
||||
def create(self, validated_data):
|
||||
tenant_id = self.context.get("tenant_id")
|
||||
validated_data["tenant_id"] = tenant_id
|
||||
return super().create(validated_data)
|
||||
|
||||
|
||||
class StateEnumSerializerField(serializers.ChoiceField):
|
||||
def __init__(self, **kwargs):
|
||||
kwargs["choices"] = StateChoices.choices
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
# Users
|
||||
|
||||
|
||||
class UserSerializer(BaseSerializerV1):
|
||||
class UserSerializer(BaseModelSerializerV1):
|
||||
"""
|
||||
Serializer for the User model.
|
||||
"""
|
||||
@@ -402,7 +407,7 @@ class UserUpdateSerializer(BaseWriteSerializer):
|
||||
return super().update(instance, validated_data)
|
||||
|
||||
|
||||
class RoleResourceIdentifierSerializer(serializers.Serializer):
|
||||
class RoleResourceIdentifierSerializer(BaseSerializerV1):
|
||||
resource_type = serializers.CharField(source="type")
|
||||
id = serializers.UUIDField()
|
||||
|
||||
@@ -585,7 +590,7 @@ class TaskSerializer(RLSSerializer, TaskBase):
|
||||
# Tenants
|
||||
|
||||
|
||||
class TenantSerializer(BaseSerializerV1):
|
||||
class TenantSerializer(BaseModelSerializerV1):
|
||||
"""
|
||||
Serializer for the Tenant model.
|
||||
"""
|
||||
@@ -597,7 +602,7 @@ class TenantSerializer(BaseSerializerV1):
|
||||
fields = ["id", "name", "memberships"]
|
||||
|
||||
|
||||
class TenantIncludeSerializer(BaseSerializerV1):
|
||||
class TenantIncludeSerializer(BaseModelSerializerV1):
|
||||
class Meta:
|
||||
model = Tenant
|
||||
fields = ["id", "name"]
|
||||
@@ -773,7 +778,7 @@ class ProviderGroupUpdateSerializer(ProviderGroupSerializer):
|
||||
return super().update(instance, validated_data)
|
||||
|
||||
|
||||
class ProviderResourceIdentifierSerializer(serializers.Serializer):
|
||||
class ProviderResourceIdentifierSerializer(BaseSerializerV1):
|
||||
resource_type = serializers.CharField(source="type")
|
||||
id = serializers.UUIDField()
|
||||
|
||||
@@ -1110,7 +1115,7 @@ class ScanTaskSerializer(RLSSerializer):
|
||||
]
|
||||
|
||||
|
||||
class ScanReportSerializer(serializers.Serializer):
|
||||
class ScanReportSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField(source="scan")
|
||||
|
||||
class Meta:
|
||||
@@ -1118,7 +1123,7 @@ class ScanReportSerializer(serializers.Serializer):
|
||||
fields = ["id"]
|
||||
|
||||
|
||||
class ScanComplianceReportSerializer(serializers.Serializer):
|
||||
class ScanComplianceReportSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField(source="scan")
|
||||
name = serializers.CharField()
|
||||
|
||||
@@ -1267,7 +1272,7 @@ class ResourceIncludeSerializer(RLSSerializer):
|
||||
return fields
|
||||
|
||||
|
||||
class ResourceMetadataSerializer(serializers.Serializer):
|
||||
class ResourceMetadataSerializer(BaseSerializerV1):
|
||||
services = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
types = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
@@ -1296,6 +1301,7 @@ class FindingSerializer(RLSSerializer):
|
||||
"severity",
|
||||
"check_id",
|
||||
"check_metadata",
|
||||
"categories",
|
||||
"raw_result",
|
||||
"inserted_at",
|
||||
"updated_at",
|
||||
@@ -1337,7 +1343,7 @@ class FindingIncludeSerializer(RLSSerializer):
|
||||
|
||||
|
||||
# To be removed when the related endpoint is removed as well
|
||||
class FindingDynamicFilterSerializer(serializers.Serializer):
|
||||
class FindingDynamicFilterSerializer(BaseSerializerV1):
|
||||
services = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
|
||||
@@ -1345,12 +1351,13 @@ class FindingDynamicFilterSerializer(serializers.Serializer):
|
||||
resource_name = "finding-dynamic-filters"
|
||||
|
||||
|
||||
class FindingMetadataSerializer(serializers.Serializer):
|
||||
class FindingMetadataSerializer(BaseSerializerV1):
|
||||
services = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
resource_types = serializers.ListField(
|
||||
child=serializers.CharField(), allow_empty=True
|
||||
)
|
||||
categories = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
|
||||
|
||||
@@ -1383,12 +1390,23 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
|
||||
serializer = OracleCloudProviderSecret(data=secret)
|
||||
elif provider_type == Provider.ProviderChoices.MONGODBATLAS.value:
|
||||
serializer = MongoDBAtlasProviderSecret(data=secret)
|
||||
elif provider_type == Provider.ProviderChoices.ALIBABACLOUD.value:
|
||||
serializer = AlibabaCloudProviderSecret(data=secret)
|
||||
else:
|
||||
raise serializers.ValidationError(
|
||||
{"provider": f"Provider type not supported {provider_type}"}
|
||||
)
|
||||
elif secret_type == ProviderSecret.TypeChoices.ROLE:
|
||||
serializer = AWSRoleAssumptionProviderSecret(data=secret)
|
||||
if provider_type == Provider.ProviderChoices.AWS.value:
|
||||
serializer = AWSRoleAssumptionProviderSecret(data=secret)
|
||||
elif provider_type == Provider.ProviderChoices.ALIBABACLOUD.value:
|
||||
serializer = AlibabaCloudRoleAssumptionProviderSecret(data=secret)
|
||||
else:
|
||||
raise serializers.ValidationError(
|
||||
{
|
||||
"secret_type": f"Role assumption not supported for provider type: {provider_type}"
|
||||
}
|
||||
)
|
||||
elif secret_type == ProviderSecret.TypeChoices.SERVICE_ACCOUNT:
|
||||
serializer = GCPServiceAccountProviderSecret(data=secret)
|
||||
else:
|
||||
@@ -1525,6 +1543,34 @@ class OracleCloudProviderSecret(serializers.Serializer):
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
|
||||
class AlibabaCloudProviderSecret(serializers.Serializer):
|
||||
access_key_id = serializers.CharField()
|
||||
access_key_secret = serializers.CharField()
|
||||
security_token = serializers.CharField(required=False)
|
||||
|
||||
class Meta:
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
|
||||
class AlibabaCloudRoleAssumptionProviderSecret(serializers.Serializer):
|
||||
role_arn = serializers.CharField(
|
||||
help_text="Access Key ID of the RAM user that will assume the role"
|
||||
)
|
||||
access_key_id = serializers.CharField(
|
||||
help_text="Access Key ID of the RAM user that will assume the role"
|
||||
)
|
||||
access_key_secret = serializers.CharField(
|
||||
help_text="Access Key Secret of the RAM user that will assume the role"
|
||||
)
|
||||
role_session_name = serializers.CharField(
|
||||
required=False,
|
||||
help_text="Session name for the assumed role session (optional, defaults to 'ProwlerSession')",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
|
||||
class AWSRoleAssumptionProviderSecret(serializers.Serializer):
|
||||
role_arn = serializers.CharField()
|
||||
external_id = serializers.CharField()
|
||||
@@ -2039,7 +2085,7 @@ class RoleProviderGroupRelationshipSerializer(RLSSerializer, BaseWriteSerializer
|
||||
# Compliance overview
|
||||
|
||||
|
||||
class ComplianceOverviewSerializer(serializers.Serializer):
|
||||
class ComplianceOverviewSerializer(BaseSerializerV1):
|
||||
"""
|
||||
Serializer for compliance requirement status aggregated by compliance framework.
|
||||
|
||||
@@ -2061,7 +2107,7 @@ class ComplianceOverviewSerializer(serializers.Serializer):
|
||||
resource_name = "compliance-overviews"
|
||||
|
||||
|
||||
class ComplianceOverviewDetailSerializer(serializers.Serializer):
|
||||
class ComplianceOverviewDetailSerializer(BaseSerializerV1):
|
||||
"""
|
||||
Serializer for detailed compliance requirement information.
|
||||
|
||||
@@ -2090,7 +2136,7 @@ class ComplianceOverviewDetailThreatscoreSerializer(ComplianceOverviewDetailSeri
|
||||
total_findings = serializers.IntegerField()
|
||||
|
||||
|
||||
class ComplianceOverviewAttributesSerializer(serializers.Serializer):
|
||||
class ComplianceOverviewAttributesSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField()
|
||||
compliance_name = serializers.CharField()
|
||||
framework_description = serializers.CharField()
|
||||
@@ -2104,7 +2150,7 @@ class ComplianceOverviewAttributesSerializer(serializers.Serializer):
|
||||
resource_name = "compliance-requirements-attributes"
|
||||
|
||||
|
||||
class ComplianceOverviewMetadataSerializer(serializers.Serializer):
|
||||
class ComplianceOverviewMetadataSerializer(BaseSerializerV1):
|
||||
regions = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||
|
||||
class JSONAPIMeta:
|
||||
@@ -2114,7 +2160,7 @@ class ComplianceOverviewMetadataSerializer(serializers.Serializer):
|
||||
# Overviews
|
||||
|
||||
|
||||
class OverviewProviderSerializer(serializers.Serializer):
|
||||
class OverviewProviderSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField(source="provider")
|
||||
findings = serializers.SerializerMethodField(read_only=True)
|
||||
resources = serializers.SerializerMethodField(read_only=True)
|
||||
@@ -2122,9 +2168,6 @@ class OverviewProviderSerializer(serializers.Serializer):
|
||||
class JSONAPIMeta:
|
||||
resource_name = "providers-overview"
|
||||
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
@extend_schema_field(
|
||||
{
|
||||
"type": "object",
|
||||
@@ -2158,18 +2201,15 @@ class OverviewProviderSerializer(serializers.Serializer):
|
||||
}
|
||||
|
||||
|
||||
class OverviewProviderCountSerializer(serializers.Serializer):
|
||||
class OverviewProviderCountSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField(source="provider")
|
||||
count = serializers.IntegerField()
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "providers-count-overview"
|
||||
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
|
||||
class OverviewFindingSerializer(serializers.Serializer):
|
||||
class OverviewFindingSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField(default="n/a")
|
||||
new = serializers.IntegerField()
|
||||
changed = serializers.IntegerField()
|
||||
@@ -2188,15 +2228,12 @@ class OverviewFindingSerializer(serializers.Serializer):
|
||||
class JSONAPIMeta:
|
||||
resource_name = "findings-overview"
|
||||
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["pass"] = self.fields.pop("_pass")
|
||||
|
||||
|
||||
class OverviewSeveritySerializer(serializers.Serializer):
|
||||
class OverviewSeveritySerializer(BaseSerializerV1):
|
||||
id = serializers.CharField(default="n/a")
|
||||
critical = serializers.IntegerField()
|
||||
high = serializers.IntegerField()
|
||||
@@ -2207,11 +2244,24 @@ class OverviewSeveritySerializer(serializers.Serializer):
|
||||
class JSONAPIMeta:
|
||||
resource_name = "findings-severity-overview"
|
||||
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
class FindingsSeverityOverTimeSerializer(BaseSerializerV1):
|
||||
"""Serializer for daily findings severity trend data."""
|
||||
|
||||
id = serializers.DateField(source="date")
|
||||
critical = serializers.IntegerField()
|
||||
high = serializers.IntegerField()
|
||||
medium = serializers.IntegerField()
|
||||
low = serializers.IntegerField()
|
||||
informational = serializers.IntegerField()
|
||||
muted = serializers.IntegerField()
|
||||
scan_ids = serializers.ListField(child=serializers.UUIDField())
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "findings-severity-over-time"
|
||||
|
||||
|
||||
class OverviewServiceSerializer(serializers.Serializer):
|
||||
class OverviewServiceSerializer(BaseSerializerV1):
|
||||
id = serializers.CharField(source="service")
|
||||
total = serializers.IntegerField()
|
||||
_pass = serializers.IntegerField()
|
||||
@@ -2225,6 +2275,54 @@ class OverviewServiceSerializer(serializers.Serializer):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["pass"] = self.fields.pop("_pass")
|
||||
|
||||
|
||||
class AttackSurfaceOverviewSerializer(BaseSerializerV1):
|
||||
"""Serializer for attack surface overview aggregations."""
|
||||
|
||||
id = serializers.CharField(source="attack_surface_type")
|
||||
total_findings = serializers.IntegerField()
|
||||
failed_findings = serializers.IntegerField()
|
||||
muted_failed_findings = serializers.IntegerField()
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "attack-surface-overviews"
|
||||
|
||||
|
||||
class CategoryOverviewSerializer(BaseSerializerV1):
|
||||
"""Serializer for category overview aggregations."""
|
||||
|
||||
id = serializers.CharField(source="category")
|
||||
total_findings = serializers.IntegerField()
|
||||
failed_findings = serializers.IntegerField()
|
||||
new_failed_findings = serializers.IntegerField()
|
||||
severity = serializers.JSONField(
|
||||
help_text="Severity breakdown: {informational, low, medium, high, critical}"
|
||||
)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "category-overviews"
|
||||
|
||||
|
||||
class OverviewRegionSerializer(serializers.Serializer):
|
||||
id = serializers.SerializerMethodField()
|
||||
provider_type = serializers.CharField()
|
||||
region = serializers.CharField()
|
||||
total = serializers.IntegerField()
|
||||
_pass = serializers.IntegerField()
|
||||
fail = serializers.IntegerField()
|
||||
muted = serializers.IntegerField()
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "regions-overview"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["pass"] = self.fields.pop("_pass")
|
||||
|
||||
def get_id(self, obj):
|
||||
"""Generate unique ID from provider_type and region."""
|
||||
return f"{obj['provider_type']}:{obj['region']}"
|
||||
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
|
||||
@@ -2232,7 +2330,7 @@ class OverviewServiceSerializer(serializers.Serializer):
|
||||
# Schedules
|
||||
|
||||
|
||||
class ScheduleDailyCreateSerializer(serializers.Serializer):
|
||||
class ScheduleDailyCreateSerializer(BaseSerializerV1):
|
||||
provider_id = serializers.UUIDField(required=True)
|
||||
|
||||
class JSONAPIMeta:
|
||||
@@ -2568,7 +2666,7 @@ class IntegrationUpdateSerializer(BaseWriteIntegrationSerializer):
|
||||
return representation
|
||||
|
||||
|
||||
class IntegrationJiraDispatchSerializer(serializers.Serializer):
|
||||
class IntegrationJiraDispatchSerializer(BaseSerializerV1):
|
||||
"""
|
||||
Serializer for dispatching findings to JIRA integration.
|
||||
"""
|
||||
@@ -2731,14 +2829,14 @@ class ProcessorUpdateSerializer(BaseWriteSerializer):
|
||||
# SSO
|
||||
|
||||
|
||||
class SamlInitiateSerializer(serializers.Serializer):
|
||||
class SamlInitiateSerializer(BaseSerializerV1):
|
||||
email_domain = serializers.CharField()
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "saml-initiate"
|
||||
|
||||
|
||||
class SamlMetadataSerializer(serializers.Serializer):
|
||||
class SamlMetadataSerializer(BaseSerializerV1):
|
||||
class JSONAPIMeta:
|
||||
resource_name = "saml-meta"
|
||||
|
||||
@@ -3270,6 +3368,19 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
|
||||
and provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
|
||||
):
|
||||
# For updates, enforce that the authentication method (access keys vs API key)
|
||||
# is immutable. To switch methods, the UI must delete and recreate the provider.
|
||||
existing_credentials = (
|
||||
self.instance.credentials_decoded if self.instance else {}
|
||||
) or {}
|
||||
|
||||
existing_uses_api_key = "api_key" in existing_credentials
|
||||
existing_uses_access_keys = any(
|
||||
k in existing_credentials
|
||||
for k in ("access_key_id", "secret_access_key")
|
||||
)
|
||||
|
||||
# First run field-level validation on the partial payload
|
||||
try:
|
||||
BedrockCredentialsUpdateSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
@@ -3280,6 +3391,31 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
|
||||
# Then enforce invariants about not changing the auth method
|
||||
# If the existing config uses an API key, forbid introducing access keys.
|
||||
if existing_uses_api_key and any(
|
||||
k in credentials for k in ("access_key_id", "secret_access_key")
|
||||
):
|
||||
raise ValidationError(
|
||||
{
|
||||
"credentials/non_field_errors": [
|
||||
"Cannot change Bedrock authentication method from API key "
|
||||
"to access key via update. Delete and recreate the provider instead."
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
# If the existing config uses access keys, forbid introducing an API key.
|
||||
if existing_uses_access_keys and "api_key" in credentials:
|
||||
raise ValidationError(
|
||||
{
|
||||
"credentials/non_field_errors": [
|
||||
"Cannot change Bedrock authentication method from access key "
|
||||
"to API key via update. Delete and recreate the provider instead."
|
||||
]
|
||||
}
|
||||
)
|
||||
elif (
|
||||
credentials is not None
|
||||
and provider_type
|
||||
|
||||
+620
-106
@@ -98,8 +98,11 @@ from api.db_router import MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.exceptions import TaskFailedException
|
||||
from api.filters import (
|
||||
AttackSurfaceOverviewFilter,
|
||||
CategoryOverviewFilter,
|
||||
ComplianceOverviewFilter,
|
||||
CustomDjangoFilterBackend,
|
||||
DailySeveritySummaryFilter,
|
||||
FindingFilter,
|
||||
IntegrationFilter,
|
||||
IntegrationJiraFindingsFilter,
|
||||
@@ -119,7 +122,6 @@ from api.filters import (
|
||||
ScanFilter,
|
||||
ScanSummaryFilter,
|
||||
ScanSummarySeverityFilter,
|
||||
ServiceOverviewFilter,
|
||||
TaskFilter,
|
||||
TenantApiKeyFilter,
|
||||
TenantFilter,
|
||||
@@ -127,8 +129,10 @@ from api.filters import (
|
||||
UserFilter,
|
||||
)
|
||||
from api.models import (
|
||||
AttackSurfaceOverview,
|
||||
ComplianceOverviewSummary,
|
||||
ComplianceRequirementOverview,
|
||||
DailySeveritySummary,
|
||||
Finding,
|
||||
Integration,
|
||||
Invitation,
|
||||
@@ -153,6 +157,7 @@ from api.models import (
|
||||
SAMLDomainIndex,
|
||||
SAMLToken,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanSummary,
|
||||
SeverityChoices,
|
||||
StateChoices,
|
||||
@@ -173,6 +178,8 @@ from api.utils import (
|
||||
from api.uuid_utils import datetime_to_uuid7, uuid7_start
|
||||
from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin
|
||||
from api.v1.serializers import (
|
||||
AttackSurfaceOverviewSerializer,
|
||||
CategoryOverviewSerializer,
|
||||
ComplianceOverviewAttributesSerializer,
|
||||
ComplianceOverviewDetailSerializer,
|
||||
ComplianceOverviewDetailThreatscoreSerializer,
|
||||
@@ -181,6 +188,7 @@ from api.v1.serializers import (
|
||||
FindingDynamicFilterSerializer,
|
||||
FindingMetadataSerializer,
|
||||
FindingSerializer,
|
||||
FindingsSeverityOverTimeSerializer,
|
||||
IntegrationCreateSerializer,
|
||||
IntegrationJiraDispatchSerializer,
|
||||
IntegrationSerializer,
|
||||
@@ -205,6 +213,7 @@ from api.v1.serializers import (
|
||||
OverviewFindingSerializer,
|
||||
OverviewProviderCountSerializer,
|
||||
OverviewProviderSerializer,
|
||||
OverviewRegionSerializer,
|
||||
OverviewServiceSerializer,
|
||||
OverviewSeveritySerializer,
|
||||
ProcessorCreateSerializer,
|
||||
@@ -350,7 +359,7 @@ class SchemaView(SpectacularAPIView):
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
spectacular_settings.TITLE = "Prowler API"
|
||||
spectacular_settings.VERSION = "1.15.0"
|
||||
spectacular_settings.VERSION = "1.18.0"
|
||||
spectacular_settings.DESCRIPTION = (
|
||||
"Prowler API specification.\n\nThis file is auto-generated."
|
||||
)
|
||||
@@ -1662,6 +1671,44 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet):
|
||||
),
|
||||
},
|
||||
),
|
||||
ens=extend_schema(
|
||||
tags=["Scan"],
|
||||
summary="Retrieve ENS RD2022 compliance report",
|
||||
description="Download ENS RD2022 compliance report (e.g., 'ens_rd2022_aws') as a PDF file.",
|
||||
request=None,
|
||||
responses={
|
||||
200: OpenApiResponse(
|
||||
description="PDF file containing the ENS compliance report"
|
||||
),
|
||||
202: OpenApiResponse(description="The task is in progress"),
|
||||
401: OpenApiResponse(
|
||||
description="API key missing or user not Authenticated"
|
||||
),
|
||||
403: OpenApiResponse(description="There is a problem with credentials"),
|
||||
404: OpenApiResponse(
|
||||
description="The scan has no ENS reports, or the ENS report generation task has not started yet"
|
||||
),
|
||||
},
|
||||
),
|
||||
nis2=extend_schema(
|
||||
tags=["Scan"],
|
||||
summary="Retrieve NIS2 compliance report",
|
||||
description="Download NIS2 compliance report (Directive (EU) 2022/2555) as a PDF file.",
|
||||
request=None,
|
||||
responses={
|
||||
200: OpenApiResponse(
|
||||
description="PDF file containing the NIS2 compliance report"
|
||||
),
|
||||
202: OpenApiResponse(description="The task is in progress"),
|
||||
401: OpenApiResponse(
|
||||
description="API key missing or user not Authenticated"
|
||||
),
|
||||
403: OpenApiResponse(description="There is a problem with credentials"),
|
||||
404: OpenApiResponse(
|
||||
description="The scan has no NIS2 reports, or the NIS2 report generation task has not started yet"
|
||||
),
|
||||
},
|
||||
),
|
||||
)
|
||||
@method_decorator(CACHE_DECORATOR, name="list")
|
||||
@method_decorator(CACHE_DECORATOR, name="retrieve")
|
||||
@@ -1721,6 +1768,12 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
elif self.action == "threatscore":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "ens":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "nis2":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
return super().get_serializer_class()
|
||||
|
||||
def partial_update(self, request, *args, **kwargs):
|
||||
@@ -1974,6 +2027,7 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
if running_resp:
|
||||
return running_resp
|
||||
|
||||
# TODO: add detailed response if the compliance framework is not supported for the provider
|
||||
if not scan.output_location:
|
||||
return Response(
|
||||
{
|
||||
@@ -2002,6 +2056,85 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
content, filename = loader
|
||||
return self._serve_file(content, filename, "application/pdf")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_name="ens",
|
||||
)
|
||||
def ens(self, request, pk=None):
|
||||
scan = self.get_object()
|
||||
running_resp = self._get_task_status(scan)
|
||||
if running_resp:
|
||||
return running_resp
|
||||
|
||||
# TODO: add detailed response if the compliance framework is not supported for the provider
|
||||
if not scan.output_location:
|
||||
return Response(
|
||||
{
|
||||
"detail": "The scan has no reports, or the ENS report generation task has not started yet."
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
if scan.output_location.startswith("s3://"):
|
||||
bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "")
|
||||
key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/")
|
||||
prefix = os.path.join(
|
||||
os.path.dirname(key_prefix),
|
||||
"ens",
|
||||
"*_ens_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "ens", "*_ens_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
return self._serve_file(content, filename, "application/pdf")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_name="nis2",
|
||||
)
|
||||
def nis2(self, request, pk=None):
|
||||
scan = self.get_object()
|
||||
running_resp = self._get_task_status(scan)
|
||||
if running_resp:
|
||||
return running_resp
|
||||
|
||||
if not scan.output_location:
|
||||
return Response(
|
||||
{
|
||||
"detail": "The scan has no reports, or the NIS2 report generation task has not started yet."
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
if scan.output_location.startswith("s3://"):
|
||||
bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "")
|
||||
key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/")
|
||||
prefix = os.path.join(
|
||||
os.path.dirname(key_prefix),
|
||||
"nis2",
|
||||
"*_nis2_report.pdf",
|
||||
)
|
||||
loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "nis2", "*_nis2_report.pdf")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, Response):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
return self._serve_file(content, filename, "application/pdf")
|
||||
|
||||
def create(self, request, *args, **kwargs):
|
||||
input_serializer = self.get_serializer(data=request.data)
|
||||
input_serializer.is_valid(raise_exception=True)
|
||||
@@ -2014,7 +2147,7 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
# Disabled for now
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute"),
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute")
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2629,12 +2762,15 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
|
||||
queryset = ResourceScanSummary.objects.filter(tenant_id=tenant_id)
|
||||
scan_based_filters = {}
|
||||
category_scan_filters = {} # Filters for ScanCategorySummary
|
||||
|
||||
if scans := query_params.get("filter[scan__in]") or query_params.get(
|
||||
"filter[scan]"
|
||||
):
|
||||
queryset = queryset.filter(scan_id__in=scans.split(","))
|
||||
scan_based_filters = {"id__in": scans.split(",")}
|
||||
scan_ids_list = scans.split(",")
|
||||
queryset = queryset.filter(scan_id__in=scan_ids_list)
|
||||
scan_based_filters = {"id__in": scan_ids_list}
|
||||
category_scan_filters = {"scan_id__in": scan_ids_list}
|
||||
else:
|
||||
exact = query_params.get("filter[inserted_at]")
|
||||
gte = query_params.get("filter[inserted_at__gte]")
|
||||
@@ -2678,6 +2814,7 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
scan_based_filters = {
|
||||
key.lstrip("scan_"): value for key, value in date_filters.items()
|
||||
}
|
||||
category_scan_filters = date_filters
|
||||
|
||||
# ToRemove: Temporary fallback mechanism
|
||||
if not queryset.exists():
|
||||
@@ -2724,10 +2861,31 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
.order_by("resource_type")
|
||||
)
|
||||
|
||||
# Get categories from ScanCategorySummary using same scan filters
|
||||
categories = list(
|
||||
ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id, **category_scan_filters
|
||||
)
|
||||
.values_list("category", flat=True)
|
||||
.distinct()
|
||||
.order_by("category")
|
||||
)
|
||||
|
||||
# Fallback to finding aggregation if no ScanCategorySummary exists
|
||||
if not categories:
|
||||
categories_set = set()
|
||||
for categories_list in filtered_queryset.values_list(
|
||||
"categories", flat=True
|
||||
):
|
||||
if categories_list:
|
||||
categories_set.update(categories_list)
|
||||
categories = sorted(categories_set)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"resource_types": resource_types,
|
||||
"categories": categories,
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
@@ -2832,10 +2990,36 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
.order_by("resource_type")
|
||||
)
|
||||
|
||||
# Get categories from ScanCategorySummary for latest scans
|
||||
categories = list(
|
||||
ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id__in=latest_scans_queryset.values_list("id", flat=True),
|
||||
)
|
||||
.values_list("category", flat=True)
|
||||
.distinct()
|
||||
.order_by("category")
|
||||
)
|
||||
|
||||
# Fallback to finding aggregation if no ScanCategorySummary exists
|
||||
if not categories:
|
||||
filtered_queryset = self.filter_queryset(self.get_queryset()).filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id__in=latest_scans_queryset.values_list("id", flat=True),
|
||||
)
|
||||
categories_set = set()
|
||||
for categories_list in filtered_queryset.values_list(
|
||||
"categories", flat=True
|
||||
):
|
||||
if categories_list:
|
||||
categories_set.update(categories_list)
|
||||
categories = sorted(categories_set)
|
||||
|
||||
result = {
|
||||
"services": services,
|
||||
"regions": regions,
|
||||
"resource_types": resource_types,
|
||||
"categories": categories,
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
@@ -3422,7 +3606,19 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
def _get_compliance_template(self, *, provider=None, scan_id=None):
|
||||
"""Return the compliance template for the given provider or scan."""
|
||||
if provider is None and scan_id is not None:
|
||||
scan = Scan.all_objects.select_related("provider").get(pk=scan_id)
|
||||
try:
|
||||
scan = Scan.all_objects.select_related("provider").get(pk=scan_id)
|
||||
except Scan.DoesNotExist:
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "Scan not found",
|
||||
"status": 404,
|
||||
"source": {"pointer": "filter[scan_id]"},
|
||||
"code": "not_found",
|
||||
}
|
||||
]
|
||||
)
|
||||
provider = scan.provider
|
||||
|
||||
if not provider:
|
||||
@@ -3564,18 +3760,8 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
|
||||
def list(self, request, *args, **kwargs):
|
||||
scan_id = request.query_params.get("filter[scan_id]")
|
||||
if not scan_id:
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "This query parameter is required.",
|
||||
"status": 400,
|
||||
"source": {"pointer": "filter[scan_id]"},
|
||||
"code": "required",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Specific scan requested - use optimized summaries with region support
|
||||
region_filter = request.query_params.get(
|
||||
"filter[region]"
|
||||
) or request.query_params.get("filter[region__in]")
|
||||
@@ -3866,11 +4052,47 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
summary="Get findings data by service",
|
||||
description=(
|
||||
"Retrieve an aggregated summary of findings grouped by service. The response includes the total count "
|
||||
"of findings for each service, as long as there are at least one finding for that service. At least "
|
||||
"one of the `inserted_at` filters must be provided."
|
||||
"of findings for each service, as long as there are at least one finding for that service."
|
||||
),
|
||||
filters=True,
|
||||
),
|
||||
regions=extend_schema(
|
||||
summary="Get findings data by region",
|
||||
description=(
|
||||
"Retrieve an aggregated summary of findings grouped by region. The response includes the total, passed, "
|
||||
"failed, and muted findings for each region based on the latest completed scans per provider. "
|
||||
"Standard overview filters (inserted_at, provider filters, region filters, etc.) are supported."
|
||||
),
|
||||
filters=True,
|
||||
),
|
||||
findings_severity_timeseries=extend_schema(
|
||||
summary="Get findings severity data over time",
|
||||
description=(
|
||||
"Retrieve daily aggregated findings data grouped by severity levels over a date range. "
|
||||
"Returns one data point per day with counts of failed findings by severity (critical, high, "
|
||||
"medium, low, informational) and muted findings. Days without scans are filled forward with "
|
||||
"the most recent known values. Use date_from (required) and date_to filters to specify the range."
|
||||
),
|
||||
filters=True,
|
||||
),
|
||||
attack_surface=extend_schema(
|
||||
summary="Get attack surface overview",
|
||||
description="Retrieve aggregated attack surface metrics from latest completed scans per provider.",
|
||||
tags=["Overview"],
|
||||
filters=True,
|
||||
responses={200: AttackSurfaceOverviewSerializer(many=True)},
|
||||
),
|
||||
categories=extend_schema(
|
||||
summary="Get category overview",
|
||||
description=(
|
||||
"Retrieve aggregated category metrics from latest completed scans per provider. "
|
||||
"Returns one row per category with total, failed, and new failed findings counts, "
|
||||
"plus a severity breakdown showing failed findings per severity level. "
|
||||
),
|
||||
tags=["Overview"],
|
||||
filters=True,
|
||||
responses={200: CategoryOverviewSerializer(many=True)},
|
||||
),
|
||||
)
|
||||
@method_decorator(CACHE_DECORATOR, name="list")
|
||||
class OverviewViewSet(BaseRLSViewSet):
|
||||
@@ -3888,7 +4110,16 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
if not role.unlimited_visibility:
|
||||
self.allowed_providers = providers
|
||||
|
||||
return ScanSummary.all_objects.filter(tenant_id=self.request.tenant_id)
|
||||
tenant_id = self.request.tenant_id
|
||||
|
||||
# Return appropriate queryset per action
|
||||
if self.action == "findings_severity_timeseries":
|
||||
qs = DailySeveritySummary.objects.filter(tenant_id=tenant_id)
|
||||
if hasattr(self, "allowed_providers"):
|
||||
qs = qs.filter(provider_id__in=self.allowed_providers)
|
||||
return qs
|
||||
|
||||
return ScanSummary.all_objects.filter(tenant_id=tenant_id)
|
||||
|
||||
def get_serializer_class(self):
|
||||
if self.action == "providers":
|
||||
@@ -3899,23 +4130,43 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
return OverviewFindingSerializer
|
||||
elif self.action == "findings_severity":
|
||||
return OverviewSeveritySerializer
|
||||
elif self.action == "findings_severity_timeseries":
|
||||
return FindingsSeverityOverTimeSerializer
|
||||
elif self.action == "services":
|
||||
return OverviewServiceSerializer
|
||||
elif self.action == "regions":
|
||||
return OverviewRegionSerializer
|
||||
elif self.action == "threatscore":
|
||||
return ThreatScoreSnapshotSerializer
|
||||
elif self.action == "attack_surface":
|
||||
return AttackSurfaceOverviewSerializer
|
||||
elif self.action == "categories":
|
||||
return CategoryOverviewSerializer
|
||||
return super().get_serializer_class()
|
||||
|
||||
def get_filterset_class(self):
|
||||
if self.action == "providers":
|
||||
return None
|
||||
elif self.action == "findings":
|
||||
elif self.action in ["findings", "services", "regions"]:
|
||||
return ScanSummaryFilter
|
||||
elif self.action == "findings_severity":
|
||||
return ScanSummarySeverityFilter
|
||||
elif self.action == "services":
|
||||
return ServiceOverviewFilter
|
||||
elif self.action == "findings_severity_timeseries":
|
||||
return DailySeveritySummaryFilter
|
||||
elif self.action == "categories":
|
||||
return CategoryOverviewFilter
|
||||
elif self.action == "attack_surface":
|
||||
return AttackSurfaceOverviewFilter
|
||||
return None
|
||||
|
||||
def filter_queryset(self, queryset):
|
||||
# Skip OrderingFilter for findings_severity_timeseries (no inserted_at field)
|
||||
if self.action == "findings_severity_timeseries":
|
||||
return CustomDjangoFilterBackend().filter_queryset(
|
||||
self.request, queryset, self
|
||||
)
|
||||
return super().filter_queryset(queryset)
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
def list(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="GET")
|
||||
@@ -3924,6 +4175,109 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
def retrieve(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="GET")
|
||||
|
||||
def _get_latest_scans_queryset(self):
|
||||
"""
|
||||
Get filtered queryset for the latest completed scans per provider.
|
||||
|
||||
Returns:
|
||||
Filtered ScanSummary queryset with latest scan IDs applied.
|
||||
"""
|
||||
tenant_id = self.request.tenant_id
|
||||
queryset = self.get_queryset()
|
||||
filtered_queryset = self.filter_queryset(queryset)
|
||||
provider_filter = (
|
||||
{"provider__in": self.allowed_providers}
|
||||
if hasattr(self, "allowed_providers")
|
||||
else {}
|
||||
)
|
||||
|
||||
latest_scan_ids = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
|
||||
return filtered_queryset.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
|
||||
def _normalize_jsonapi_params(self, query_params, exclude_keys=None):
|
||||
"""Convert JSON:API filter params (filter[X]) to flat params (X)."""
|
||||
exclude_keys = exclude_keys or set()
|
||||
normalized = QueryDict(mutable=True)
|
||||
for key, values in query_params.lists():
|
||||
normalized_key = (
|
||||
key[7:-1] if key.startswith("filter[") and key.endswith("]") else key
|
||||
)
|
||||
if normalized_key not in exclude_keys:
|
||||
normalized.setlist(normalized_key, values)
|
||||
return normalized
|
||||
|
||||
def _ensure_allowed_providers(self):
|
||||
"""Populate allowed providers for RBAC-aware queries once per request."""
|
||||
if getattr(self, "_providers_initialized", False):
|
||||
return
|
||||
self.get_queryset()
|
||||
self._providers_initialized = True
|
||||
|
||||
def _get_provider_filter(self, provider_field="provider"):
|
||||
self._ensure_allowed_providers()
|
||||
if hasattr(self, "allowed_providers"):
|
||||
return {f"{provider_field}__in": self.allowed_providers}
|
||||
return {}
|
||||
|
||||
def _apply_provider_filter(self, queryset, provider_field="provider"):
|
||||
provider_filter = self._get_provider_filter(provider_field)
|
||||
if provider_filter:
|
||||
return queryset.filter(**provider_filter)
|
||||
return queryset
|
||||
|
||||
def _apply_filterset(self, queryset, filterset_class, exclude_keys=None):
|
||||
normalized_params = self._normalize_jsonapi_params(
|
||||
self.request.query_params, exclude_keys=set(exclude_keys or [])
|
||||
)
|
||||
filterset = filterset_class(normalized_params, queryset=queryset)
|
||||
return filterset.qs
|
||||
|
||||
def _latest_scan_ids_for_allowed_providers(self, tenant_id, provider_filters=None):
|
||||
provider_filter = self._get_provider_filter()
|
||||
queryset = Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
if provider_filters:
|
||||
queryset = queryset.filter(**provider_filters)
|
||||
return (
|
||||
queryset.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
|
||||
def _extract_provider_filters_from_params(self):
|
||||
"""Extract provider filters from query params to apply on Scan queryset."""
|
||||
params = self.request.query_params
|
||||
filters = {}
|
||||
|
||||
provider_id = params.get("filter[provider_id]")
|
||||
if provider_id:
|
||||
filters["provider_id"] = provider_id
|
||||
|
||||
provider_id_in = params.get("filter[provider_id__in]")
|
||||
if provider_id_in:
|
||||
filters["provider_id__in"] = provider_id_in.split(",")
|
||||
|
||||
provider_type = params.get("filter[provider_type]")
|
||||
if provider_type:
|
||||
filters["provider__provider"] = provider_type
|
||||
|
||||
provider_type_in = params.get("filter[provider_type__in]")
|
||||
if provider_type_in:
|
||||
filters["provider__provider__in"] = provider_type_in.split(",")
|
||||
|
||||
return filters
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="providers")
|
||||
def providers(self, request):
|
||||
tenant_id = self.request.tenant_id
|
||||
@@ -4016,26 +4370,7 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="findings")
|
||||
def findings(self, request):
|
||||
tenant_id = self.request.tenant_id
|
||||
queryset = self.get_queryset()
|
||||
filtered_queryset = self.filter_queryset(queryset)
|
||||
provider_filter = (
|
||||
{"provider__in": self.allowed_providers}
|
||||
if hasattr(self, "allowed_providers")
|
||||
else {}
|
||||
)
|
||||
|
||||
latest_scan_ids = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
filtered_queryset = filtered_queryset.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
filtered_queryset = self._get_latest_scans_queryset()
|
||||
|
||||
aggregated_totals = filtered_queryset.aggregate(
|
||||
_pass=Sum("_pass") or 0,
|
||||
@@ -4062,37 +4397,14 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="findings_severity")
|
||||
def findings_severity(self, request):
|
||||
tenant_id = self.request.tenant_id
|
||||
|
||||
# Load only required fields
|
||||
queryset = self.get_queryset().only(
|
||||
"tenant_id", "scan_id", "severity", "fail", "_pass", "total"
|
||||
)
|
||||
|
||||
filtered_queryset = self.filter_queryset(queryset)
|
||||
provider_filter = (
|
||||
{"provider__in": self.allowed_providers}
|
||||
if hasattr(self, "allowed_providers")
|
||||
else {}
|
||||
)
|
||||
|
||||
latest_scan_ids = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
filtered_queryset = filtered_queryset.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
filtered_queryset = self._get_latest_scans_queryset()
|
||||
|
||||
# The filter will have added a status_count annotation if any status filter was used
|
||||
if "status_count" in filtered_queryset.query.annotations:
|
||||
sum_expression = Sum("status_count")
|
||||
else:
|
||||
sum_expression = Sum("total")
|
||||
# Exclude muted findings by default
|
||||
sum_expression = Sum(F("_pass") + F("fail"))
|
||||
|
||||
severity_counts = (
|
||||
filtered_queryset.values("severity")
|
||||
@@ -4110,26 +4422,7 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="services")
|
||||
def services(self, request):
|
||||
tenant_id = self.request.tenant_id
|
||||
queryset = self.get_queryset()
|
||||
filtered_queryset = self.filter_queryset(queryset)
|
||||
provider_filter = (
|
||||
{"provider__in": self.allowed_providers}
|
||||
if hasattr(self, "allowed_providers")
|
||||
else {}
|
||||
)
|
||||
|
||||
latest_scan_ids = (
|
||||
Scan.all_objects.filter(
|
||||
tenant_id=tenant_id, state=StateChoices.COMPLETED, **provider_filter
|
||||
)
|
||||
.order_by("provider_id", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
filtered_queryset = filtered_queryset.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
filtered_queryset = self._get_latest_scans_queryset()
|
||||
|
||||
services_data = (
|
||||
filtered_queryset.values("service")
|
||||
@@ -4144,13 +4437,133 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="regions")
|
||||
def regions(self, request):
|
||||
filtered_queryset = self._get_latest_scans_queryset()
|
||||
|
||||
regions_data = (
|
||||
filtered_queryset.annotate(provider_type=F("scan__provider__provider"))
|
||||
.values("provider_type", "region")
|
||||
.annotate(_pass=Sum("_pass"))
|
||||
.annotate(fail=Sum("fail"))
|
||||
.annotate(muted=Sum("muted"))
|
||||
.annotate(total=Sum("total"))
|
||||
.order_by("provider_type", "region")
|
||||
)
|
||||
|
||||
serializer = self.get_serializer(regions_data, many=True)
|
||||
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@action(
|
||||
detail=False,
|
||||
methods=["get"],
|
||||
url_path="findings_severity/timeseries",
|
||||
url_name="findings_severity_timeseries",
|
||||
)
|
||||
def findings_severity_timeseries(self, request):
|
||||
"""
|
||||
Daily severity trends for charts. Uses DailySeveritySummary pre-aggregation.
|
||||
Requires date_from filter.
|
||||
"""
|
||||
# Get queryset with RBAC, provider, and date filters applied
|
||||
# Date validation is handled by DailySeveritySummaryFilter
|
||||
daily_qs = self.filter_queryset(self.get_queryset())
|
||||
|
||||
date_from = request._date_from
|
||||
date_to = request._date_to
|
||||
|
||||
if not daily_qs.exists():
|
||||
# No data matches filters - return zeros
|
||||
result = self._generate_zero_result(date_from, date_to)
|
||||
serializer = self.get_serializer(result, many=True)
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
# Fetch all data for fill-forward logic
|
||||
daily_summaries = list(
|
||||
daily_qs.order_by("provider_id", "-date").values(
|
||||
"provider_id",
|
||||
"scan_id",
|
||||
"date",
|
||||
"critical",
|
||||
"high",
|
||||
"medium",
|
||||
"low",
|
||||
"informational",
|
||||
"muted",
|
||||
)
|
||||
)
|
||||
|
||||
if not daily_summaries:
|
||||
result = self._generate_zero_result(date_from, date_to)
|
||||
serializer = self.get_serializer(result, many=True)
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
# Build provider_data: {provider_id: [(date, data), ...]} sorted by date desc
|
||||
provider_data = defaultdict(list)
|
||||
for summary in daily_summaries:
|
||||
provider_data[summary["provider_id"]].append(summary)
|
||||
|
||||
# For each day, find the latest data per provider and sum values
|
||||
result = []
|
||||
current_date = date_from
|
||||
while current_date <= date_to:
|
||||
day_totals = {
|
||||
"critical": 0,
|
||||
"high": 0,
|
||||
"medium": 0,
|
||||
"low": 0,
|
||||
"informational": 0,
|
||||
"muted": 0,
|
||||
}
|
||||
day_scan_ids = []
|
||||
|
||||
for provider_id, summaries in provider_data.items():
|
||||
# Find the latest data for this provider <= current_date
|
||||
for summary in summaries: # Already sorted by date desc
|
||||
if summary["date"] <= current_date:
|
||||
day_totals["critical"] += summary["critical"] or 0
|
||||
day_totals["high"] += summary["high"] or 0
|
||||
day_totals["medium"] += summary["medium"] or 0
|
||||
day_totals["low"] += summary["low"] or 0
|
||||
day_totals["informational"] += summary["informational"] or 0
|
||||
day_totals["muted"] += summary["muted"] or 0
|
||||
day_scan_ids.append(summary["scan_id"])
|
||||
break # Found the latest data for this provider
|
||||
|
||||
result.append(
|
||||
{"date": current_date, "scan_ids": day_scan_ids, **day_totals}
|
||||
)
|
||||
current_date += timedelta(days=1)
|
||||
|
||||
serializer = self.get_serializer(result, many=True)
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
def _generate_zero_result(self, date_from, date_to):
|
||||
"""Generate a list of zero-filled results for each date in range."""
|
||||
result = []
|
||||
current_date = date_from
|
||||
zero_values = {
|
||||
"critical": 0,
|
||||
"high": 0,
|
||||
"medium": 0,
|
||||
"low": 0,
|
||||
"informational": 0,
|
||||
"muted": 0,
|
||||
"scan_ids": [],
|
||||
}
|
||||
while current_date <= date_to:
|
||||
result.append({"date": current_date, **zero_values})
|
||||
current_date += timedelta(days=1)
|
||||
return result
|
||||
|
||||
@extend_schema(
|
||||
summary="Get ThreatScore snapshots",
|
||||
description=(
|
||||
"Retrieve ThreatScore metrics. By default, returns the latest snapshot for each provider. "
|
||||
"Use snapshot_id to retrieve a specific historical snapshot."
|
||||
),
|
||||
tags=["Overviews"],
|
||||
tags=["Overview"],
|
||||
parameters=[
|
||||
OpenApiParameter(
|
||||
name="snapshot_id",
|
||||
@@ -4196,11 +4609,9 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
snapshot_id = request.query_params.get("snapshot_id")
|
||||
|
||||
# Base queryset with RLS
|
||||
base_queryset = ThreatScoreSnapshot.objects.filter(tenant_id=tenant_id)
|
||||
|
||||
# Apply RBAC filtering
|
||||
if hasattr(self, "allowed_providers"):
|
||||
base_queryset = base_queryset.filter(provider__in=self.allowed_providers)
|
||||
base_queryset = self._apply_provider_filter(
|
||||
ThreatScoreSnapshot.objects.filter(tenant_id=tenant_id)
|
||||
)
|
||||
|
||||
# Case 1: Specific snapshot requested
|
||||
if snapshot_id:
|
||||
@@ -4216,17 +4627,9 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
# Case 2: Latest snapshot per provider (default)
|
||||
# Apply filters manually: this @action is outside the standard list endpoint flow,
|
||||
# so DRF's filter backends don't execute and we must flatten JSON:API params ourselves.
|
||||
normalized_params = QueryDict(mutable=True)
|
||||
for param_key, values in request.query_params.lists():
|
||||
normalized_key = param_key
|
||||
if param_key.startswith("filter[") and param_key.endswith("]"):
|
||||
normalized_key = param_key[7:-1]
|
||||
if normalized_key == "snapshot_id":
|
||||
continue
|
||||
normalized_params.setlist(normalized_key, values)
|
||||
|
||||
filterset = ThreatScoreSnapshotFilter(normalized_params, queryset=base_queryset)
|
||||
filtered_queryset = filterset.qs
|
||||
filtered_queryset = self._apply_filterset(
|
||||
base_queryset, ThreatScoreSnapshotFilter, exclude_keys={"snapshot_id"}
|
||||
)
|
||||
|
||||
# Get distinct provider IDs from filtered queryset
|
||||
# Pick the latest snapshot per provider using Postgres DISTINCT ON pattern.
|
||||
@@ -4470,6 +4873,117 @@ class OverviewViewSet(BaseRLSViewSet):
|
||||
|
||||
return aggregated_snapshot
|
||||
|
||||
@action(
|
||||
detail=False,
|
||||
methods=["get"],
|
||||
url_name="attack-surface",
|
||||
url_path="attack-surfaces",
|
||||
)
|
||||
def attack_surface(self, request):
|
||||
tenant_id = request.tenant_id
|
||||
latest_scan_ids = self._latest_scan_ids_for_allowed_providers(tenant_id)
|
||||
|
||||
base_queryset = AttackSurfaceOverview.objects.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
filtered_queryset = self._apply_filterset(
|
||||
base_queryset, AttackSurfaceOverviewFilter
|
||||
)
|
||||
|
||||
aggregation = filtered_queryset.values("attack_surface_type").annotate(
|
||||
total_findings=Coalesce(Sum("total_findings"), 0),
|
||||
failed_findings=Coalesce(Sum("failed_findings"), 0),
|
||||
muted_failed_findings=Coalesce(Sum("muted_failed_findings"), 0),
|
||||
)
|
||||
|
||||
results = {
|
||||
attack_surface_type: {
|
||||
"total_findings": 0,
|
||||
"failed_findings": 0,
|
||||
"muted_failed_findings": 0,
|
||||
}
|
||||
for attack_surface_type in AttackSurfaceOverview.AttackSurfaceTypeChoices.values
|
||||
}
|
||||
for item in aggregation:
|
||||
results[item["attack_surface_type"]] = {
|
||||
"total_findings": item["total_findings"],
|
||||
"failed_findings": item["failed_findings"],
|
||||
"muted_failed_findings": item["muted_failed_findings"],
|
||||
}
|
||||
|
||||
response_data = [
|
||||
{"attack_surface_type": key, **value} for key, value in results.items()
|
||||
]
|
||||
|
||||
return Response(
|
||||
self.get_serializer(response_data, many=True).data,
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="categories")
|
||||
def categories(self, request):
|
||||
tenant_id = request.tenant_id
|
||||
provider_filters = self._extract_provider_filters_from_params()
|
||||
latest_scan_ids = self._latest_scan_ids_for_allowed_providers(
|
||||
tenant_id, provider_filters
|
||||
)
|
||||
|
||||
base_queryset = ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id__in=latest_scan_ids
|
||||
)
|
||||
provider_filter_keys = {
|
||||
"provider_id",
|
||||
"provider_id__in",
|
||||
"provider_type",
|
||||
"provider_type__in",
|
||||
}
|
||||
filtered_queryset = self._apply_filterset(
|
||||
base_queryset, CategoryOverviewFilter, exclude_keys=provider_filter_keys
|
||||
)
|
||||
|
||||
aggregation = (
|
||||
filtered_queryset.values("category", "severity")
|
||||
.annotate(
|
||||
total=Coalesce(Sum("total_findings"), 0),
|
||||
failed=Coalesce(Sum("failed_findings"), 0),
|
||||
new_failed=Coalesce(Sum("new_failed_findings"), 0),
|
||||
)
|
||||
.order_by("category", "severity")
|
||||
)
|
||||
|
||||
category_data = defaultdict(
|
||||
lambda: {
|
||||
"total_findings": 0,
|
||||
"failed_findings": 0,
|
||||
"new_failed_findings": 0,
|
||||
"severity": {
|
||||
"informational": 0,
|
||||
"low": 0,
|
||||
"medium": 0,
|
||||
"high": 0,
|
||||
"critical": 0,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
for row in aggregation:
|
||||
cat = row["category"]
|
||||
sev = row["severity"]
|
||||
category_data[cat]["total_findings"] += row["total"]
|
||||
category_data[cat]["failed_findings"] += row["failed"]
|
||||
category_data[cat]["new_failed_findings"] += row["new_failed"]
|
||||
if sev in category_data[cat]["severity"]:
|
||||
category_data[cat]["severity"][sev] = row["failed"]
|
||||
|
||||
response_data = [
|
||||
{"category": cat, **data} for cat, data in sorted(category_data.items())
|
||||
]
|
||||
|
||||
return Response(
|
||||
self.get_serializer(response_data, many=True).data,
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["Schedule"])
|
||||
@extend_schema_view(
|
||||
|
||||
@@ -36,6 +36,14 @@ DATABASES = {
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
"admin_replica": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"NAME": env("POSTGRES_REPLICA_DB", default=default_db_name),
|
||||
"USER": env("POSTGRES_ADMIN_USER", default="prowler"),
|
||||
"PASSWORD": env("POSTGRES_ADMIN_PASSWORD", default="S3cret"),
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
@@ -37,6 +37,14 @@ DATABASES = {
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
"admin_replica": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"NAME": env("POSTGRES_REPLICA_DB", default=default_db_name),
|
||||
"USER": env("POSTGRES_ADMIN_USER"),
|
||||
"PASSWORD": env("POSTGRES_ADMIN_PASSWORD"),
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
@@ -5,6 +5,9 @@ IGNORED_EXCEPTIONS = [
|
||||
# Provider is not connected due to credentials errors
|
||||
"is not connected",
|
||||
"ProviderConnectionError",
|
||||
# Provider was deleted during a scan
|
||||
"ProviderDeletedException",
|
||||
"violates foreign key constraint",
|
||||
# Authentication Errors from AWS
|
||||
"InvalidToken",
|
||||
"AccessDeniedException",
|
||||
|
||||
+168
-10
@@ -11,10 +11,14 @@ from django.urls import reverse
|
||||
from django_celery_results.models import TaskResult
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
from tasks.jobs.backfill import backfill_resource_scan_summaries
|
||||
from tasks.jobs.backfill import (
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
)
|
||||
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
AttackSurfaceOverview,
|
||||
ComplianceOverview,
|
||||
ComplianceRequirementOverview,
|
||||
Finding,
|
||||
@@ -35,6 +39,7 @@ from api.models import (
|
||||
SAMLConfiguration,
|
||||
SAMLDomainIndex,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
@@ -512,6 +517,12 @@ def providers_fixture(tenants_fixture):
|
||||
alias="mongodbatlas_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider9 = Provider.objects.create(
|
||||
provider="alibabacloud",
|
||||
uid="1234567890123456",
|
||||
alias="alibabacloud_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
return (
|
||||
provider1,
|
||||
@@ -522,6 +533,7 @@ def providers_fixture(tenants_fixture):
|
||||
provider6,
|
||||
provider7,
|
||||
provider8,
|
||||
provider9,
|
||||
)
|
||||
|
||||
|
||||
@@ -1108,8 +1120,8 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
|
||||
region="region1",
|
||||
_pass=1,
|
||||
fail=0,
|
||||
muted=0,
|
||||
total=1,
|
||||
muted=2,
|
||||
total=3,
|
||||
new=1,
|
||||
changed=0,
|
||||
unchanged=0,
|
||||
@@ -1117,7 +1129,7 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
|
||||
fail_changed=0,
|
||||
pass_new=1,
|
||||
pass_changed=0,
|
||||
muted_new=0,
|
||||
muted_new=2,
|
||||
muted_changed=0,
|
||||
scan=scan,
|
||||
)
|
||||
@@ -1130,8 +1142,8 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
|
||||
region="region2",
|
||||
_pass=0,
|
||||
fail=1,
|
||||
muted=1,
|
||||
total=2,
|
||||
muted=3,
|
||||
total=4,
|
||||
new=2,
|
||||
changed=0,
|
||||
unchanged=0,
|
||||
@@ -1139,7 +1151,7 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
|
||||
fail_changed=0,
|
||||
pass_new=0,
|
||||
pass_changed=0,
|
||||
muted_new=1,
|
||||
muted_new=3,
|
||||
muted_changed=0,
|
||||
scan=scan,
|
||||
)
|
||||
@@ -1152,8 +1164,8 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
|
||||
region="region1",
|
||||
_pass=1,
|
||||
fail=0,
|
||||
muted=0,
|
||||
total=1,
|
||||
muted=1,
|
||||
total=2,
|
||||
new=1,
|
||||
changed=0,
|
||||
unchanged=0,
|
||||
@@ -1161,7 +1173,7 @@ def scan_summaries_fixture(tenants_fixture, providers_fixture):
|
||||
fail_changed=0,
|
||||
pass_new=1,
|
||||
pass_changed=0,
|
||||
muted_new=0,
|
||||
muted_new=1,
|
||||
muted_changed=0,
|
||||
scan=scan,
|
||||
)
|
||||
@@ -1270,6 +1282,113 @@ def latest_scan_finding(authenticated_client, providers_fixture, resources_fixtu
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_categories(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource = resources_fixture[0]
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_with_categories_1",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="genai_check",
|
||||
check_metadata={"CheckId": "genai_check"},
|
||||
categories=["gen-ai", "security"],
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_multiple_categories(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource1, resource2 = resources_fixture[:2]
|
||||
|
||||
finding1 = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_multi_cat_1",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="genai_check",
|
||||
check_metadata={"CheckId": "genai_check"},
|
||||
categories=["gen-ai", "security"],
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding1.add_resources([resource1])
|
||||
|
||||
finding2 = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_multi_cat_2",
|
||||
scan=scan,
|
||||
delta=None,
|
||||
status=Status.FAIL,
|
||||
status_extended="test status 2",
|
||||
impact=Severity.high,
|
||||
impact_extended="test impact 2",
|
||||
severity=Severity.high,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="iam_check",
|
||||
check_metadata={"CheckId": "iam_check"},
|
||||
categories=["iam", "security"],
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding2.add_resources([resource2])
|
||||
|
||||
backfill_resource_scan_summaries(str(scan.tenant_id), str(scan.id))
|
||||
return finding1, finding2
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def latest_scan_finding_with_categories(
|
||||
authenticated_client, providers_fixture, resources_fixture
|
||||
):
|
||||
provider = providers_fixture[0]
|
||||
tenant_id = str(providers_fixture[0].tenant_id)
|
||||
resource = resources_fixture[0]
|
||||
scan = Scan.objects.create(
|
||||
name="latest completed scan with categories",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.COMPLETED,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
uid="latest_finding_with_categories",
|
||||
scan=scan,
|
||||
delta="new",
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="genai_iam_check",
|
||||
check_metadata={"CheckId": "genai_iam_check"},
|
||||
categories=["gen-ai", "iam"],
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
backfill_resource_scan_summaries(tenant_id, str(scan.id))
|
||||
backfill_scan_category_summaries(tenant_id, str(scan.id))
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def latest_scan_resource(authenticated_client, providers_fixture):
|
||||
provider = providers_fixture[0]
|
||||
@@ -1469,6 +1588,45 @@ def mute_rules_fixture(tenants_fixture, create_test_user, findings_fixture):
|
||||
return mute_rule1, mute_rule2
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_attack_surface_overview():
|
||||
def _create(tenant, scan, attack_surface_type, total=10, failed=5, muted_failed=2):
|
||||
return AttackSurfaceOverview.objects.create(
|
||||
tenant=tenant,
|
||||
scan=scan,
|
||||
attack_surface_type=attack_surface_type,
|
||||
total_findings=total,
|
||||
failed_findings=failed,
|
||||
muted_failed_findings=muted_failed,
|
||||
)
|
||||
|
||||
return _create
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_scan_category_summary():
|
||||
def _create(
|
||||
tenant,
|
||||
scan,
|
||||
category,
|
||||
severity,
|
||||
total_findings=10,
|
||||
failed_findings=5,
|
||||
new_failed_findings=2,
|
||||
):
|
||||
return ScanCategorySummary.objects.create(
|
||||
tenant=tenant,
|
||||
scan=scan,
|
||||
category=category,
|
||||
severity=severity,
|
||||
total_findings=total_findings,
|
||||
failed_findings=failed_findings,
|
||||
new_failed_findings=new_failed_findings,
|
||||
)
|
||||
|
||||
return _create
|
||||
|
||||
|
||||
def get_authorization_header(access_token: str) -> dict:
|
||||
return {"Authorization": f"Bearer {access_token}"}
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 95 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 94 KiB |
@@ -61,4 +61,5 @@ def schedule_provider_scan(provider_instance: Provider):
|
||||
"tenant_id": str(provider_instance.tenant_id),
|
||||
"provider_id": provider_id,
|
||||
},
|
||||
countdown=5, # Avoid race conditions between the worker and the database
|
||||
)
|
||||
|
||||
@@ -1,14 +1,23 @@
|
||||
from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
|
||||
from django.db.models import Sum
|
||||
from django.utils import timezone
|
||||
from tasks.jobs.scan import aggregate_category_counts
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
ComplianceOverviewSummary,
|
||||
ComplianceRequirementOverview,
|
||||
DailySeveritySummary,
|
||||
Finding,
|
||||
Resource,
|
||||
ResourceFindingMapping,
|
||||
ResourceScanSummary,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
)
|
||||
|
||||
@@ -175,3 +184,160 @@ def backfill_compliance_summaries(tenant_id: str, scan_id: str):
|
||||
)
|
||||
|
||||
return {"status": "backfilled", "inserted": len(summary_objects)}
|
||||
|
||||
|
||||
def backfill_daily_severity_summaries(tenant_id: str, days: int = None):
|
||||
"""
|
||||
Backfill DailySeveritySummary from completed scans.
|
||||
Groups by provider+date, keeps latest scan per day.
|
||||
"""
|
||||
created_count = 0
|
||||
updated_count = 0
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
scan_filter = {
|
||||
"tenant_id": tenant_id,
|
||||
"state": StateChoices.COMPLETED,
|
||||
"completed_at__isnull": False,
|
||||
}
|
||||
|
||||
if days is not None:
|
||||
cutoff_date = timezone.now() - timedelta(days=days)
|
||||
scan_filter["completed_at__gte"] = cutoff_date
|
||||
|
||||
completed_scans = (
|
||||
Scan.objects.filter(**scan_filter)
|
||||
.order_by("provider_id", "-completed_at")
|
||||
.values("id", "provider_id", "completed_at")
|
||||
)
|
||||
|
||||
if not completed_scans:
|
||||
return {"status": "no scans to backfill"}
|
||||
|
||||
# Keep only latest scan per provider/day
|
||||
latest_scans_by_day = {}
|
||||
for scan in completed_scans:
|
||||
key = (scan["provider_id"], scan["completed_at"].date())
|
||||
if key not in latest_scans_by_day:
|
||||
latest_scans_by_day[key] = scan
|
||||
|
||||
# Process each provider/day
|
||||
for (provider_id, scan_date), scan in latest_scans_by_day.items():
|
||||
scan_id = scan["id"]
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
severity_totals = (
|
||||
ScanSummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
)
|
||||
.values("severity")
|
||||
.annotate(total_fail=Sum("fail"), total_muted=Sum("muted"))
|
||||
)
|
||||
|
||||
severity_data = {
|
||||
"critical": 0,
|
||||
"high": 0,
|
||||
"medium": 0,
|
||||
"low": 0,
|
||||
"informational": 0,
|
||||
"muted": 0,
|
||||
}
|
||||
|
||||
for row in severity_totals:
|
||||
severity = row["severity"]
|
||||
if severity in severity_data:
|
||||
severity_data[severity] = row["total_fail"] or 0
|
||||
severity_data["muted"] += row["total_muted"] or 0
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
_, created = DailySeveritySummary.objects.update_or_create(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
date=scan_date,
|
||||
defaults={
|
||||
"scan_id": scan_id,
|
||||
"critical": severity_data["critical"],
|
||||
"high": severity_data["high"],
|
||||
"medium": severity_data["medium"],
|
||||
"low": severity_data["low"],
|
||||
"informational": severity_data["informational"],
|
||||
"muted": severity_data["muted"],
|
||||
},
|
||||
)
|
||||
|
||||
if created:
|
||||
created_count += 1
|
||||
else:
|
||||
updated_count += 1
|
||||
|
||||
return {
|
||||
"status": "backfilled",
|
||||
"created": created_count,
|
||||
"updated": updated_count,
|
||||
"total_days": len(latest_scans_by_day),
|
||||
}
|
||||
|
||||
|
||||
def backfill_scan_category_summaries(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanCategorySummary for a completed scan.
|
||||
|
||||
Aggregates category counts from all findings in the scan and creates
|
||||
one ScanCategorySummary row per (category, severity) combination.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID
|
||||
scan_id: Scan UUID to backfill
|
||||
|
||||
Returns:
|
||||
dict: Status indicating whether backfill was performed
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
if ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).exists():
|
||||
return {"status": "already backfilled"}
|
||||
|
||||
if not Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id=scan_id,
|
||||
state__in=(StateChoices.COMPLETED, StateChoices.FAILED),
|
||||
).exists():
|
||||
return {"status": "scan is not completed"}
|
||||
|
||||
category_counts: dict[tuple[str, str], dict[str, int]] = {}
|
||||
for finding in Finding.all_objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
).values("categories", "severity", "status", "delta", "muted"):
|
||||
aggregate_category_counts(
|
||||
categories=finding.get("categories") or [],
|
||||
severity=finding.get("severity"),
|
||||
status=finding.get("status"),
|
||||
delta=finding.get("delta"),
|
||||
muted=finding.get("muted", False),
|
||||
cache=category_counts,
|
||||
)
|
||||
|
||||
if not category_counts:
|
||||
return {"status": "no categories to backfill"}
|
||||
|
||||
category_summaries = [
|
||||
ScanCategorySummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
category=category,
|
||||
severity=severity,
|
||||
total_findings=counts["total"],
|
||||
failed_findings=counts["failed"],
|
||||
new_failed_findings=counts["new_failed"],
|
||||
)
|
||||
for (category, severity), counts in category_counts.items()
|
||||
]
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
ScanCategorySummary.objects.bulk_create(
|
||||
category_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
|
||||
return {"status": "backfilled", "categories_count": len(category_counts)}
|
||||
|
||||
@@ -15,6 +15,7 @@ from prowler.config.config import (
|
||||
html_file_suffix,
|
||||
json_asff_file_suffix,
|
||||
json_ocsf_file_suffix,
|
||||
set_output_timestamp,
|
||||
)
|
||||
from prowler.lib.outputs.asff.asff import ASFF
|
||||
from prowler.lib.outputs.compliance.aws_well_architected.aws_well_architected import (
|
||||
@@ -26,6 +27,7 @@ from prowler.lib.outputs.compliance.c5.c5_gcp import GCPC5
|
||||
from prowler.lib.outputs.compliance.ccc.ccc_aws import CCC_AWS
|
||||
from prowler.lib.outputs.compliance.ccc.ccc_azure import CCC_Azure
|
||||
from prowler.lib.outputs.compliance.ccc.ccc_gcp import CCC_GCP
|
||||
from prowler.lib.outputs.compliance.cis.cis_alibabacloud import AlibabaCloudCIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_aws import AWSCIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_azure import AzureCIS
|
||||
from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS
|
||||
@@ -49,6 +51,9 @@ from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_azure import (
|
||||
AzureMitreAttack,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.mitre_attack.mitre_attack_gcp import GCPMitreAttack
|
||||
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_alibaba import (
|
||||
ProwlerThreatScoreAlibaba,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_aws import (
|
||||
ProwlerThreatScoreAWS,
|
||||
)
|
||||
@@ -58,6 +63,9 @@ from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_azur
|
||||
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_gcp import (
|
||||
ProwlerThreatScoreGCP,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_kubernetes import (
|
||||
ProwlerThreatScoreKubernetes,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.prowler_threatscore.prowler_threatscore_m365 import (
|
||||
ProwlerThreatScoreM365,
|
||||
)
|
||||
@@ -104,6 +112,10 @@ COMPLIANCE_CLASS_MAP = {
|
||||
"kubernetes": [
|
||||
(lambda name: name.startswith("cis_"), KubernetesCIS),
|
||||
(lambda name: name.startswith("iso27001_"), KubernetesISO27001),
|
||||
(
|
||||
lambda name: name == "prowler_threatscore_kubernetes",
|
||||
ProwlerThreatScoreKubernetes,
|
||||
),
|
||||
],
|
||||
"m365": [
|
||||
(lambda name: name.startswith("cis_"), M365CIS),
|
||||
@@ -120,6 +132,13 @@ COMPLIANCE_CLASS_MAP = {
|
||||
"oraclecloud": [
|
||||
(lambda name: name.startswith("cis_"), OracleCloudCIS),
|
||||
],
|
||||
"alibabacloud": [
|
||||
(lambda name: name.startswith("cis_"), AlibabaCloudCIS),
|
||||
(
|
||||
lambda name: name == "prowler_threatscore_alibabacloud",
|
||||
ProwlerThreatScoreAlibaba,
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -234,36 +253,33 @@ def _upload_to_s3(
|
||||
logger.error(f"S3 upload failed: {str(e)}")
|
||||
|
||||
|
||||
def _generate_output_directory(
|
||||
output_directory, prowler_provider: object, tenant_id: str, scan_id: str
|
||||
) -> tuple[str, str, str]:
|
||||
def _build_output_path(
|
||||
output_directory: str,
|
||||
prowler_provider: str,
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
subdirectory: str = None,
|
||||
) -> str:
|
||||
"""
|
||||
Generate a file system path for the output directory of a prowler scan.
|
||||
|
||||
This function constructs the output directory path by combining a base
|
||||
temporary output directory, the tenant ID, the scan ID, and details about
|
||||
the prowler provider along with a timestamp. The resulting path is used to
|
||||
store the output files of a prowler scan.
|
||||
|
||||
Note:
|
||||
This function depends on one external variable:
|
||||
- `output_file_timestamp`: A timestamp (as a string) used to uniquely identify the output.
|
||||
Build a file system path for the output directory of a prowler scan.
|
||||
|
||||
Args:
|
||||
output_directory (str): The base output directory.
|
||||
prowler_provider (object): An identifier or descriptor for the prowler provider.
|
||||
Typically, this is a string indicating the provider (e.g., "aws").
|
||||
prowler_provider (str): An identifier or descriptor for the prowler provider.
|
||||
Typically, this is a string indicating the provider (e.g., "aws").
|
||||
tenant_id (str): The unique identifier for the tenant.
|
||||
scan_id (str): The unique identifier for the scan.
|
||||
subdirectory (str, optional): Optional subdirectory to include in the path
|
||||
(e.g., "compliance", "threatscore", "ens").
|
||||
|
||||
Returns:
|
||||
str: The constructed file system path for the prowler scan output directory.
|
||||
str: The constructed path with directory created.
|
||||
|
||||
Example:
|
||||
>>> _generate_output_directory("/tmp", "aws", "tenant-1234", "scan-5678")
|
||||
'/tmp/tenant-1234/aws/scan-5678/prowler-output-2023-02-15T12:34:56',
|
||||
'/tmp/tenant-1234/aws/scan-5678/compliance/prowler-output-2023-02-15T12:34:56'
|
||||
'/tmp/tenant-1234/aws/scan-5678/threatscore/prowler-output-2023-02-15T12:34:56'
|
||||
>>> _build_output_path("/tmp", "aws", "tenant-1234", "scan-5678")
|
||||
'/tmp/tenant-1234/scan-5678/prowler-output-aws-20230215123456'
|
||||
>>> _build_output_path("/tmp", "aws", "tenant-1234", "scan-5678", "threatscore")
|
||||
'/tmp/tenant-1234/scan-5678/threatscore/prowler-output-aws-20230215123456'
|
||||
"""
|
||||
# Sanitize the prowler provider name to ensure it is a valid directory name
|
||||
prowler_provider_sanitized = re.sub(r"[^\w\-]", "-", prowler_provider)
|
||||
@@ -271,23 +287,107 @@ def _generate_output_directory(
|
||||
with rls_transaction(tenant_id):
|
||||
started_at = Scan.objects.get(id=scan_id).started_at
|
||||
|
||||
set_output_timestamp(started_at)
|
||||
|
||||
timestamp = started_at.strftime("%Y%m%d%H%M%S")
|
||||
path = (
|
||||
f"{output_directory}/{tenant_id}/{scan_id}/prowler-output-"
|
||||
f"{prowler_provider_sanitized}-{timestamp}"
|
||||
)
|
||||
|
||||
if subdirectory:
|
||||
path = (
|
||||
f"{output_directory}/{tenant_id}/{scan_id}/{subdirectory}/prowler-output-"
|
||||
f"{prowler_provider_sanitized}-{timestamp}"
|
||||
)
|
||||
else:
|
||||
path = (
|
||||
f"{output_directory}/{tenant_id}/{scan_id}/prowler-output-"
|
||||
f"{prowler_provider_sanitized}-{timestamp}"
|
||||
)
|
||||
|
||||
# Create directory for the path if it doesn't exist
|
||||
os.makedirs("/".join(path.split("/")[:-1]), exist_ok=True)
|
||||
|
||||
compliance_path = (
|
||||
f"{output_directory}/{tenant_id}/{scan_id}/compliance/prowler-output-"
|
||||
f"{prowler_provider_sanitized}-{timestamp}"
|
||||
)
|
||||
os.makedirs("/".join(compliance_path.split("/")[:-1]), exist_ok=True)
|
||||
return path
|
||||
|
||||
threatscore_path = (
|
||||
f"{output_directory}/{tenant_id}/{scan_id}/threatscore/prowler-output-"
|
||||
f"{prowler_provider_sanitized}-{timestamp}"
|
||||
)
|
||||
os.makedirs("/".join(threatscore_path.split("/")[:-1]), exist_ok=True)
|
||||
|
||||
return path, compliance_path, threatscore_path
|
||||
def _generate_compliance_output_directory(
|
||||
output_directory: str,
|
||||
prowler_provider: str,
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
compliance_framework: str,
|
||||
) -> str:
|
||||
"""
|
||||
Generate a file system path for a compliance framework output directory.
|
||||
|
||||
This function constructs the output directory path specifically for a compliance
|
||||
framework (e.g., "threatscore", "ens") by combining a base temporary output directory,
|
||||
the tenant ID, the scan ID, the compliance framework name, and details about the
|
||||
prowler provider along with a timestamp.
|
||||
|
||||
Args:
|
||||
output_directory (str): The base output directory.
|
||||
prowler_provider (str): An identifier or descriptor for the prowler provider.
|
||||
Typically, this is a string indicating the provider (e.g., "aws").
|
||||
tenant_id (str): The unique identifier for the tenant.
|
||||
scan_id (str): The unique identifier for the scan.
|
||||
compliance_framework (str): The compliance framework name (e.g., "threatscore", "ens").
|
||||
|
||||
Returns:
|
||||
str: The path for the compliance framework output directory.
|
||||
|
||||
Example:
|
||||
>>> _generate_compliance_output_directory("/tmp", "aws", "tenant-1234", "scan-5678", "threatscore")
|
||||
'/tmp/tenant-1234/scan-5678/threatscore/prowler-output-aws-20230215123456'
|
||||
>>> _generate_compliance_output_directory("/tmp", "aws", "tenant-1234", "scan-5678", "ens")
|
||||
'/tmp/tenant-1234/scan-5678/ens/prowler-output-aws-20230215123456'
|
||||
>>> _generate_compliance_output_directory("/tmp", "aws", "tenant-1234", "scan-5678", "nis2")
|
||||
'/tmp/tenant-1234/scan-5678/nis2/prowler-output-aws-20230215123456'
|
||||
"""
|
||||
return _build_output_path(
|
||||
output_directory,
|
||||
prowler_provider,
|
||||
tenant_id,
|
||||
scan_id,
|
||||
subdirectory=compliance_framework,
|
||||
)
|
||||
|
||||
|
||||
def _generate_output_directory(
|
||||
output_directory: str,
|
||||
prowler_provider: str,
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
) -> tuple[str, str]:
|
||||
"""
|
||||
Generate file system paths for the standard and compliance output directories of a prowler scan.
|
||||
|
||||
This function constructs both the standard output directory path and the compliance
|
||||
output directory path by combining a base temporary output directory, the tenant ID,
|
||||
the scan ID, and details about the prowler provider along with a timestamp.
|
||||
|
||||
Args:
|
||||
output_directory (str): The base output directory.
|
||||
prowler_provider (str): An identifier or descriptor for the prowler provider.
|
||||
Typically, this is a string indicating the provider (e.g., "aws").
|
||||
tenant_id (str): The unique identifier for the tenant.
|
||||
scan_id (str): The unique identifier for the scan.
|
||||
|
||||
Returns:
|
||||
tuple[str, str]: A tuple containing (standard_path, compliance_path).
|
||||
|
||||
Example:
|
||||
>>> _generate_output_directory("/tmp", "aws", "tenant-1234", "scan-5678")
|
||||
('/tmp/tenant-1234/scan-5678/prowler-output-aws-20230215123456',
|
||||
'/tmp/tenant-1234/scan-5678/compliance/prowler-output-aws-20230215123456')
|
||||
"""
|
||||
standard_path = _build_output_path(
|
||||
output_directory, prowler_provider, tenant_id, scan_id
|
||||
)
|
||||
compliance_path = _build_output_path(
|
||||
output_directory,
|
||||
prowler_provider,
|
||||
tenant_id,
|
||||
scan_id,
|
||||
subdirectory="compliance",
|
||||
)
|
||||
|
||||
return standard_path, compliance_path
|
||||
|
||||
@@ -19,6 +19,9 @@ from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.aws.lib.s3.s3 import S3
|
||||
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub
|
||||
from prowler.providers.common.models import Connection
|
||||
from prowler.providers.aws.lib.security_hub.exceptions.exceptions import (
|
||||
SecurityHubNoEnabledRegionsError,
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@@ -222,8 +225,9 @@ def get_security_hub_client_from_integration(
|
||||
)
|
||||
return True, security_hub
|
||||
else:
|
||||
# Reset regions information if connection fails
|
||||
# Reset regions information if connection fails and integration is not connected
|
||||
with rls_transaction(tenant_id, using=MainRouter.default_db):
|
||||
integration.connected = False
|
||||
integration.configuration["regions"] = {}
|
||||
integration.save()
|
||||
|
||||
@@ -330,15 +334,18 @@ def upload_security_hub_integration(
|
||||
)
|
||||
|
||||
if not connected:
|
||||
logger.error(
|
||||
f"Security Hub connection failed for integration {integration.id}: "
|
||||
f"{security_hub.error}"
|
||||
)
|
||||
with rls_transaction(
|
||||
tenant_id, using=MainRouter.default_db
|
||||
if isinstance(
|
||||
security_hub.error,
|
||||
SecurityHubNoEnabledRegionsError,
|
||||
):
|
||||
integration.connected = False
|
||||
integration.save()
|
||||
logger.warning(
|
||||
f"Security Hub integration {integration.id} has no enabled regions"
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"Security Hub connection failed for integration {integration.id}: "
|
||||
f"{security_hub.error}"
|
||||
)
|
||||
break # Skip this integration
|
||||
|
||||
security_hub_client = security_hub
|
||||
@@ -409,22 +416,16 @@ def upload_security_hub_integration(
|
||||
logger.warning(
|
||||
f"Failed to archive previous findings: {str(archive_error)}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Security Hub integration {integration.id} failed: {str(e)}"
|
||||
)
|
||||
continue
|
||||
|
||||
result = integration_executions == len(integrations)
|
||||
if result:
|
||||
logger.info(
|
||||
f"All Security Hub integrations completed successfully for provider {provider_id}"
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"Some Security Hub integrations failed for provider {provider_id}"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ from typing import Dict
|
||||
|
||||
import boto3
|
||||
import openai
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
@@ -9,6 +11,74 @@ from api.models import LighthouseProviderConfiguration, LighthouseProviderModels
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# OpenAI model prefixes to exclude from Lighthouse model selection.
|
||||
# These models don't support text chat completions and tool calling.
|
||||
EXCLUDED_OPENAI_MODEL_PREFIXES = (
|
||||
"dall-e", # Image generation
|
||||
"whisper", # Audio transcription
|
||||
"tts-", # Text-to-speech (tts-1, tts-1-hd, etc.)
|
||||
"sora", # Text-to-video (sora-2, sora-2-pro, etc.)
|
||||
"text-embedding", # Embeddings
|
||||
"embedding", # Embeddings (alternative naming)
|
||||
"text-moderation", # Content moderation
|
||||
"omni-moderation", # Content moderation
|
||||
"text-davinci", # Legacy completion models
|
||||
"text-curie", # Legacy completion models
|
||||
"text-babbage", # Legacy completion models
|
||||
"text-ada", # Legacy completion models
|
||||
"davinci", # Legacy completion models
|
||||
"curie", # Legacy completion models
|
||||
"babbage", # Legacy completion models
|
||||
"ada", # Legacy completion models
|
||||
"computer-use", # Computer control agent
|
||||
"gpt-image", # Image generation
|
||||
"gpt-audio", # Audio models
|
||||
"gpt-realtime", # Realtime voice API
|
||||
)
|
||||
|
||||
# OpenAI model substrings to exclude (patterns that can appear anywhere in model ID).
|
||||
# These patterns identify non-chat model variants.
|
||||
EXCLUDED_OPENAI_MODEL_SUBSTRINGS = (
|
||||
"-audio-", # Audio preview models (gpt-4o-audio-preview, etc.)
|
||||
"-realtime-", # Realtime preview models (gpt-4o-realtime-preview, etc.)
|
||||
"-transcribe", # Transcription models (gpt-4o-transcribe, etc.)
|
||||
"-tts", # TTS models (gpt-4o-mini-tts)
|
||||
"-instruct", # Legacy instruct models (gpt-3.5-turbo-instruct, etc.)
|
||||
)
|
||||
|
||||
|
||||
def _extract_error_message(e: Exception) -> str:
|
||||
"""
|
||||
Extract a user-friendly error message from various exception types.
|
||||
|
||||
This function handles exceptions from different providers (OpenAI, AWS Bedrock)
|
||||
and extracts the most relevant error message for display to users.
|
||||
|
||||
Args:
|
||||
e: The exception to extract a message from.
|
||||
|
||||
Returns:
|
||||
str: A user-friendly error message.
|
||||
"""
|
||||
# For OpenAI SDK errors (>= v1.0)
|
||||
# OpenAI exceptions have a 'body' attribute with error details
|
||||
if hasattr(e, "body") and isinstance(e.body, dict):
|
||||
if "message" in e.body:
|
||||
return e.body["message"]
|
||||
# Sometimes nested under 'error' key
|
||||
if "error" in e.body and isinstance(e.body["error"], dict):
|
||||
return e.body["error"].get("message", str(e))
|
||||
|
||||
# For boto3 ClientError
|
||||
# Boto3 exceptions have a 'response' attribute with error details
|
||||
if hasattr(e, "response") and isinstance(e.response, dict):
|
||||
error_info = e.response.get("Error", {})
|
||||
if error_info.get("Message"):
|
||||
return error_info["Message"]
|
||||
|
||||
# Fallback to string representation for unknown error types
|
||||
return str(e)
|
||||
|
||||
|
||||
def _extract_openai_api_key(
|
||||
provider_cfg: LighthouseProviderConfiguration,
|
||||
@@ -56,21 +126,39 @@ def _extract_bedrock_credentials(
|
||||
"""
|
||||
Safely extract AWS Bedrock credentials from a provider configuration.
|
||||
|
||||
Supports two authentication methods:
|
||||
1. AWS access key + secret key + region
|
||||
2. Bedrock API key (bearer token) + region
|
||||
|
||||
Args:
|
||||
provider_cfg (LighthouseProviderConfiguration): The provider configuration instance
|
||||
containing the credentials.
|
||||
|
||||
Returns:
|
||||
Dict[str, str] | None: Dictionary with 'access_key_id', 'secret_access_key', and
|
||||
'region' if present and valid, otherwise None.
|
||||
Dict[str, str] | None: Dictionary with either:
|
||||
- 'access_key_id', 'secret_access_key', and 'region' for access key auth
|
||||
- 'api_key' and 'region' for API key (bearer token) auth
|
||||
Returns None if credentials are invalid or missing.
|
||||
"""
|
||||
creds = provider_cfg.credentials_decoded
|
||||
if not isinstance(creds, dict):
|
||||
return None
|
||||
|
||||
region = creds.get("region")
|
||||
if not isinstance(region, str) or not region:
|
||||
return None
|
||||
|
||||
# Check for API key authentication first
|
||||
api_key = creds.get("api_key")
|
||||
if isinstance(api_key, str) and api_key:
|
||||
return {
|
||||
"api_key": api_key,
|
||||
"region": region,
|
||||
}
|
||||
|
||||
# Fall back to access key authentication
|
||||
access_key_id = creds.get("access_key_id")
|
||||
secret_access_key = creds.get("secret_access_key")
|
||||
region = creds.get("region")
|
||||
|
||||
# Validate all required fields are present and are strings
|
||||
if (
|
||||
@@ -78,8 +166,6 @@ def _extract_bedrock_credentials(
|
||||
or not access_key_id
|
||||
or not isinstance(secret_access_key, str)
|
||||
or not secret_access_key
|
||||
or not isinstance(region, str)
|
||||
or not region
|
||||
):
|
||||
return None
|
||||
|
||||
@@ -90,6 +176,51 @@ def _extract_bedrock_credentials(
|
||||
}
|
||||
|
||||
|
||||
def _create_bedrock_client(
|
||||
bedrock_creds: Dict[str, str], service_name: str = "bedrock"
|
||||
):
|
||||
"""
|
||||
Create a boto3 Bedrock client with the appropriate authentication method.
|
||||
|
||||
Supports two authentication methods:
|
||||
1. API key (bearer token) - uses unsigned requests with Authorization header
|
||||
2. AWS access key + secret key - uses standard SigV4 signing
|
||||
|
||||
Args:
|
||||
bedrock_creds: Dictionary with either:
|
||||
- 'api_key' and 'region' for API key (bearer token) auth
|
||||
- 'access_key_id', 'secret_access_key', and 'region' for access key auth
|
||||
service_name: The Bedrock service name. Use 'bedrock' for control plane
|
||||
operations (list_foundation_models, etc.) or 'bedrock-runtime' for
|
||||
inference operations.
|
||||
|
||||
Returns:
|
||||
boto3 client configured for the specified Bedrock service.
|
||||
"""
|
||||
region = bedrock_creds["region"]
|
||||
|
||||
if "api_key" in bedrock_creds:
|
||||
bearer_token = bedrock_creds["api_key"]
|
||||
client = boto3.client(
|
||||
service_name=service_name,
|
||||
region_name=region,
|
||||
config=Config(signature_version=UNSIGNED),
|
||||
)
|
||||
|
||||
def inject_bearer_token(request, **kwargs):
|
||||
request.headers["Authorization"] = f"Bearer {bearer_token}"
|
||||
|
||||
client.meta.events.register("before-send.*.*", inject_bearer_token)
|
||||
return client
|
||||
|
||||
return boto3.client(
|
||||
service_name=service_name,
|
||||
region_name=region,
|
||||
aws_access_key_id=bedrock_creds["access_key_id"],
|
||||
aws_secret_access_key=bedrock_creds["secret_access_key"],
|
||||
)
|
||||
|
||||
|
||||
def check_lighthouse_provider_connection(provider_config_id: str) -> Dict:
|
||||
"""
|
||||
Validate a Lighthouse provider configuration by calling the provider API and
|
||||
@@ -141,12 +272,7 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> Dict:
|
||||
}
|
||||
|
||||
# Test connection by listing foundation models
|
||||
bedrock_client = boto3.client(
|
||||
"bedrock",
|
||||
aws_access_key_id=bedrock_creds["access_key_id"],
|
||||
aws_secret_access_key=bedrock_creds["secret_access_key"],
|
||||
region_name=bedrock_creds["region"],
|
||||
)
|
||||
bedrock_client = _create_bedrock_client(bedrock_creds)
|
||||
_ = bedrock_client.list_foundation_models()
|
||||
|
||||
elif (
|
||||
@@ -179,32 +305,54 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> Dict:
|
||||
return {"connected": True, "error": None}
|
||||
|
||||
except Exception as e:
|
||||
error_message = _extract_error_message(e)
|
||||
logger.warning(
|
||||
"%s connection check failed: %s", provider_cfg.provider_type, str(e)
|
||||
"%s connection check failed: %s", provider_cfg.provider_type, error_message
|
||||
)
|
||||
provider_cfg.is_active = False
|
||||
provider_cfg.save()
|
||||
return {"connected": False, "error": str(e)}
|
||||
return {"connected": False, "error": error_message}
|
||||
|
||||
|
||||
def _fetch_openai_models(api_key: str) -> Dict[str, str]:
|
||||
"""
|
||||
Fetch available models from OpenAI API.
|
||||
|
||||
Filters out models that don't support text input/output and tool calling,
|
||||
such as image generation (DALL-E), audio transcription (Whisper),
|
||||
text-to-speech (TTS), embeddings, and moderation models.
|
||||
|
||||
Args:
|
||||
api_key: OpenAI API key for authentication.
|
||||
|
||||
Returns:
|
||||
Dict mapping model_id to model_name. For OpenAI, both are the same
|
||||
as the API doesn't provide separate display names.
|
||||
as the API doesn't provide separate display names. Only includes
|
||||
models that support text input, text output or tool calling.
|
||||
|
||||
Raises:
|
||||
Exception: If the API call fails.
|
||||
"""
|
||||
client = openai.OpenAI(api_key=api_key)
|
||||
models = client.models.list()
|
||||
# OpenAI uses model.id for both ID and display name
|
||||
return {m.id: m.id for m in getattr(models, "data", [])}
|
||||
|
||||
# Filter models to only include those supporting chat completions + tool calling
|
||||
filtered_models = {}
|
||||
for model in getattr(models, "data", []):
|
||||
model_id = model.id
|
||||
|
||||
# Skip if model ID starts with excluded prefixes
|
||||
if model_id.startswith(EXCLUDED_OPENAI_MODEL_PREFIXES):
|
||||
continue
|
||||
|
||||
# Skip if model ID contains excluded substrings
|
||||
if any(substring in model_id for substring in EXCLUDED_OPENAI_MODEL_SUBSTRINGS):
|
||||
continue
|
||||
|
||||
# Include model (supports chat completions + tool calling)
|
||||
filtered_models[model_id] = model_id
|
||||
|
||||
return filtered_models
|
||||
|
||||
|
||||
def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, str]:
|
||||
@@ -232,105 +380,219 @@ def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, st
|
||||
return available_models
|
||||
|
||||
|
||||
def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]:
|
||||
def _get_region_prefix(region: str) -> str:
|
||||
"""
|
||||
Fetch available models from AWS Bedrock with entitlement verification.
|
||||
Determine geographic prefix for AWS region.
|
||||
|
||||
This function:
|
||||
1. Lists foundation models with TEXT modality support
|
||||
2. Lists inference profiles with TEXT modality support
|
||||
3. Verifies user has entitlement access to each model
|
||||
Examples: ap-south-1 -> apac, us-east-1 -> us, eu-west-1 -> eu
|
||||
"""
|
||||
if region.startswith(("us-", "ca-", "sa-")):
|
||||
return "us"
|
||||
elif region.startswith("eu-"):
|
||||
return "eu"
|
||||
elif region.startswith("ap-"):
|
||||
return "apac"
|
||||
return "global"
|
||||
|
||||
Args:
|
||||
bedrock_creds: Dictionary with 'access_key_id', 'secret_access_key', and 'region'.
|
||||
|
||||
def _clean_inference_profile_name(profile_name: str) -> str:
|
||||
"""
|
||||
Remove geographic prefix from inference profile name.
|
||||
|
||||
AWS includes geographic prefixes in profile names which are redundant
|
||||
since the profile ID already contains this information.
|
||||
|
||||
Examples:
|
||||
"APAC Anthropic Claude 3.5 Sonnet" -> "Anthropic Claude 3.5 Sonnet"
|
||||
"GLOBAL Claude Sonnet 4.5" -> "Claude Sonnet 4.5"
|
||||
"US Anthropic Claude 3 Haiku" -> "Anthropic Claude 3 Haiku"
|
||||
"""
|
||||
prefixes = ["APAC ", "GLOBAL ", "US ", "EU ", "APAC-", "GLOBAL-", "US-", "EU-"]
|
||||
|
||||
for prefix in prefixes:
|
||||
if profile_name.upper().startswith(prefix.upper()):
|
||||
return profile_name[len(prefix) :].strip()
|
||||
|
||||
return profile_name
|
||||
|
||||
|
||||
def _supports_text_modality(input_modalities: list, output_modalities: list) -> bool:
|
||||
"""Check if model supports TEXT for both input and output."""
|
||||
return "TEXT" in input_modalities and "TEXT" in output_modalities
|
||||
|
||||
|
||||
def _get_foundation_model_modalities(
|
||||
bedrock_client, model_id: str
|
||||
) -> tuple[list, list] | None:
|
||||
"""
|
||||
Fetch input and output modalities for a foundation model.
|
||||
|
||||
Returns:
|
||||
Dict mapping model_id to model_name for all accessible models.
|
||||
|
||||
Raises:
|
||||
BotoCoreError, ClientError: If AWS API calls fail.
|
||||
(input_modalities, output_modalities) or None if fetch fails
|
||||
"""
|
||||
bedrock_client = boto3.client(
|
||||
"bedrock",
|
||||
aws_access_key_id=bedrock_creds["access_key_id"],
|
||||
aws_secret_access_key=bedrock_creds["secret_access_key"],
|
||||
region_name=bedrock_creds["region"],
|
||||
)
|
||||
try:
|
||||
model_info = bedrock_client.get_foundation_model(modelIdentifier=model_id)
|
||||
model_details = model_info.get("modelDetails", {})
|
||||
input_mods = model_details.get("inputModalities", [])
|
||||
output_mods = model_details.get("outputModalities", [])
|
||||
return (input_mods, output_mods)
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
logger.debug("Could not fetch model details for %s: %s", model_id, str(e))
|
||||
return None
|
||||
|
||||
models_to_check: Dict[str, str] = {}
|
||||
|
||||
# Step 1: Get foundation models with TEXT modality
|
||||
def _extract_foundation_model_ids(profile_models: list) -> list[str]:
|
||||
"""
|
||||
Extract foundation model IDs from inference profile model ARNs.
|
||||
|
||||
Args:
|
||||
profile_models: List of model references from inference profile
|
||||
|
||||
Returns:
|
||||
List of foundation model IDs extracted from ARNs
|
||||
"""
|
||||
model_ids = []
|
||||
for model_ref in profile_models:
|
||||
model_arn = model_ref.get("modelArn", "")
|
||||
if "foundation-model/" in model_arn:
|
||||
model_id = model_arn.split("foundation-model/")[1]
|
||||
model_ids.append(model_id)
|
||||
return model_ids
|
||||
|
||||
|
||||
def _build_inference_profile_map(
|
||||
bedrock_client, region: str
|
||||
) -> Dict[str, tuple[str, str]]:
|
||||
"""
|
||||
Build map of foundation_model_id -> best inference profile.
|
||||
|
||||
Returns:
|
||||
Dict mapping foundation_model_id to (profile_id, profile_name)
|
||||
Only includes profiles with TEXT modality support
|
||||
Prefers region-matched profiles over others
|
||||
"""
|
||||
region_prefix = _get_region_prefix(region)
|
||||
model_to_profile: Dict[str, tuple[str, str]] = {}
|
||||
|
||||
try:
|
||||
response = bedrock_client.list_inference_profiles()
|
||||
profiles = response.get("inferenceProfileSummaries", [])
|
||||
|
||||
for profile in profiles:
|
||||
profile_id = profile.get("inferenceProfileId")
|
||||
profile_name = profile.get("inferenceProfileName")
|
||||
|
||||
if not profile_id or not profile_name:
|
||||
continue
|
||||
|
||||
profile_models = profile.get("models", [])
|
||||
if not profile_models:
|
||||
continue
|
||||
|
||||
foundation_model_ids = _extract_foundation_model_ids(profile_models)
|
||||
if not foundation_model_ids:
|
||||
continue
|
||||
|
||||
modalities = _get_foundation_model_modalities(
|
||||
bedrock_client, foundation_model_ids[0]
|
||||
)
|
||||
if not modalities:
|
||||
continue
|
||||
|
||||
input_mods, output_mods = modalities
|
||||
if not _supports_text_modality(input_mods, output_mods):
|
||||
continue
|
||||
|
||||
is_preferred = profile_id.startswith(f"{region_prefix}.")
|
||||
clean_name = _clean_inference_profile_name(profile_name)
|
||||
|
||||
for foundation_model_id in foundation_model_ids:
|
||||
if foundation_model_id not in model_to_profile:
|
||||
model_to_profile[foundation_model_id] = (profile_id, clean_name)
|
||||
elif is_preferred and not model_to_profile[foundation_model_id][
|
||||
0
|
||||
].startswith(f"{region_prefix}."):
|
||||
model_to_profile[foundation_model_id] = (profile_id, clean_name)
|
||||
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
logger.info("Could not fetch inference profiles in %s: %s", region, str(e))
|
||||
|
||||
return model_to_profile
|
||||
|
||||
|
||||
def _check_on_demand_availability(bedrock_client, model_id: str) -> bool:
|
||||
"""Check if an ON_DEMAND foundation model is entitled and available."""
|
||||
try:
|
||||
availability = bedrock_client.get_foundation_model_availability(
|
||||
modelId=model_id
|
||||
)
|
||||
entitlement = availability.get("entitlementAvailability")
|
||||
return entitlement == "AVAILABLE"
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
logger.debug("Could not check availability for %s: %s", model_id, str(e))
|
||||
return False
|
||||
|
||||
|
||||
def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]:
|
||||
"""
|
||||
Fetch available models from AWS Bedrock, preferring inference profiles over ON_DEMAND.
|
||||
|
||||
Strategy:
|
||||
1. Build map of foundation_model -> best_inference_profile (with TEXT validation)
|
||||
2. For each TEXT-capable foundation model:
|
||||
- Use inference profile ID if available (preferred - better throughput)
|
||||
- Fallback to foundation model ID if only ON_DEMAND available
|
||||
3. Verify entitlement for ON_DEMAND models
|
||||
|
||||
Args:
|
||||
bedrock_creds: Dict with 'region' and auth credentials
|
||||
|
||||
Returns:
|
||||
Dict mapping model_id to model_name. IDs can be:
|
||||
- Inference profile IDs (e.g., "apac.anthropic.claude-3-5-sonnet-20240620-v1:0")
|
||||
- Foundation model IDs (e.g., "anthropic.claude-3-5-sonnet-20240620-v1:0")
|
||||
"""
|
||||
bedrock_client = _create_bedrock_client(bedrock_creds)
|
||||
region = bedrock_creds["region"]
|
||||
|
||||
model_to_profile = _build_inference_profile_map(bedrock_client, region)
|
||||
|
||||
foundation_response = bedrock_client.list_foundation_models()
|
||||
model_summaries = foundation_response.get("modelSummaries", [])
|
||||
|
||||
for model in model_summaries:
|
||||
# Check if model supports TEXT input and output modality
|
||||
input_modalities = model.get("inputModalities", [])
|
||||
output_modalities = model.get("outputModalities", [])
|
||||
models_to_return: Dict[str, str] = {}
|
||||
on_demand_models: set[str] = set()
|
||||
|
||||
if "TEXT" not in input_modalities or "TEXT" not in output_modalities:
|
||||
for model in model_summaries:
|
||||
input_mods = model.get("inputModalities", [])
|
||||
output_mods = model.get("outputModalities", [])
|
||||
|
||||
if not _supports_text_modality(input_mods, output_mods):
|
||||
continue
|
||||
|
||||
model_id = model.get("modelId")
|
||||
if not model_id:
|
||||
model_name = model.get("modelName")
|
||||
|
||||
if not model_id or not model_name:
|
||||
continue
|
||||
|
||||
inference_types = model.get("inferenceTypesSupported", [])
|
||||
if model_id in model_to_profile:
|
||||
profile_id, profile_name = model_to_profile[model_id]
|
||||
models_to_return[profile_id] = profile_name
|
||||
else:
|
||||
inference_types = model.get("inferenceTypesSupported", [])
|
||||
if "ON_DEMAND" in inference_types:
|
||||
models_to_return[model_id] = model_name
|
||||
on_demand_models.add(model_id)
|
||||
|
||||
# Only include models with ON_DEMAND inference support
|
||||
if "ON_DEMAND" in inference_types:
|
||||
models_to_check[model_id] = model["modelName"]
|
||||
|
||||
# Step 2: Get inference profiles
|
||||
try:
|
||||
inference_profiles_response = bedrock_client.list_inference_profiles()
|
||||
inference_profiles = inference_profiles_response.get(
|
||||
"inferenceProfileSummaries", []
|
||||
)
|
||||
|
||||
for profile in inference_profiles:
|
||||
# Check if profile supports TEXT modality
|
||||
input_modalities = profile.get("inputModalities", [])
|
||||
output_modalities = profile.get("outputModalities", [])
|
||||
|
||||
if "TEXT" not in input_modalities or "TEXT" not in output_modalities:
|
||||
continue
|
||||
|
||||
profile_id = profile.get("inferenceProfileId")
|
||||
if profile_id:
|
||||
models_to_check[profile_id] = profile["inferenceProfileName"]
|
||||
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
logger.info(
|
||||
"Could not fetch inference profiles in %s: %s",
|
||||
bedrock_creds["region"],
|
||||
str(e),
|
||||
)
|
||||
|
||||
# Step 3: Verify entitlement availability for each model
|
||||
available_models: Dict[str, str] = {}
|
||||
|
||||
for model_id, model_name in models_to_check.items():
|
||||
try:
|
||||
availability = bedrock_client.get_foundation_model_availability(
|
||||
modelId=model_id
|
||||
)
|
||||
|
||||
entitlement = availability.get("entitlementAvailability")
|
||||
|
||||
# Only include models user has access to
|
||||
if entitlement == "AVAILABLE":
|
||||
for model_id, model_name in models_to_return.items():
|
||||
if model_id in on_demand_models:
|
||||
if _check_on_demand_availability(bedrock_client, model_id):
|
||||
available_models[model_id] = model_name
|
||||
else:
|
||||
logger.debug(
|
||||
"Skipping model %s - entitlement status: %s", model_id, entitlement
|
||||
)
|
||||
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
logger.debug(
|
||||
"Could not check availability for model %s: %s", model_id, str(e)
|
||||
)
|
||||
continue
|
||||
else:
|
||||
available_models[model_id] = model_name
|
||||
|
||||
return available_models
|
||||
|
||||
@@ -359,7 +621,6 @@ def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict:
|
||||
provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id)
|
||||
fetched_models: Dict[str, str] = {}
|
||||
|
||||
# Fetch models from the appropriate provider
|
||||
try:
|
||||
if (
|
||||
provider_cfg.provider_type
|
||||
@@ -414,12 +675,13 @@ def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict:
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_message = _extract_error_message(e)
|
||||
logger.warning(
|
||||
"Unexpected error refreshing %s models: %s",
|
||||
provider_cfg.provider_type,
|
||||
str(e),
|
||||
error_message,
|
||||
)
|
||||
return {"created": 0, "updated": 0, "deleted": 0, "error": str(e)}
|
||||
return {"created": 0, "updated": 0, "deleted": 0, "error": error_message}
|
||||
|
||||
# Upsert models into the catalog
|
||||
created = 0
|
||||
|
||||
+2940
-333
File diff suppressed because it is too large
Load Diff
@@ -8,11 +8,12 @@ from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import sentry_sdk
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.env import env
|
||||
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
|
||||
from django.db import IntegrityError, OperationalError
|
||||
from django.db.models import Case, Count, IntegerField, Prefetch, Sum, When
|
||||
from django.db.models import Case, Count, IntegerField, Prefetch, Q, Sum, When
|
||||
from tasks.utils import CustomEncoder
|
||||
|
||||
from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE
|
||||
@@ -26,8 +27,10 @@ from api.db_utils import (
|
||||
)
|
||||
from api.exceptions import ProviderConnectionError
|
||||
from api.models import (
|
||||
AttackSurfaceOverview,
|
||||
ComplianceOverviewSummary,
|
||||
ComplianceRequirementOverview,
|
||||
DailySeveritySummary,
|
||||
Finding,
|
||||
MuteRule,
|
||||
Processor,
|
||||
@@ -37,12 +40,14 @@ from api.models import (
|
||||
ResourceScanSummary,
|
||||
ResourceTag,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
)
|
||||
from api.models import StatusChoices as FindingStatus
|
||||
from api.utils import initialize_prowler_provider, return_prowler_provider
|
||||
from api.v1.serializers import ScanTaskSerializer
|
||||
from prowler.lib.check.models import CheckMetadata
|
||||
from prowler.lib.outputs.finding import Finding as ProwlerFinding
|
||||
from prowler.lib.scan.scan import Scan as ProwlerScan
|
||||
|
||||
@@ -74,6 +79,77 @@ FINDINGS_MICRO_BATCH_SIZE = env.int("DJANGO_FINDINGS_MICRO_BATCH_SIZE", default=
|
||||
# Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres
|
||||
SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=500)
|
||||
|
||||
ATTACK_SURFACE_PROVIDER_COMPATIBILITY = {
|
||||
"internet-exposed": None, # Compatible with all providers
|
||||
"secrets": None, # Compatible with all providers
|
||||
"privilege-escalation": ["aws", "kubernetes"],
|
||||
"ec2-imdsv1": ["aws"],
|
||||
}
|
||||
|
||||
_ATTACK_SURFACE_MAPPING_CACHE: dict[str, dict] = {}
|
||||
|
||||
|
||||
def aggregate_category_counts(
|
||||
categories: list[str],
|
||||
severity: str,
|
||||
status: str,
|
||||
delta: str | None,
|
||||
muted: bool,
|
||||
cache: dict[tuple[str, str], dict[str, int]],
|
||||
) -> None:
|
||||
"""
|
||||
Increment category counters in-place for a finding.
|
||||
|
||||
Args:
|
||||
categories: List of categories from finding metadata.
|
||||
severity: Severity level (e.g., "high", "medium").
|
||||
status: Finding status as string ("FAIL", "PASS").
|
||||
delta: Delta value as string ("new", "changed") or None.
|
||||
muted: Whether the finding is muted.
|
||||
cache: Dict {(category, severity): {"total", "failed", "new_failed"}} to update.
|
||||
"""
|
||||
is_failed = status == "FAIL" and not muted
|
||||
is_new_failed = is_failed and delta == "new"
|
||||
|
||||
for cat in categories:
|
||||
key = (cat, severity)
|
||||
if key not in cache:
|
||||
cache[key] = {"total": 0, "failed": 0, "new_failed": 0}
|
||||
if not muted:
|
||||
cache[key]["total"] += 1
|
||||
if is_failed:
|
||||
cache[key]["failed"] += 1
|
||||
if is_new_failed:
|
||||
cache[key]["new_failed"] += 1
|
||||
|
||||
|
||||
def _get_attack_surface_mapping_from_provider(provider_type: str) -> dict:
|
||||
global _ATTACK_SURFACE_MAPPING_CACHE
|
||||
|
||||
if provider_type in _ATTACK_SURFACE_MAPPING_CACHE:
|
||||
return _ATTACK_SURFACE_MAPPING_CACHE[provider_type]
|
||||
|
||||
attack_surface_check_mappings = {
|
||||
"internet-exposed": None,
|
||||
"secrets": None,
|
||||
"privilege-escalation": {
|
||||
"iam_policy_allows_privilege_escalation",
|
||||
"iam_inline_policy_allows_privilege_escalation",
|
||||
},
|
||||
"ec2-imdsv1": {
|
||||
"ec2_instance_imdsv2_enabled"
|
||||
}, # AWS only - IMDSv1 enabled findings
|
||||
}
|
||||
for category_name, check_ids in attack_surface_check_mappings.items():
|
||||
if check_ids is None:
|
||||
sdk_check_ids = CheckMetadata.list(
|
||||
provider=provider_type, category=category_name
|
||||
)
|
||||
attack_surface_check_mappings[category_name] = sdk_check_ids
|
||||
|
||||
_ATTACK_SURFACE_MAPPING_CACHE[provider_type] = attack_surface_check_mappings
|
||||
return attack_surface_check_mappings
|
||||
|
||||
|
||||
def _create_finding_delta(
|
||||
last_status: FindingStatus | None | str, new_status: FindingStatus | None
|
||||
@@ -330,7 +406,7 @@ def _create_compliance_summaries(
|
||||
if summary_objects:
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceOverviewSummary.objects.bulk_create(
|
||||
summary_objects, batch_size=500
|
||||
summary_objects, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
|
||||
|
||||
@@ -357,6 +433,7 @@ def _process_finding_micro_batch(
|
||||
unique_resources: set,
|
||||
scan_resource_cache: set,
|
||||
mute_rules_cache: dict,
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]],
|
||||
) -> None:
|
||||
"""
|
||||
Process a micro-batch of findings and persist them using bulk operations.
|
||||
@@ -377,6 +454,7 @@ def _process_finding_micro_batch(
|
||||
unique_resources: Set tracking (uid, region) pairs seen in the scan.
|
||||
scan_resource_cache: Set of tuples used to create `ResourceScanSummary` rows.
|
||||
mute_rules_cache: Map of finding UID -> mute reason gathered before the scan.
|
||||
scan_categories_cache: Dict tracking category counts {(category, severity): {"total", "failed", "new_failed"}}.
|
||||
"""
|
||||
# Accumulate objects for bulk operations
|
||||
findings_to_create = []
|
||||
@@ -532,11 +610,12 @@ def _process_finding_micro_batch(
|
||||
resource_failed_findings_cache[resource_uid] += 1
|
||||
|
||||
# Create finding object (don't save yet)
|
||||
check_metadata = finding.get_metadata()
|
||||
finding_instance = Finding(
|
||||
tenant_id=tenant_id,
|
||||
uid=finding_uid,
|
||||
delta=delta,
|
||||
check_metadata=finding.get_metadata(),
|
||||
check_metadata=check_metadata,
|
||||
status=status,
|
||||
status_extended=finding.status_extended,
|
||||
severity=finding.severity,
|
||||
@@ -549,6 +628,7 @@ def _process_finding_micro_batch(
|
||||
muted_at=datetime.now(tz=timezone.utc) if is_muted else None,
|
||||
muted_reason=muted_reason,
|
||||
compliance=finding.compliance,
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
)
|
||||
findings_to_create.append(finding_instance)
|
||||
resource_denormalized_data.append((finding_instance, resource_instance))
|
||||
@@ -563,6 +643,16 @@ def _process_finding_micro_batch(
|
||||
)
|
||||
)
|
||||
|
||||
# Track categories with counts for ScanCategorySummary by (category, severity)
|
||||
aggregate_category_counts(
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
cache=scan_categories_cache,
|
||||
)
|
||||
|
||||
# Bulk operations within single transaction
|
||||
with rls_transaction(tenant_id):
|
||||
# Bulk create findings
|
||||
@@ -662,6 +752,7 @@ def perform_prowler_scan(
|
||||
exception = None
|
||||
unique_resources = set()
|
||||
scan_resource_cache: set[tuple[str, str, str, str]] = set()
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
start_time = time.time()
|
||||
exc = None
|
||||
|
||||
@@ -751,6 +842,7 @@ def perform_prowler_scan(
|
||||
unique_resources=unique_resources,
|
||||
scan_resource_cache=scan_resource_cache,
|
||||
mute_rules_cache=mute_rules_cache,
|
||||
scan_categories_cache=scan_categories_cache,
|
||||
)
|
||||
|
||||
# Update scan progress
|
||||
@@ -810,13 +902,33 @@ def perform_prowler_scan(
|
||||
resource_scan_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
except Exception as filter_exception:
|
||||
import sentry_sdk
|
||||
|
||||
sentry_sdk.capture_exception(filter_exception)
|
||||
logger.error(
|
||||
f"Error storing filter values for scan {scan_id}: {filter_exception}"
|
||||
)
|
||||
|
||||
try:
|
||||
if scan_categories_cache:
|
||||
category_summaries = [
|
||||
ScanCategorySummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
category=category,
|
||||
severity=severity,
|
||||
total_findings=counts["total"],
|
||||
failed_findings=counts["failed"],
|
||||
new_failed_findings=counts["new_failed"],
|
||||
)
|
||||
for (category, severity), counts in scan_categories_cache.items()
|
||||
]
|
||||
with rls_transaction(tenant_id):
|
||||
ScanCategorySummary.objects.bulk_create(
|
||||
category_summaries, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
except Exception as cat_exception:
|
||||
sentry_sdk.capture_exception(cat_exception)
|
||||
logger.error(f"Error storing categories for scan {scan_id}: {cat_exception}")
|
||||
|
||||
serializer = ScanTaskSerializer(instance=scan_instance)
|
||||
return serializer.data
|
||||
|
||||
@@ -979,11 +1091,14 @@ def _aggregate_findings_by_region(
|
||||
findings_count_by_compliance = {}
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
# Fetch findings with resources in a single efficient query
|
||||
# Use select_related for finding fields and prefetch_related for many-to-many resources
|
||||
# Fetch only PASS/FAIL findings (optimized query reduces data transfer)
|
||||
# Other statuses are not needed for check_status or ThreatScore calculation
|
||||
findings = (
|
||||
Finding.all_objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id, muted=False
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
muted=False,
|
||||
status__in=["PASS", "FAIL"],
|
||||
)
|
||||
.only("id", "check_id", "status", "compliance")
|
||||
.prefetch_related(
|
||||
@@ -1001,6 +1116,8 @@ def _aggregate_findings_by_region(
|
||||
)
|
||||
|
||||
for finding in findings:
|
||||
status = finding.status
|
||||
|
||||
for resource in finding.small_resources:
|
||||
region = resource.region
|
||||
|
||||
@@ -1008,7 +1125,7 @@ def _aggregate_findings_by_region(
|
||||
current_status = check_status_by_region.setdefault(region, {})
|
||||
# Priority: FAIL > any other status
|
||||
if current_status.get(finding.check_id) != "FAIL":
|
||||
current_status[finding.check_id] = finding.status
|
||||
current_status[finding.check_id] = status
|
||||
|
||||
# Aggregate ThreatScore compliance counts
|
||||
if modeled_threatscore_compliance_id in (finding.compliance or {}):
|
||||
@@ -1023,7 +1140,7 @@ def _aggregate_findings_by_region(
|
||||
requirement_id, {"total": 0, "pass": 0}
|
||||
)
|
||||
requirement_stats["total"] += 1
|
||||
if finding.status == "PASS":
|
||||
if status == "PASS":
|
||||
requirement_stats["pass"] += 1
|
||||
|
||||
return check_status_by_region, findings_count_by_compliance
|
||||
@@ -1191,3 +1308,184 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating compliance requirements for scan {scan_id}: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
def aggregate_attack_surface(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Aggregate findings into attack surface overview records.
|
||||
|
||||
Creates one AttackSurfaceOverview record per attack surface type
|
||||
for the given scan, based on check_id mappings.
|
||||
|
||||
Args:
|
||||
tenant_id: Tenant that owns the scan.
|
||||
scan_id: Scan UUID whose findings should be aggregated.
|
||||
"""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
scan_instance = Scan.all_objects.select_related("provider").get(pk=scan_id)
|
||||
provider_type = scan_instance.provider.provider
|
||||
|
||||
provider_attack_surface_mapping = _get_attack_surface_mapping_from_provider(
|
||||
provider_type=provider_type
|
||||
)
|
||||
|
||||
# Filter out attack surfaces that are not compatible or have no resolved check IDs
|
||||
supported_mappings: dict[str, list[str]] = {}
|
||||
for attack_surface_type, check_ids in provider_attack_surface_mapping.items():
|
||||
compatible_providers = ATTACK_SURFACE_PROVIDER_COMPATIBILITY.get(
|
||||
attack_surface_type
|
||||
)
|
||||
if (
|
||||
compatible_providers is not None
|
||||
and provider_type not in compatible_providers
|
||||
):
|
||||
logger.info(
|
||||
f"Skipping {attack_surface_type} - not supported for {provider_type}"
|
||||
)
|
||||
continue
|
||||
|
||||
if not check_ids:
|
||||
logger.info(
|
||||
f"Skipping {attack_surface_type} - no check IDs resolved for {provider_type}"
|
||||
)
|
||||
continue
|
||||
|
||||
supported_mappings[attack_surface_type] = list(check_ids)
|
||||
|
||||
if not supported_mappings:
|
||||
logger.info(
|
||||
f"No attack surface mappings available for scan {scan_id} and provider {provider_type}"
|
||||
)
|
||||
logger.info(f"No attack surface overview records created for scan {scan_id}")
|
||||
return
|
||||
|
||||
# Map every check_id to its attack surface, so we can aggregate with a single query
|
||||
check_id_to_surface: dict[str, str] = {}
|
||||
for attack_surface_type, check_ids in supported_mappings.items():
|
||||
for check_id in check_ids:
|
||||
check_id_to_surface[check_id] = attack_surface_type
|
||||
|
||||
aggregated_counts = {
|
||||
attack_surface_type: {"total": 0, "failed": 0, "muted": 0}
|
||||
for attack_surface_type in supported_mappings.keys()
|
||||
}
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
finding_stats = (
|
||||
Finding.all_objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
check_id__in=list(check_id_to_surface.keys()),
|
||||
)
|
||||
.values("check_id")
|
||||
.annotate(
|
||||
total=Count("id"),
|
||||
failed=Count("id", filter=Q(status="FAIL", muted=False)),
|
||||
muted=Count("id", filter=Q(status="FAIL", muted=True)),
|
||||
)
|
||||
)
|
||||
|
||||
for stats in finding_stats:
|
||||
attack_surface_type = check_id_to_surface.get(stats["check_id"])
|
||||
if not attack_surface_type:
|
||||
continue
|
||||
|
||||
aggregated_counts[attack_surface_type]["total"] += stats["total"] or 0
|
||||
aggregated_counts[attack_surface_type]["failed"] += stats["failed"] or 0
|
||||
aggregated_counts[attack_surface_type]["muted"] += stats["muted"] or 0
|
||||
|
||||
overview_objects = []
|
||||
for attack_surface_type, counts in aggregated_counts.items():
|
||||
total = counts["total"]
|
||||
if not total:
|
||||
continue
|
||||
|
||||
overview_objects.append(
|
||||
AttackSurfaceOverview(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
attack_surface_type=attack_surface_type,
|
||||
total_findings=total,
|
||||
failed_findings=counts["failed"],
|
||||
muted_failed_findings=counts["muted"],
|
||||
)
|
||||
)
|
||||
|
||||
# Bulk create overview records
|
||||
if overview_objects:
|
||||
with rls_transaction(tenant_id):
|
||||
AttackSurfaceOverview.objects.bulk_create(overview_objects, batch_size=500)
|
||||
logger.info(
|
||||
f"Created {len(overview_objects)} attack surface overview records for scan {scan_id}"
|
||||
)
|
||||
else:
|
||||
logger.info(f"No attack surface overview records created for scan {scan_id}")
|
||||
|
||||
|
||||
def aggregate_daily_severity(tenant_id: str, scan_id: str):
|
||||
"""Aggregate scan severity counts into DailySeveritySummary (one record per provider/day)."""
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
scan = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
id=scan_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
).first()
|
||||
|
||||
if not scan:
|
||||
logger.warning(f"Scan {scan_id} not found or not completed")
|
||||
return {"status": "scan is not completed"}
|
||||
|
||||
provider_id = scan.provider_id
|
||||
scan_date = scan.completed_at.date()
|
||||
|
||||
severity_totals = (
|
||||
ScanSummary.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
)
|
||||
.values("severity")
|
||||
.annotate(total_fail=Sum("fail"), total_muted=Sum("muted"))
|
||||
)
|
||||
|
||||
severity_data = {
|
||||
"critical": 0,
|
||||
"high": 0,
|
||||
"medium": 0,
|
||||
"low": 0,
|
||||
"informational": 0,
|
||||
"muted": 0,
|
||||
}
|
||||
|
||||
for row in severity_totals:
|
||||
severity = row["severity"]
|
||||
if severity in severity_data:
|
||||
severity_data[severity] = row["total_fail"] or 0
|
||||
severity_data["muted"] += row["total_muted"] or 0
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
summary, created = DailySeveritySummary.objects.update_or_create(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
date=scan_date,
|
||||
defaults={
|
||||
"scan_id": scan_id,
|
||||
"critical": severity_data["critical"],
|
||||
"high": severity_data["high"],
|
||||
"medium": severity_data["medium"],
|
||||
"low": severity_data["low"],
|
||||
"informational": severity_data["informational"],
|
||||
"muted": severity_data["muted"],
|
||||
},
|
||||
)
|
||||
|
||||
action = "created" if created else "updated"
|
||||
logger.info(
|
||||
f"Daily severity summary {action} for provider {provider_id} on {scan_date}"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": action,
|
||||
"provider_id": str(provider_id),
|
||||
"date": str(scan_date),
|
||||
"severity_data": severity_data,
|
||||
}
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
from collections import defaultdict
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
|
||||
from django.db.models import Count, Q
|
||||
from tasks.utils import batched
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Finding, StatusChoices
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@@ -36,10 +41,15 @@ def _aggregate_requirement_statistics_from_database(
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
aggregated_statistics_queryset = (
|
||||
Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id)
|
||||
Finding.all_objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id, muted=False
|
||||
)
|
||||
.values("check_id")
|
||||
.annotate(
|
||||
total_findings=Count("id"),
|
||||
total_findings=Count(
|
||||
"id",
|
||||
filter=Q(status__in=[StatusChoices.PASS, StatusChoices.FAIL]),
|
||||
),
|
||||
passed_findings=Count("id", filter=Q(status=StatusChoices.PASS)),
|
||||
)
|
||||
)
|
||||
@@ -125,3 +135,105 @@ def _calculate_requirements_data_from_statistics(
|
||||
)
|
||||
|
||||
return attributes_by_requirement_id, requirements_list
|
||||
|
||||
|
||||
def _load_findings_for_requirement_checks(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
check_ids: list[str],
|
||||
prowler_provider,
|
||||
findings_cache: dict[str, list[FindingOutput]] | None = None,
|
||||
) -> dict[str, list[FindingOutput]]:
|
||||
"""
|
||||
Load findings for specific check IDs on-demand with optional caching.
|
||||
|
||||
This function loads only the findings needed for a specific set of checks,
|
||||
minimizing memory usage by avoiding loading all findings at once. This is used
|
||||
when generating detailed findings tables for specific requirements in the PDF.
|
||||
|
||||
Supports optional caching to avoid duplicate queries when generating multiple
|
||||
reports for the same scan.
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant ID for Row-Level Security context.
|
||||
scan_id (str): The ID of the scan to retrieve findings for.
|
||||
check_ids (list[str]): List of check IDs to load findings for.
|
||||
prowler_provider: The initialized Prowler provider instance.
|
||||
findings_cache (dict, optional): Cache of already loaded findings.
|
||||
If provided, checks are first looked up in cache before querying database.
|
||||
|
||||
Returns:
|
||||
dict[str, list[FindingOutput]]: Dictionary mapping check_id to list of FindingOutput objects.
|
||||
|
||||
Example:
|
||||
{
|
||||
'aws_iam_user_mfa_enabled': [FindingOutput(...), FindingOutput(...)],
|
||||
'aws_s3_bucket_public_access': [FindingOutput(...)]
|
||||
}
|
||||
"""
|
||||
findings_by_check_id = defaultdict(list)
|
||||
|
||||
if not check_ids:
|
||||
return dict(findings_by_check_id)
|
||||
|
||||
# Initialize cache if not provided
|
||||
if findings_cache is None:
|
||||
findings_cache = {}
|
||||
|
||||
# Separate cached and non-cached check_ids
|
||||
check_ids_to_load = []
|
||||
cache_hits = 0
|
||||
cache_misses = 0
|
||||
|
||||
for check_id in check_ids:
|
||||
if check_id in findings_cache:
|
||||
# Reuse from cache
|
||||
findings_by_check_id[check_id] = findings_cache[check_id]
|
||||
cache_hits += 1
|
||||
else:
|
||||
# Need to load from database
|
||||
check_ids_to_load.append(check_id)
|
||||
cache_misses += 1
|
||||
|
||||
if cache_hits > 0:
|
||||
logger.info(
|
||||
f"Findings cache: {cache_hits} hits, {cache_misses} misses "
|
||||
f"({cache_hits / (cache_hits + cache_misses) * 100:.1f}% hit rate)"
|
||||
)
|
||||
|
||||
# If all check_ids were in cache, return early
|
||||
if not check_ids_to_load:
|
||||
return dict(findings_by_check_id)
|
||||
|
||||
logger.info(f"Loading findings for {len(check_ids_to_load)} checks on-demand")
|
||||
|
||||
findings_queryset = (
|
||||
Finding.all_objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id, check_id__in=check_ids_to_load
|
||||
)
|
||||
.order_by("uid")
|
||||
.iterator()
|
||||
)
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
for batch, is_last_batch in batched(
|
||||
findings_queryset, DJANGO_FINDINGS_BATCH_SIZE
|
||||
):
|
||||
for finding_model in batch:
|
||||
finding_output = FindingOutput.transform_api_finding(
|
||||
finding_model, prowler_provider
|
||||
)
|
||||
findings_by_check_id[finding_output.check_id].append(finding_output)
|
||||
# Update cache with newly loaded findings
|
||||
if finding_output.check_id not in findings_cache:
|
||||
findings_cache[finding_output.check_id] = []
|
||||
findings_cache[finding_output.check_id].append(finding_output)
|
||||
|
||||
total_findings_loaded = sum(
|
||||
len(findings) for findings in findings_by_check_id.values()
|
||||
)
|
||||
logger.info(
|
||||
f"Loaded {total_findings_loaded} findings for {len(findings_by_check_id)} checks"
|
||||
)
|
||||
|
||||
return dict(findings_by_check_id)
|
||||
|
||||
+150
-11
@@ -10,7 +10,9 @@ from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIR
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from tasks.jobs.backfill import (
|
||||
backfill_compliance_summaries,
|
||||
backfill_daily_severity_summaries,
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
)
|
||||
from tasks.jobs.connection import (
|
||||
check_integration_connection,
|
||||
@@ -35,8 +37,10 @@ from tasks.jobs.lighthouse_providers import (
|
||||
refresh_lighthouse_provider_models,
|
||||
)
|
||||
from tasks.jobs.muting import mute_historical_findings
|
||||
from tasks.jobs.report import generate_threatscore_report_job
|
||||
from tasks.jobs.report import generate_compliance_reports_job
|
||||
from tasks.jobs.scan import (
|
||||
aggregate_attack_surface,
|
||||
aggregate_daily_severity,
|
||||
aggregate_findings,
|
||||
create_compliance_requirements,
|
||||
perform_prowler_scan,
|
||||
@@ -46,7 +50,7 @@ from tasks.utils import batched, get_next_execution_datetime
|
||||
from api.compliance import get_compliance_frameworks
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.decorators import set_tenant
|
||||
from api.decorators import handle_provider_deletion, set_tenant
|
||||
from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices
|
||||
from api.utils import initialize_prowler_provider
|
||||
from api.v1.serializers import ScanTaskSerializer
|
||||
@@ -57,6 +61,58 @@ from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def _cleanup_orphan_scheduled_scans(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
scheduler_task_id: int,
|
||||
) -> int:
|
||||
"""
|
||||
TEMPORARY WORKAROUND: Clean up orphan AVAILABLE scans.
|
||||
|
||||
Detects and removes AVAILABLE scans that were never used due to an
|
||||
issue during the first scheduled scan setup.
|
||||
|
||||
An AVAILABLE scan is considered orphan if there's also a SCHEDULED scan for
|
||||
the same provider with the same scheduler_task_id. This situation indicates
|
||||
that the first scan execution didn't find the AVAILABLE scan (because it
|
||||
wasn't committed yet, probably) and created a new one, leaving the AVAILABLE orphaned.
|
||||
|
||||
Args:
|
||||
tenant_id: The tenant ID.
|
||||
provider_id: The provider ID.
|
||||
scheduler_task_id: The PeriodicTask ID that triggers these scans.
|
||||
|
||||
Returns:
|
||||
Number of orphan scans deleted (0 if none found).
|
||||
"""
|
||||
orphan_available_scans = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=scheduler_task_id,
|
||||
)
|
||||
|
||||
scheduled_scan_exists = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=scheduler_task_id,
|
||||
).exists()
|
||||
|
||||
if scheduled_scan_exists and orphan_available_scans.exists():
|
||||
orphan_count = orphan_available_scans.count()
|
||||
logger.warning(
|
||||
f"[WORKAROUND] Found {orphan_count} orphan AVAILABLE scan(s) for "
|
||||
f"provider {provider_id} alongside a SCHEDULED scan. Cleaning up orphans..."
|
||||
)
|
||||
orphan_available_scans.delete()
|
||||
return orphan_count
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str):
|
||||
"""
|
||||
Helper function to perform tasks after a scan is completed.
|
||||
@@ -69,13 +125,20 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str)
|
||||
create_compliance_requirements_task.apply_async(
|
||||
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
|
||||
)
|
||||
aggregate_attack_surface_task.apply_async(
|
||||
kwargs={"tenant_id": tenant_id, "scan_id": scan_id}
|
||||
)
|
||||
chain(
|
||||
perform_scan_summary_task.si(tenant_id=tenant_id, scan_id=scan_id),
|
||||
generate_outputs_task.si(
|
||||
scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id
|
||||
group(
|
||||
aggregate_daily_severity_task.si(tenant_id=tenant_id, scan_id=scan_id),
|
||||
generate_outputs_task.si(
|
||||
scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id
|
||||
),
|
||||
),
|
||||
group(
|
||||
generate_threatscore_report_task.si(
|
||||
# Use optimized task that generates both reports with shared queries
|
||||
generate_compliance_reports_task.si(
|
||||
tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id
|
||||
),
|
||||
check_integrations_task.si(
|
||||
@@ -139,6 +202,7 @@ def delete_provider_task(provider_id: str, tenant_id: str):
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="scan-perform", queue="scans")
|
||||
@handle_provider_deletion
|
||||
def perform_scan_task(
|
||||
tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None
|
||||
):
|
||||
@@ -171,6 +235,7 @@ def perform_scan_task(
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, bind=True, name="scan-perform-scheduled", queue="scans")
|
||||
@handle_provider_deletion
|
||||
def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
"""
|
||||
Task to perform a scheduled Prowler scan on a given provider.
|
||||
@@ -234,6 +299,14 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
return serializer.data
|
||||
|
||||
next_scan_datetime = get_next_execution_datetime(task_id, provider_id)
|
||||
|
||||
# TEMPORARY WORKAROUND: Clean up orphan scans from transaction isolation issue
|
||||
_cleanup_orphan_scheduled_scans(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
)
|
||||
|
||||
scan_instance, _ = Scan.objects.get_or_create(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
@@ -276,6 +349,7 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
|
||||
|
||||
@shared_task(name="scan-summary", queue="overview")
|
||||
@handle_provider_deletion
|
||||
def perform_scan_summary_task(tenant_id: str, scan_id: str):
|
||||
return aggregate_findings(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
@@ -291,6 +365,7 @@ def delete_tenant_task(tenant_id: str):
|
||||
queue="scan-reports",
|
||||
)
|
||||
@set_tenant(keep_tenant=True)
|
||||
@handle_provider_deletion
|
||||
def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
"""
|
||||
Process findings in batches and generate output files in multiple formats.
|
||||
@@ -319,7 +394,7 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
|
||||
frameworks_bulk = Compliance.get_bulk(provider_type)
|
||||
frameworks_avail = get_compliance_frameworks(provider_type)
|
||||
out_dir, comp_dir, _ = _generate_output_directory(
|
||||
out_dir, comp_dir = _generate_output_directory(
|
||||
DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id
|
||||
)
|
||||
|
||||
@@ -486,6 +561,7 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
|
||||
|
||||
@shared_task(name="backfill-scan-resource-summaries", queue="backfill")
|
||||
@handle_provider_deletion
|
||||
def backfill_scan_resource_summaries_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Tries to backfill the resource scan summaries table for a given scan.
|
||||
@@ -498,6 +574,7 @@ def backfill_scan_resource_summaries_task(tenant_id: str, scan_id: str):
|
||||
|
||||
|
||||
@shared_task(name="backfill-compliance-summaries", queue="backfill")
|
||||
@handle_provider_deletion
|
||||
def backfill_compliance_summaries_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Tries to backfill compliance overview summaries for a completed scan.
|
||||
@@ -512,7 +589,29 @@ def backfill_compliance_summaries_task(tenant_id: str, scan_id: str):
|
||||
return backfill_compliance_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="backfill-daily-severity-summaries", queue="backfill")
|
||||
def backfill_daily_severity_summaries_task(tenant_id: str, days: int = None):
|
||||
"""Backfill DailySeveritySummary from historical scans. Use days param to limit scope."""
|
||||
return backfill_daily_severity_summaries(tenant_id=tenant_id, days=days)
|
||||
|
||||
|
||||
@shared_task(name="backfill-scan-category-summaries", queue="backfill")
|
||||
@handle_provider_deletion
|
||||
def backfill_scan_category_summaries_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Backfill ScanCategorySummary for a completed scan.
|
||||
|
||||
Aggregates unique categories from findings and creates a summary row.
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant identifier.
|
||||
scan_id (str): The scan identifier.
|
||||
"""
|
||||
return backfill_scan_category_summaries(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="scan-compliance-overviews", queue="compliance")
|
||||
@handle_provider_deletion
|
||||
def create_compliance_requirements_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Creates detailed compliance requirement records for a scan.
|
||||
@@ -528,6 +627,29 @@ def create_compliance_requirements_task(tenant_id: str, scan_id: str):
|
||||
return create_compliance_requirements(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="scan-attack-surface-overviews", queue="overview")
|
||||
@handle_provider_deletion
|
||||
def aggregate_attack_surface_task(tenant_id: str, scan_id: str):
|
||||
"""
|
||||
Creates attack surface overview records for a scan.
|
||||
|
||||
This task processes findings and aggregates them into attack surface categories
|
||||
(internet-exposed, secrets, privilege-escalation, ec2-imdsv1) for quick overview queries.
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant ID for which to create records.
|
||||
scan_id (str): The ID of the scan for which to create records.
|
||||
"""
|
||||
return aggregate_attack_surface(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(name="scan-daily-severity", queue="overview")
|
||||
@handle_provider_deletion
|
||||
def aggregate_daily_severity_task(tenant_id: str, scan_id: str):
|
||||
"""Aggregate scan severity into DailySeveritySummary for findings_severity/timeseries endpoint."""
|
||||
return aggregate_daily_severity(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="lighthouse-connection-check")
|
||||
@set_tenant
|
||||
def check_lighthouse_connection_task(lighthouse_config_id: str, tenant_id: str = None):
|
||||
@@ -566,6 +688,7 @@ def refresh_lighthouse_provider_models_task(
|
||||
|
||||
|
||||
@shared_task(name="integration-check")
|
||||
@handle_provider_deletion
|
||||
def check_integrations_task(tenant_id: str, provider_id: str, scan_id: str = None):
|
||||
"""
|
||||
Check and execute all configured integrations for a provider.
|
||||
@@ -630,6 +753,7 @@ def check_integrations_task(tenant_id: str, provider_id: str, scan_id: str = Non
|
||||
name="integration-s3",
|
||||
queue="integrations",
|
||||
)
|
||||
@handle_provider_deletion
|
||||
def s3_integration_task(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
@@ -686,19 +810,34 @@ def jira_integration_task(
|
||||
|
||||
@shared_task(
|
||||
base=RLSTask,
|
||||
name="scan-threatscore-report",
|
||||
name="scan-compliance-reports",
|
||||
queue="scan-reports",
|
||||
)
|
||||
def generate_threatscore_report_task(tenant_id: str, scan_id: str, provider_id: str):
|
||||
@handle_provider_deletion
|
||||
def generate_compliance_reports_task(tenant_id: str, scan_id: str, provider_id: str):
|
||||
"""
|
||||
Task to generate a threatscore report for a given scan.
|
||||
Optimized task to generate ThreatScore, ENS, and NIS2 reports with shared queries.
|
||||
|
||||
This task is more efficient than running separate report tasks because it reuses database queries:
|
||||
- Provider object fetched once (instead of three times)
|
||||
- Requirement statistics aggregated once (instead of three times)
|
||||
- Can reduce database load by up to 50-70%
|
||||
|
||||
Args:
|
||||
tenant_id (str): The tenant identifier.
|
||||
scan_id (str): The scan identifier.
|
||||
provider_id (str): The provider identifier.
|
||||
|
||||
Returns:
|
||||
dict: Results for all reports containing upload status and paths.
|
||||
"""
|
||||
return generate_threatscore_report_job(
|
||||
tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id
|
||||
return generate_compliance_reports_job(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
provider_id=provider_id,
|
||||
generate_threatscore=True,
|
||||
generate_ens=True,
|
||||
generate_nis2=True,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -4,14 +4,19 @@ import pytest
|
||||
from tasks.jobs.backfill import (
|
||||
backfill_compliance_summaries,
|
||||
backfill_resource_scan_summaries,
|
||||
backfill_scan_category_summaries,
|
||||
)
|
||||
|
||||
from api.models import (
|
||||
ComplianceOverviewSummary,
|
||||
Finding,
|
||||
ResourceScanSummary,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
StateChoices,
|
||||
)
|
||||
from prowler.lib.check.models import Severity
|
||||
from prowler.lib.outputs.finding import Status
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
@@ -46,6 +51,45 @@ def get_not_completed_scans(providers_fixture):
|
||||
return scan_1, scan_2
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def findings_with_categories_fixture(scans_fixture, resources_fixture):
|
||||
scan = scans_fixture[0]
|
||||
resource = resources_fixture[0]
|
||||
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
uid="finding_with_categories",
|
||||
scan=scan,
|
||||
delta="new",
|
||||
status=Status.FAIL,
|
||||
status_extended="test status",
|
||||
impact=Severity.critical,
|
||||
impact_extended="test impact",
|
||||
severity=Severity.critical,
|
||||
raw_result={"status": Status.FAIL},
|
||||
check_id="test_check",
|
||||
check_metadata={"CheckId": "test_check"},
|
||||
categories=["gen-ai", "security"],
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
return finding
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def scan_category_summary_fixture(scans_fixture):
|
||||
scan = scans_fixture[0]
|
||||
return ScanCategorySummary.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
scan=scan,
|
||||
category="existing-category",
|
||||
severity=Severity.critical,
|
||||
total_findings=1,
|
||||
failed_findings=0,
|
||||
new_failed_findings=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBackfillResourceScanSummaries:
|
||||
def test_already_backfilled(self, resource_scan_summary_data):
|
||||
@@ -172,3 +216,47 @@ class TestBackfillComplianceSummaries:
|
||||
assert summary.requirements_failed == expected_counts["requirements_failed"]
|
||||
assert summary.requirements_manual == expected_counts["requirements_manual"]
|
||||
assert summary.total_requirements == expected_counts["total_requirements"]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBackfillScanCategorySummaries:
|
||||
def test_already_backfilled(self, scan_category_summary_fixture):
|
||||
tenant_id = scan_category_summary_fixture.tenant_id
|
||||
scan_id = scan_category_summary_fixture.scan_id
|
||||
|
||||
result = backfill_scan_category_summaries(str(tenant_id), str(scan_id))
|
||||
|
||||
assert result == {"status": "already backfilled"}
|
||||
|
||||
def test_not_completed_scan(self, get_not_completed_scans):
|
||||
for scan in get_not_completed_scans:
|
||||
result = backfill_scan_category_summaries(str(scan.tenant_id), str(scan.id))
|
||||
assert result == {"status": "scan is not completed"}
|
||||
|
||||
def test_no_categories_to_backfill(self, scans_fixture):
|
||||
scan = scans_fixture[1] # Failed scan with no findings
|
||||
result = backfill_scan_category_summaries(str(scan.tenant_id), str(scan.id))
|
||||
assert result == {"status": "no categories to backfill"}
|
||||
|
||||
def test_successful_backfill(self, findings_with_categories_fixture):
|
||||
finding = findings_with_categories_fixture
|
||||
tenant_id = str(finding.tenant_id)
|
||||
scan_id = str(finding.scan_id)
|
||||
|
||||
result = backfill_scan_category_summaries(tenant_id, scan_id)
|
||||
|
||||
# 2 categories × 1 severity = 2 rows
|
||||
assert result == {"status": "backfilled", "categories_count": 2}
|
||||
|
||||
summaries = ScanCategorySummary.objects.filter(
|
||||
tenant_id=tenant_id, scan_id=scan_id
|
||||
)
|
||||
assert summaries.count() == 2
|
||||
categories = set(summaries.values_list("category", flat=True))
|
||||
assert categories == {"gen-ai", "security"}
|
||||
|
||||
for summary in summaries:
|
||||
assert summary.severity == Severity.critical
|
||||
assert summary.total_findings == 1
|
||||
assert summary.failed_findings == 1
|
||||
assert summary.new_failed_findings == 1
|
||||
|
||||
@@ -28,6 +28,7 @@ class TestScheduleProviderScan:
|
||||
"tenant_id": str(provider_instance.tenant_id),
|
||||
"provider_id": str(provider_instance.id),
|
||||
},
|
||||
countdown=5,
|
||||
)
|
||||
|
||||
task_name = f"scan-perform-scheduled-{provider_instance.id}"
|
||||
|
||||
@@ -9,6 +9,7 @@ import pytest
|
||||
from botocore.exceptions import ClientError
|
||||
from tasks.jobs.export import (
|
||||
_compress_output_files,
|
||||
_generate_compliance_output_directory,
|
||||
_generate_output_directory,
|
||||
_upload_to_s3,
|
||||
get_s3_client,
|
||||
@@ -147,10 +148,11 @@ class TestOutputs:
|
||||
)
|
||||
mock_logger.assert_called()
|
||||
|
||||
@patch("tasks.jobs.export.set_output_timestamp")
|
||||
@patch("tasks.jobs.export.rls_transaction")
|
||||
@patch("tasks.jobs.export.Scan")
|
||||
def test_generate_output_directory_creates_paths(
|
||||
self, mock_scan, mock_rls_transaction, tmpdir
|
||||
self, mock_scan, mock_rls_transaction, mock_set_timestamp, tmpdir
|
||||
):
|
||||
# Mock the scan object with a started_at timestamp
|
||||
mock_scan_instance = MagicMock()
|
||||
@@ -168,22 +170,40 @@ class TestOutputs:
|
||||
provider = "aws"
|
||||
expected_timestamp = "20230615103045"
|
||||
|
||||
path, compliance, threatscore = _generate_output_directory(
|
||||
# Test _generate_output_directory (returns standard and compliance paths)
|
||||
path, compliance = _generate_output_directory(
|
||||
base_dir, provider, tenant_id, scan_id
|
||||
)
|
||||
|
||||
assert os.path.isdir(os.path.dirname(path))
|
||||
assert os.path.isdir(os.path.dirname(compliance))
|
||||
assert os.path.isdir(os.path.dirname(threatscore))
|
||||
|
||||
assert path.endswith(f"{provider}-{expected_timestamp}")
|
||||
assert compliance.endswith(f"{provider}-{expected_timestamp}")
|
||||
assert threatscore.endswith(f"{provider}-{expected_timestamp}")
|
||||
assert "/compliance/" in compliance
|
||||
|
||||
# Test _generate_compliance_output_directory with "threatscore"
|
||||
threatscore = _generate_compliance_output_directory(
|
||||
base_dir, provider, tenant_id, scan_id, compliance_framework="threatscore"
|
||||
)
|
||||
|
||||
assert os.path.isdir(os.path.dirname(threatscore))
|
||||
assert threatscore.endswith(f"{provider}-{expected_timestamp}")
|
||||
assert "/threatscore/" in threatscore
|
||||
|
||||
# Test _generate_compliance_output_directory with "ens"
|
||||
ens = _generate_compliance_output_directory(
|
||||
base_dir, provider, tenant_id, scan_id, compliance_framework="ens"
|
||||
)
|
||||
|
||||
assert os.path.isdir(os.path.dirname(ens))
|
||||
assert ens.endswith(f"{provider}-{expected_timestamp}")
|
||||
assert "/ens/" in ens
|
||||
|
||||
@patch("tasks.jobs.export.set_output_timestamp")
|
||||
@patch("tasks.jobs.export.rls_transaction")
|
||||
@patch("tasks.jobs.export.Scan")
|
||||
def test_generate_output_directory_invalid_character(
|
||||
self, mock_scan, mock_rls_transaction, tmpdir
|
||||
self, mock_scan, mock_rls_transaction, mock_set_timestamp, tmpdir
|
||||
):
|
||||
# Mock the scan object with a started_at timestamp
|
||||
mock_scan_instance = MagicMock()
|
||||
@@ -201,14 +221,25 @@ class TestOutputs:
|
||||
provider = "aws/test@check"
|
||||
expected_timestamp = "20230615103045"
|
||||
|
||||
path, compliance, threatscore = _generate_output_directory(
|
||||
# Test provider name sanitization with _generate_output_directory
|
||||
path, compliance = _generate_output_directory(
|
||||
base_dir, provider, tenant_id, scan_id
|
||||
)
|
||||
|
||||
assert os.path.isdir(os.path.dirname(path))
|
||||
assert os.path.isdir(os.path.dirname(compliance))
|
||||
assert os.path.isdir(os.path.dirname(threatscore))
|
||||
|
||||
assert path.endswith(f"aws-test-check-{expected_timestamp}")
|
||||
assert compliance.endswith(f"aws-test-check-{expected_timestamp}")
|
||||
|
||||
# Test provider name sanitization with _generate_compliance_output_directory
|
||||
threatscore = _generate_compliance_output_directory(
|
||||
base_dir, provider, tenant_id, scan_id, compliance_framework="threatscore"
|
||||
)
|
||||
ens = _generate_compliance_output_directory(
|
||||
base_dir, provider, tenant_id, scan_id, compliance_framework="ens"
|
||||
)
|
||||
|
||||
assert os.path.isdir(os.path.dirname(threatscore))
|
||||
assert os.path.isdir(os.path.dirname(ens))
|
||||
assert threatscore.endswith(f"aws-test-check-{expected_timestamp}")
|
||||
assert ens.endswith(f"aws-test-check-{expected_timestamp}")
|
||||
|
||||
@@ -1199,9 +1199,6 @@ class TestSecurityHubIntegrationUploads:
|
||||
)
|
||||
|
||||
assert result is False
|
||||
# Integration should be marked as disconnected
|
||||
integration.save.assert_called_once()
|
||||
assert integration.connected is False
|
||||
|
||||
@patch("tasks.jobs.integrations.ASFF")
|
||||
@patch("tasks.jobs.integrations.FindingOutput")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,14 +9,18 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from tasks.jobs.scan import (
|
||||
_ATTACK_SURFACE_MAPPING_CACHE,
|
||||
_aggregate_findings_by_region,
|
||||
_copy_compliance_requirement_rows,
|
||||
_create_compliance_summaries,
|
||||
_create_finding_delta,
|
||||
_get_attack_surface_mapping_from_provider,
|
||||
_normalized_compliance_key,
|
||||
_persist_compliance_requirement_rows,
|
||||
_process_finding_micro_batch,
|
||||
_store_resources,
|
||||
aggregate_attack_surface,
|
||||
aggregate_category_counts,
|
||||
aggregate_findings,
|
||||
create_compliance_requirements,
|
||||
perform_prowler_scan,
|
||||
@@ -1374,6 +1378,7 @@ class TestProcessFindingMicroBatch:
|
||||
unique_resources: set[tuple[str, str]] = set()
|
||||
scan_resource_cache: set[tuple[str, str, str, str]] = set()
|
||||
mute_rules_cache = {}
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
|
||||
with (
|
||||
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
|
||||
@@ -1391,6 +1396,7 @@ class TestProcessFindingMicroBatch:
|
||||
unique_resources,
|
||||
scan_resource_cache,
|
||||
mute_rules_cache,
|
||||
scan_categories_cache,
|
||||
)
|
||||
|
||||
created_finding = Finding.objects.get(uid=finding.uid)
|
||||
@@ -1483,6 +1489,7 @@ class TestProcessFindingMicroBatch:
|
||||
unique_resources: set[tuple[str, str]] = set()
|
||||
scan_resource_cache: set[tuple[str, str, str, str]] = set()
|
||||
mute_rules_cache = {finding.uid: "Muted via rule"}
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
|
||||
with (
|
||||
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
|
||||
@@ -1500,6 +1507,7 @@ class TestProcessFindingMicroBatch:
|
||||
unique_resources,
|
||||
scan_resource_cache,
|
||||
mute_rules_cache,
|
||||
scan_categories_cache,
|
||||
)
|
||||
|
||||
existing_resource.refresh_from_db()
|
||||
@@ -1607,6 +1615,7 @@ class TestProcessFindingMicroBatch:
|
||||
unique_resources: set[tuple[str, str]] = set()
|
||||
scan_resource_cache: set[tuple[str, str, str, str]] = set()
|
||||
mute_rules_cache = {}
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
|
||||
with (
|
||||
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
|
||||
@@ -1625,6 +1634,7 @@ class TestProcessFindingMicroBatch:
|
||||
unique_resources,
|
||||
scan_resource_cache,
|
||||
mute_rules_cache,
|
||||
scan_categories_cache,
|
||||
)
|
||||
|
||||
# Verify the long UID finding was NOT created
|
||||
@@ -1645,6 +1655,118 @@ class TestProcessFindingMicroBatch:
|
||||
for call in warning_calls
|
||||
)
|
||||
|
||||
def test_process_finding_micro_batch_tracks_categories(
|
||||
self, tenants_fixture, scans_fixture
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
provider = scan.provider
|
||||
|
||||
finding1 = FakeFinding(
|
||||
uid="finding-cat-1",
|
||||
status=StatusChoices.PASS,
|
||||
status_extended="all good",
|
||||
severity=Severity.low,
|
||||
check_id="genai_check",
|
||||
resource_uid="arn:aws:bedrock:::model/test",
|
||||
resource_name="test-model",
|
||||
region="us-east-1",
|
||||
service_name="bedrock",
|
||||
resource_type="model",
|
||||
resource_tags={},
|
||||
resource_metadata={},
|
||||
resource_details={},
|
||||
partition="aws",
|
||||
raw={},
|
||||
compliance={},
|
||||
metadata={"categories": ["gen-ai", "security"]},
|
||||
muted=False,
|
||||
)
|
||||
|
||||
finding2 = FakeFinding(
|
||||
uid="finding-cat-2",
|
||||
status=StatusChoices.FAIL,
|
||||
status_extended="bad",
|
||||
severity=Severity.high,
|
||||
check_id="iam_check",
|
||||
resource_uid="arn:aws:iam:::user/test",
|
||||
resource_name="test-user",
|
||||
region="us-east-1",
|
||||
service_name="iam",
|
||||
resource_type="user",
|
||||
resource_tags={},
|
||||
resource_metadata={},
|
||||
resource_details={},
|
||||
partition="aws",
|
||||
raw={},
|
||||
compliance={},
|
||||
metadata={"categories": ["security", "iam"]},
|
||||
muted=False,
|
||||
)
|
||||
|
||||
resource_cache = {}
|
||||
tag_cache = {}
|
||||
last_status_cache = {}
|
||||
resource_failed_findings_cache = {}
|
||||
unique_resources: set[tuple[str, str]] = set()
|
||||
scan_resource_cache: set[tuple[str, str, str, str]] = set()
|
||||
mute_rules_cache = {}
|
||||
scan_categories_cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
|
||||
with (
|
||||
patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction),
|
||||
patch("api.db_utils.rls_transaction", new=noop_rls_transaction),
|
||||
):
|
||||
_process_finding_micro_batch(
|
||||
str(tenant.id),
|
||||
[finding1, finding2],
|
||||
scan,
|
||||
provider,
|
||||
resource_cache,
|
||||
tag_cache,
|
||||
last_status_cache,
|
||||
resource_failed_findings_cache,
|
||||
unique_resources,
|
||||
scan_resource_cache,
|
||||
mute_rules_cache,
|
||||
scan_categories_cache,
|
||||
)
|
||||
|
||||
# finding1: PASS, severity=low, categories=["gen-ai", "security"]
|
||||
# finding2: FAIL, severity=high, categories=["security", "iam"]
|
||||
# Keys are (category, severity) tuples
|
||||
assert set(scan_categories_cache.keys()) == {
|
||||
("gen-ai", "low"),
|
||||
("security", "low"),
|
||||
("security", "high"),
|
||||
("iam", "high"),
|
||||
}
|
||||
assert scan_categories_cache[("gen-ai", "low")] == {
|
||||
"total": 1,
|
||||
"failed": 0,
|
||||
"new_failed": 0,
|
||||
}
|
||||
assert scan_categories_cache[("security", "low")] == {
|
||||
"total": 1,
|
||||
"failed": 0,
|
||||
"new_failed": 0,
|
||||
}
|
||||
assert scan_categories_cache[("security", "high")] == {
|
||||
"total": 1,
|
||||
"failed": 1,
|
||||
"new_failed": 1,
|
||||
}
|
||||
assert scan_categories_cache[("iam", "high")] == {
|
||||
"total": 1,
|
||||
"failed": 1,
|
||||
"new_failed": 1,
|
||||
}
|
||||
|
||||
created_finding1 = Finding.objects.get(uid="finding-cat-1")
|
||||
created_finding2 = Finding.objects.get(uid="finding-cat-2")
|
||||
assert set(created_finding1.categories) == {"gen-ai", "security"}
|
||||
assert set(created_finding2.categories) == {"security", "iam"}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestCreateComplianceRequirements:
|
||||
@@ -3338,7 +3460,10 @@ class TestAggregateFindingsByRegion:
|
||||
|
||||
# Verify filter was called with muted=False
|
||||
mock_findings_filter.assert_called_once_with(
|
||||
tenant_id=tenant_id, scan_id=scan_id, muted=False
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
muted=False,
|
||||
status__in=["PASS", "FAIL"],
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.scan.Finding.all_objects.filter")
|
||||
@@ -3471,3 +3596,429 @@ class TestAggregateFindingsByRegion:
|
||||
|
||||
assert check_status_by_region == {}
|
||||
assert findings_count_by_compliance == {}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestAggregateAttackSurface:
|
||||
"""Test aggregate_attack_surface function and related caching."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Clear cache before each test."""
|
||||
_ATTACK_SURFACE_MAPPING_CACHE.clear()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clear cache after each test."""
|
||||
_ATTACK_SURFACE_MAPPING_CACHE.clear()
|
||||
|
||||
@patch("tasks.jobs.scan.CheckMetadata.list")
|
||||
def test_get_attack_surface_mapping_caches_result(self, mock_check_metadata_list):
|
||||
"""Test that _get_attack_surface_mapping_from_provider caches results."""
|
||||
mock_check_metadata_list.return_value = {"check_internet_exposed_1"}
|
||||
|
||||
# First call should hit CheckMetadata.list
|
||||
result1 = _get_attack_surface_mapping_from_provider("aws")
|
||||
assert mock_check_metadata_list.call_count == 2 # internet-exposed, secrets
|
||||
|
||||
# Second call should use cache
|
||||
result2 = _get_attack_surface_mapping_from_provider("aws")
|
||||
assert mock_check_metadata_list.call_count == 2 # No additional calls
|
||||
|
||||
assert result1 is result2
|
||||
assert "aws" in _ATTACK_SURFACE_MAPPING_CACHE
|
||||
|
||||
@patch("tasks.jobs.scan.CheckMetadata.list")
|
||||
def test_get_attack_surface_mapping_different_providers(
|
||||
self, mock_check_metadata_list
|
||||
):
|
||||
"""Test caching works independently for different providers."""
|
||||
mock_check_metadata_list.return_value = {"check_1"}
|
||||
|
||||
_get_attack_surface_mapping_from_provider("aws")
|
||||
aws_call_count = mock_check_metadata_list.call_count
|
||||
|
||||
_get_attack_surface_mapping_from_provider("gcp")
|
||||
gcp_call_count = mock_check_metadata_list.call_count
|
||||
|
||||
# Both providers should have made calls
|
||||
assert gcp_call_count > aws_call_count
|
||||
assert "aws" in _ATTACK_SURFACE_MAPPING_CACHE
|
||||
assert "gcp" in _ATTACK_SURFACE_MAPPING_CACHE
|
||||
|
||||
@patch("tasks.jobs.scan.CheckMetadata.list")
|
||||
def test_get_attack_surface_mapping_returns_hardcoded_checks(
|
||||
self, mock_check_metadata_list
|
||||
):
|
||||
"""Test that hardcoded check IDs are returned for privilege-escalation and ec2-imdsv1."""
|
||||
mock_check_metadata_list.return_value = set()
|
||||
|
||||
result = _get_attack_surface_mapping_from_provider("aws")
|
||||
|
||||
# Hardcoded checks should be present
|
||||
assert (
|
||||
"iam_policy_allows_privilege_escalation" in result["privilege-escalation"]
|
||||
)
|
||||
assert (
|
||||
"iam_inline_policy_allows_privilege_escalation"
|
||||
in result["privilege-escalation"]
|
||||
)
|
||||
assert "ec2_instance_imdsv2_enabled" in result["ec2-imdsv1"]
|
||||
|
||||
@patch("tasks.jobs.scan.AttackSurfaceOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.Finding.all_objects.filter")
|
||||
@patch("tasks.jobs.scan._get_attack_surface_mapping_from_provider")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_attack_surface_creates_overview_records(
|
||||
self,
|
||||
mock_rls_transaction,
|
||||
mock_get_mapping,
|
||||
mock_findings_filter,
|
||||
mock_bulk_create,
|
||||
tenants_fixture,
|
||||
scans_fixture,
|
||||
):
|
||||
"""Test that aggregate_attack_surface creates AttackSurfaceOverview records."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
scan.provider.provider = "aws"
|
||||
scan.provider.save()
|
||||
|
||||
mock_get_mapping.return_value = {
|
||||
"internet-exposed": {"check_internet_1", "check_internet_2"},
|
||||
"secrets": {"check_secrets_1"},
|
||||
"privilege-escalation": {"check_privesc_1"},
|
||||
"ec2-imdsv1": {"check_imdsv1_1"},
|
||||
}
|
||||
|
||||
# Mock findings aggregation
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.values.return_value = mock_queryset
|
||||
mock_queryset.annotate.return_value = [
|
||||
{"check_id": "check_internet_1", "total": 10, "failed": 3, "muted": 1},
|
||||
{"check_id": "check_secrets_1", "total": 5, "failed": 2, "muted": 0},
|
||||
]
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
mock_findings_filter.return_value = mock_queryset
|
||||
|
||||
aggregate_attack_surface(str(tenant.id), str(scan.id))
|
||||
|
||||
mock_bulk_create.assert_called_once()
|
||||
args, kwargs = mock_bulk_create.call_args
|
||||
objects = args[0]
|
||||
|
||||
# Should create records for internet-exposed and secrets (the ones with findings)
|
||||
assert len(objects) == 2
|
||||
assert kwargs["batch_size"] == 500
|
||||
|
||||
@patch("tasks.jobs.scan.AttackSurfaceOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.Finding.all_objects.filter")
|
||||
@patch("tasks.jobs.scan._get_attack_surface_mapping_from_provider")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_attack_surface_skips_unsupported_provider(
|
||||
self,
|
||||
mock_rls_transaction,
|
||||
mock_get_mapping,
|
||||
mock_findings_filter,
|
||||
mock_bulk_create,
|
||||
tenants_fixture,
|
||||
scans_fixture,
|
||||
):
|
||||
"""Test that ec2-imdsv1 is skipped for non-AWS providers."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
scan.provider.provider = "gcp"
|
||||
scan.provider.uid = "gcp-test-project-id"
|
||||
scan.provider.save()
|
||||
|
||||
mock_get_mapping.return_value = {
|
||||
"internet-exposed": {"check_internet_1"},
|
||||
"secrets": {"check_secrets_1"},
|
||||
"privilege-escalation": set(), # Not supported for GCP
|
||||
"ec2-imdsv1": {"check_imdsv1_1"}, # Should be skipped for GCP
|
||||
}
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.values.return_value = mock_queryset
|
||||
mock_queryset.annotate.return_value = [
|
||||
{"check_id": "check_internet_1", "total": 5, "failed": 1, "muted": 0},
|
||||
]
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
mock_findings_filter.return_value = mock_queryset
|
||||
|
||||
aggregate_attack_surface(str(tenant.id), str(scan.id))
|
||||
|
||||
# ec2-imdsv1 check_ids should not be in the filter
|
||||
filter_call = mock_findings_filter.call_args
|
||||
check_ids_in_filter = filter_call[1]["check_id__in"]
|
||||
assert "check_imdsv1_1" not in check_ids_in_filter
|
||||
|
||||
@patch("tasks.jobs.scan.AttackSurfaceOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.Finding.all_objects.filter")
|
||||
@patch("tasks.jobs.scan._get_attack_surface_mapping_from_provider")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_attack_surface_no_findings(
|
||||
self,
|
||||
mock_rls_transaction,
|
||||
mock_get_mapping,
|
||||
mock_findings_filter,
|
||||
mock_bulk_create,
|
||||
tenants_fixture,
|
||||
scans_fixture,
|
||||
):
|
||||
"""Test that no records are created when there are no findings."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
|
||||
mock_get_mapping.return_value = {
|
||||
"internet-exposed": {"check_1"},
|
||||
"secrets": {"check_2"},
|
||||
"privilege-escalation": set(),
|
||||
"ec2-imdsv1": set(),
|
||||
}
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.values.return_value = mock_queryset
|
||||
mock_queryset.annotate.return_value = [] # No findings
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
mock_findings_filter.return_value = mock_queryset
|
||||
|
||||
aggregate_attack_surface(str(tenant.id), str(scan.id))
|
||||
|
||||
mock_bulk_create.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.scan.AttackSurfaceOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.Finding.all_objects.filter")
|
||||
@patch("tasks.jobs.scan._get_attack_surface_mapping_from_provider")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_attack_surface_aggregates_counts_correctly(
|
||||
self,
|
||||
mock_rls_transaction,
|
||||
mock_get_mapping,
|
||||
mock_findings_filter,
|
||||
mock_bulk_create,
|
||||
tenants_fixture,
|
||||
scans_fixture,
|
||||
):
|
||||
"""Test that counts from multiple check_ids are aggregated per attack surface type."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
scan.provider.provider = "aws"
|
||||
scan.provider.save()
|
||||
|
||||
mock_get_mapping.return_value = {
|
||||
"internet-exposed": {"check_internet_1", "check_internet_2"},
|
||||
"secrets": set(),
|
||||
"privilege-escalation": set(),
|
||||
"ec2-imdsv1": set(),
|
||||
}
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.values.return_value = mock_queryset
|
||||
mock_queryset.annotate.return_value = [
|
||||
{"check_id": "check_internet_1", "total": 10, "failed": 3, "muted": 1},
|
||||
{"check_id": "check_internet_2", "total": 5, "failed": 2, "muted": 0},
|
||||
]
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
mock_findings_filter.return_value = mock_queryset
|
||||
|
||||
aggregate_attack_surface(str(tenant.id), str(scan.id))
|
||||
|
||||
args, kwargs = mock_bulk_create.call_args
|
||||
objects = args[0]
|
||||
|
||||
assert len(objects) == 1
|
||||
overview = objects[0]
|
||||
assert overview.attack_surface_type == "internet-exposed"
|
||||
assert overview.total_findings == 15 # 10 + 5
|
||||
assert overview.failed_findings == 5 # 3 + 2
|
||||
assert overview.muted_failed_findings == 1 # 1 + 0
|
||||
|
||||
@patch("tasks.jobs.scan.Scan.all_objects.select_related")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_attack_surface_uses_select_related(
|
||||
self, mock_rls_transaction, mock_select_related, tenants_fixture, scans_fixture
|
||||
):
|
||||
"""Test that select_related is used to avoid N+1 query."""
|
||||
tenant = tenants_fixture[0]
|
||||
scan = scans_fixture[0]
|
||||
|
||||
mock_scan = MagicMock()
|
||||
mock_scan.provider.provider = "aws"
|
||||
|
||||
mock_select_related.return_value.get.return_value = mock_scan
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.scan._get_attack_surface_mapping_from_provider"
|
||||
) as mock_map:
|
||||
mock_map.return_value = {}
|
||||
|
||||
aggregate_attack_surface(str(tenant.id), str(scan.id))
|
||||
|
||||
mock_select_related.assert_called_once_with("provider")
|
||||
|
||||
|
||||
class TestAggregateCategoryCounts:
|
||||
"""Test aggregate_category_counts helper function."""
|
||||
|
||||
def test_aggregate_category_counts_basic(self):
|
||||
"""Test basic category counting for a non-muted PASS finding."""
|
||||
cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
aggregate_category_counts(
|
||||
categories=["security", "iam"],
|
||||
severity="high",
|
||||
status="PASS",
|
||||
delta=None,
|
||||
muted=False,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
assert ("security", "high") in cache
|
||||
assert ("iam", "high") in cache
|
||||
assert cache[("security", "high")] == {"total": 1, "failed": 0, "new_failed": 0}
|
||||
assert cache[("iam", "high")] == {"total": 1, "failed": 0, "new_failed": 0}
|
||||
|
||||
def test_aggregate_category_counts_fail_not_muted(self):
|
||||
"""Test category counting for a non-muted FAIL finding."""
|
||||
cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
aggregate_category_counts(
|
||||
categories=["security"],
|
||||
severity="critical",
|
||||
status="FAIL",
|
||||
delta=None,
|
||||
muted=False,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
assert cache[("security", "critical")] == {
|
||||
"total": 1,
|
||||
"failed": 1,
|
||||
"new_failed": 0,
|
||||
}
|
||||
|
||||
def test_aggregate_category_counts_new_fail(self):
|
||||
"""Test category counting for a new FAIL finding (delta='new')."""
|
||||
cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
aggregate_category_counts(
|
||||
categories=["gen-ai"],
|
||||
severity="high",
|
||||
status="FAIL",
|
||||
delta="new",
|
||||
muted=False,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
assert cache[("gen-ai", "high")] == {"total": 1, "failed": 1, "new_failed": 1}
|
||||
|
||||
def test_aggregate_category_counts_muted_finding(self):
|
||||
"""Test that muted findings are excluded from all counts."""
|
||||
cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
aggregate_category_counts(
|
||||
categories=["security"],
|
||||
severity="high",
|
||||
status="FAIL",
|
||||
delta="new",
|
||||
muted=True,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
assert cache[("security", "high")] == {"total": 0, "failed": 0, "new_failed": 0}
|
||||
|
||||
def test_aggregate_category_counts_accumulates(self):
|
||||
"""Test that multiple calls accumulate counts."""
|
||||
cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
|
||||
# First finding: PASS
|
||||
aggregate_category_counts(
|
||||
categories=["security"],
|
||||
severity="high",
|
||||
status="PASS",
|
||||
delta=None,
|
||||
muted=False,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
# Second finding: FAIL (new)
|
||||
aggregate_category_counts(
|
||||
categories=["security"],
|
||||
severity="high",
|
||||
status="FAIL",
|
||||
delta="new",
|
||||
muted=False,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
# Third finding: FAIL (changed)
|
||||
aggregate_category_counts(
|
||||
categories=["security"],
|
||||
severity="high",
|
||||
status="FAIL",
|
||||
delta="changed",
|
||||
muted=False,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
assert cache[("security", "high")] == {"total": 3, "failed": 2, "new_failed": 1}
|
||||
|
||||
def test_aggregate_category_counts_empty_categories(self):
|
||||
"""Test with empty categories list."""
|
||||
cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
aggregate_category_counts(
|
||||
categories=[],
|
||||
severity="high",
|
||||
status="FAIL",
|
||||
delta="new",
|
||||
muted=False,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
assert cache == {}
|
||||
|
||||
def test_aggregate_category_counts_changed_delta(self):
|
||||
"""Test that changed delta increments failed but not new_failed."""
|
||||
cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
aggregate_category_counts(
|
||||
categories=["iam"],
|
||||
severity="medium",
|
||||
status="FAIL",
|
||||
delta="changed",
|
||||
muted=False,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
assert cache[("iam", "medium")] == {"total": 1, "failed": 1, "new_failed": 0}
|
||||
|
||||
def test_aggregate_category_counts_multiple_categories_single_finding(self):
|
||||
"""Test single finding with multiple categories."""
|
||||
cache: dict[tuple[str, str], dict[str, int]] = {}
|
||||
aggregate_category_counts(
|
||||
categories=["security", "compliance", "data-protection"],
|
||||
severity="low",
|
||||
status="FAIL",
|
||||
delta="new",
|
||||
muted=False,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
assert len(cache) == 3
|
||||
for cat in ["security", "compliance", "data-protection"]:
|
||||
assert cache[(cat, "low")] == {"total": 1, "failed": 1, "new_failed": 1}
|
||||
|
||||
@@ -4,7 +4,13 @@ from unittest.mock import MagicMock, patch
|
||||
import openai
|
||||
import pytest
|
||||
from botocore.exceptions import ClientError
|
||||
from django_celery_beat.models import IntervalSchedule, PeriodicTask
|
||||
from tasks.jobs.lighthouse_providers import (
|
||||
_create_bedrock_client,
|
||||
_extract_bedrock_credentials,
|
||||
)
|
||||
from tasks.tasks import (
|
||||
_cleanup_orphan_scheduled_scans,
|
||||
_perform_scan_complete_tasks,
|
||||
check_integrations_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
@@ -18,9 +24,203 @@ from api.models import (
|
||||
Integration,
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseProviderModels,
|
||||
Scan,
|
||||
StateChoices,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestExtractBedrockCredentials:
|
||||
"""Unit tests for _extract_bedrock_credentials helper function."""
|
||||
|
||||
def test_extract_access_key_credentials(self, tenants_fixture):
|
||||
"""Test extraction of access key + secret key credentials."""
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "us-east-1",
|
||||
}
|
||||
provider_cfg.save()
|
||||
|
||||
result = _extract_bedrock_credentials(provider_cfg)
|
||||
|
||||
assert result is not None
|
||||
assert result["access_key_id"] == "AKIAIOSFODNN7EXAMPLE"
|
||||
assert result["secret_access_key"] == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
assert result["region"] == "us-east-1"
|
||||
assert "api_key" not in result
|
||||
|
||||
def test_extract_api_key_credentials(self, tenants_fixture):
|
||||
"""Test extraction of API key (bearer token) credentials."""
|
||||
valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110)
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {
|
||||
"api_key": valid_api_key,
|
||||
"region": "us-west-2",
|
||||
}
|
||||
provider_cfg.save()
|
||||
|
||||
result = _extract_bedrock_credentials(provider_cfg)
|
||||
|
||||
assert result is not None
|
||||
assert result["api_key"] == valid_api_key
|
||||
assert result["region"] == "us-west-2"
|
||||
assert "access_key_id" not in result
|
||||
assert "secret_access_key" not in result
|
||||
|
||||
def test_api_key_takes_precedence_over_access_keys(self, tenants_fixture):
|
||||
"""Test that API key is preferred when both auth methods are present."""
|
||||
valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("B" * 110)
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {
|
||||
"api_key": valid_api_key,
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "eu-west-1",
|
||||
}
|
||||
provider_cfg.save()
|
||||
|
||||
result = _extract_bedrock_credentials(provider_cfg)
|
||||
|
||||
assert result is not None
|
||||
assert result["api_key"] == valid_api_key
|
||||
assert result["region"] == "eu-west-1"
|
||||
assert "access_key_id" not in result
|
||||
|
||||
def test_missing_region_returns_none(self, tenants_fixture):
|
||||
"""Test that missing region returns None."""
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {
|
||||
"api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110),
|
||||
}
|
||||
provider_cfg.save()
|
||||
|
||||
result = _extract_bedrock_credentials(provider_cfg)
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_empty_credentials_returns_none(self, tenants_fixture):
|
||||
"""Test that empty credentials dict returns None (region only is not enough)."""
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
|
||||
is_active=True,
|
||||
)
|
||||
# Only region, no auth credentials - should return None
|
||||
provider_cfg.credentials_decoded = {
|
||||
"region": "us-east-1",
|
||||
}
|
||||
provider_cfg.save()
|
||||
|
||||
result = _extract_bedrock_credentials(provider_cfg)
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_non_dict_credentials_returns_none(self, tenants_fixture):
|
||||
"""Test that non-dict credentials returns None."""
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
|
||||
is_active=True,
|
||||
)
|
||||
# Store valid credentials first to pass model validation
|
||||
provider_cfg.credentials_decoded = {
|
||||
"api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110),
|
||||
"region": "us-east-1",
|
||||
}
|
||||
provider_cfg.save()
|
||||
|
||||
# Mock the credentials_decoded property to return a non-dict value
|
||||
# This simulates corrupted/invalid stored data
|
||||
with patch.object(
|
||||
type(provider_cfg),
|
||||
"credentials_decoded",
|
||||
new_callable=lambda: property(lambda self: "invalid"),
|
||||
):
|
||||
result = _extract_bedrock_credentials(provider_cfg)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestCreateBedrockClient:
|
||||
"""Unit tests for _create_bedrock_client helper function."""
|
||||
|
||||
@patch("tasks.jobs.lighthouse_providers.boto3.client")
|
||||
def test_create_client_with_access_keys(self, mock_boto_client):
|
||||
"""Test creating client with access key authentication."""
|
||||
mock_client = MagicMock()
|
||||
mock_boto_client.return_value = mock_client
|
||||
|
||||
creds = {
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "us-east-1",
|
||||
}
|
||||
|
||||
result = _create_bedrock_client(creds)
|
||||
|
||||
assert result == mock_client
|
||||
mock_boto_client.assert_called_once_with(
|
||||
service_name="bedrock",
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
|
||||
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.lighthouse_providers.Config")
|
||||
@patch("tasks.jobs.lighthouse_providers.boto3.client")
|
||||
def test_create_client_with_api_key(self, mock_boto_client, mock_config):
|
||||
"""Test creating client with API key authentication."""
|
||||
mock_client = MagicMock()
|
||||
mock_events = MagicMock()
|
||||
mock_client.meta.events = mock_events
|
||||
mock_boto_client.return_value = mock_client
|
||||
mock_config_instance = MagicMock()
|
||||
mock_config.return_value = mock_config_instance
|
||||
valid_api_key = "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110)
|
||||
|
||||
creds = {
|
||||
"api_key": valid_api_key,
|
||||
"region": "us-west-2",
|
||||
}
|
||||
|
||||
result = _create_bedrock_client(creds)
|
||||
|
||||
assert result == mock_client
|
||||
mock_boto_client.assert_called_once_with(
|
||||
service_name="bedrock",
|
||||
region_name="us-west-2",
|
||||
config=mock_config_instance,
|
||||
)
|
||||
mock_events.register.assert_called_once()
|
||||
call_args = mock_events.register.call_args
|
||||
assert call_args[0][0] == "before-send.*.*"
|
||||
|
||||
# Verify handler injects bearer token
|
||||
handler_fn = call_args[0][1]
|
||||
mock_request = MagicMock()
|
||||
mock_request.headers = {}
|
||||
handler_fn(mock_request)
|
||||
assert mock_request.headers["Authorization"] == f"Bearer {valid_api_key}"
|
||||
|
||||
|
||||
# TODO Move this to outputs/reports jobs
|
||||
@pytest.mark.django_db
|
||||
class TestGenerateOutputs:
|
||||
@@ -109,7 +309,6 @@ class TestGenerateOutputs:
|
||||
return_value=(
|
||||
"/tmp/test/out-dir",
|
||||
"/tmp/test/comp-dir",
|
||||
"/tmp/test/threat-dir",
|
||||
),
|
||||
),
|
||||
patch("tasks.tasks.Scan.all_objects.filter") as mock_scan_update,
|
||||
@@ -139,7 +338,7 @@ class TestGenerateOutputs:
|
||||
patch("tasks.tasks.Finding.all_objects.filter") as mock_findings,
|
||||
patch(
|
||||
"tasks.tasks._generate_output_directory",
|
||||
return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"),
|
||||
return_value=("/tmp/test/out", "/tmp/test/comp"),
|
||||
),
|
||||
patch("tasks.tasks.FindingOutput._transform_findings_stats"),
|
||||
patch("tasks.tasks.FindingOutput.transform_api_finding"),
|
||||
@@ -209,7 +408,7 @@ class TestGenerateOutputs:
|
||||
patch("tasks.tasks.Finding.all_objects.filter") as mock_findings,
|
||||
patch(
|
||||
"tasks.tasks._generate_output_directory",
|
||||
return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"),
|
||||
return_value=("/tmp/test/out", "/tmp/test/comp"),
|
||||
),
|
||||
patch(
|
||||
"tasks.tasks.FindingOutput._transform_findings_stats",
|
||||
@@ -289,7 +488,6 @@ class TestGenerateOutputs:
|
||||
return_value=(
|
||||
"/tmp/test/outdir",
|
||||
"/tmp/test/compdir",
|
||||
"/tmp/test/threatdir",
|
||||
),
|
||||
),
|
||||
patch("tasks.tasks._compress_output_files", return_value="outdir.zip"),
|
||||
@@ -368,7 +566,6 @@ class TestGenerateOutputs:
|
||||
return_value=(
|
||||
"/tmp/test/outdir",
|
||||
"/tmp/test/compdir",
|
||||
"/tmp/test/threatdir",
|
||||
),
|
||||
),
|
||||
patch("tasks.tasks.FindingOutput._transform_findings_stats"),
|
||||
@@ -436,7 +633,7 @@ class TestGenerateOutputs:
|
||||
patch("tasks.tasks.Finding.all_objects.filter") as mock_findings,
|
||||
patch(
|
||||
"tasks.tasks._generate_output_directory",
|
||||
return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"),
|
||||
return_value=("/tmp/test/out", "/tmp/test/comp"),
|
||||
),
|
||||
patch(
|
||||
"tasks.tasks.FindingOutput._transform_findings_stats",
|
||||
@@ -494,7 +691,7 @@ class TestGenerateOutputs:
|
||||
patch("tasks.tasks.Finding.all_objects.filter") as mock_findings,
|
||||
patch(
|
||||
"tasks.tasks._generate_output_directory",
|
||||
return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"),
|
||||
return_value=("/tmp/test/out", "/tmp/test/comp"),
|
||||
),
|
||||
patch("tasks.tasks.FindingOutput._transform_findings_stats"),
|
||||
patch("tasks.tasks.FindingOutput.transform_api_finding"),
|
||||
@@ -532,37 +729,55 @@ class TestGenerateOutputs:
|
||||
|
||||
|
||||
class TestScanCompleteTasks:
|
||||
@patch("tasks.tasks.aggregate_attack_surface_task.apply_async")
|
||||
@patch("tasks.tasks.create_compliance_requirements_task.apply_async")
|
||||
@patch("tasks.tasks.perform_scan_summary_task.si")
|
||||
@patch("tasks.tasks.generate_outputs_task.si")
|
||||
@patch("tasks.tasks.generate_threatscore_report_task.si")
|
||||
@patch("tasks.tasks.generate_compliance_reports_task.si")
|
||||
@patch("tasks.tasks.check_integrations_task.si")
|
||||
def test_scan_complete_tasks(
|
||||
self,
|
||||
mock_check_integrations_task,
|
||||
mock_threatscore_task,
|
||||
mock_compliance_reports_task,
|
||||
mock_outputs_task,
|
||||
mock_scan_summary_task,
|
||||
mock_compliance_tasks,
|
||||
mock_compliance_requirements_task,
|
||||
mock_attack_surface_task,
|
||||
):
|
||||
"""Test that scan complete tasks are properly orchestrated with optimized reports."""
|
||||
_perform_scan_complete_tasks("tenant-id", "scan-id", "provider-id")
|
||||
mock_compliance_tasks.assert_called_once_with(
|
||||
|
||||
# Verify compliance requirements task is called
|
||||
mock_compliance_requirements_task.assert_called_once_with(
|
||||
kwargs={"tenant_id": "tenant-id", "scan_id": "scan-id"},
|
||||
)
|
||||
|
||||
# Verify attack surface task is called
|
||||
mock_attack_surface_task.assert_called_once_with(
|
||||
kwargs={"tenant_id": "tenant-id", "scan_id": "scan-id"},
|
||||
)
|
||||
|
||||
# Verify scan summary task is called
|
||||
mock_scan_summary_task.assert_called_once_with(
|
||||
scan_id="scan-id",
|
||||
tenant_id="tenant-id",
|
||||
)
|
||||
|
||||
# Verify outputs task is called
|
||||
mock_outputs_task.assert_called_once_with(
|
||||
scan_id="scan-id",
|
||||
provider_id="provider-id",
|
||||
tenant_id="tenant-id",
|
||||
)
|
||||
mock_threatscore_task.assert_called_once_with(
|
||||
|
||||
# Verify optimized compliance reports task is called (replaces individual tasks)
|
||||
mock_compliance_reports_task.assert_called_once_with(
|
||||
tenant_id="tenant-id",
|
||||
scan_id="scan-id",
|
||||
provider_id="provider-id",
|
||||
)
|
||||
|
||||
# Verify integrations task is called
|
||||
mock_check_integrations_task.assert_called_once_with(
|
||||
tenant_id="tenant-id",
|
||||
provider_id="provider-id",
|
||||
@@ -738,7 +953,7 @@ class TestCheckIntegrationsTask:
|
||||
mock_initialize_provider.return_value = MagicMock()
|
||||
mock_compliance_bulk.return_value = {}
|
||||
mock_get_frameworks.return_value = []
|
||||
mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir")
|
||||
mock_generate_dir.return_value = ("out-dir", "comp-dir")
|
||||
mock_transform_stats.return_value = {"stats": "data"}
|
||||
|
||||
# Mock findings
|
||||
@@ -863,7 +1078,7 @@ class TestCheckIntegrationsTask:
|
||||
mock_initialize_provider.return_value = MagicMock()
|
||||
mock_compliance_bulk.return_value = {}
|
||||
mock_get_frameworks.return_value = []
|
||||
mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir")
|
||||
mock_generate_dir.return_value = ("out-dir", "comp-dir")
|
||||
mock_transform_stats.return_value = {"stats": "data"}
|
||||
|
||||
# Mock findings
|
||||
@@ -979,7 +1194,7 @@ class TestCheckIntegrationsTask:
|
||||
mock_initialize_provider.return_value = MagicMock()
|
||||
mock_compliance_bulk.return_value = {}
|
||||
mock_get_frameworks.return_value = []
|
||||
mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir")
|
||||
mock_generate_dir.return_value = ("out-dir", "comp-dir")
|
||||
mock_transform_stats.return_value = {"stats": "data"}
|
||||
|
||||
# Mock findings
|
||||
@@ -1137,6 +1352,16 @@ class TestCheckLighthouseProviderConnectionTask:
|
||||
None,
|
||||
{"connected": True, "error": None},
|
||||
),
|
||||
# Bedrock API key authentication
|
||||
(
|
||||
LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
|
||||
{
|
||||
"api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110),
|
||||
"region": "us-east-1",
|
||||
},
|
||||
None,
|
||||
{"connected": True, "error": None},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_check_connection_success_all_providers(
|
||||
@@ -1205,6 +1430,24 @@ class TestCheckLighthouseProviderConnectionTask:
|
||||
"list_foundation_models",
|
||||
),
|
||||
),
|
||||
# Bedrock API key authentication failure
|
||||
(
|
||||
LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
|
||||
{
|
||||
"api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("X" * 110),
|
||||
"region": "us-east-1",
|
||||
},
|
||||
None,
|
||||
ClientError(
|
||||
{
|
||||
"Error": {
|
||||
"Code": "UnrecognizedClientException",
|
||||
"Message": "Invalid API key",
|
||||
}
|
||||
},
|
||||
"list_foundation_models",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_check_connection_api_failure(
|
||||
@@ -1329,6 +1572,17 @@ class TestRefreshLighthouseProviderModelsTask:
|
||||
{"openai.gpt-oss-120b-1:0": "gpt-oss-120b"},
|
||||
1,
|
||||
),
|
||||
# Bedrock API key authentication
|
||||
(
|
||||
LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK,
|
||||
{
|
||||
"api_key": "ABSKQmVkcm9ja0FQSUtleS" + ("A" * 110),
|
||||
"region": "us-east-1",
|
||||
},
|
||||
None,
|
||||
{"anthropic.claude-v3": "Claude 3"},
|
||||
1,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_refresh_models_create_new(
|
||||
@@ -1465,3 +1719,343 @@ class TestRefreshLighthouseProviderModelsTask:
|
||||
assert result["deleted"] == 0
|
||||
assert "error" in result
|
||||
assert result["error"] is not None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestCleanupOrphanScheduledScans:
|
||||
"""Unit tests for _cleanup_orphan_scheduled_scans helper function."""
|
||||
|
||||
def _create_periodic_task(self, provider_id, tenant_id):
|
||||
"""Helper to create a PeriodicTask for testing."""
|
||||
interval, _ = IntervalSchedule.objects.get_or_create(every=24, period="hours")
|
||||
return PeriodicTask.objects.create(
|
||||
name=f"scan-perform-scheduled-{provider_id}",
|
||||
task="scan-perform-scheduled",
|
||||
interval=interval,
|
||||
kwargs=f'{{"tenant_id": "{tenant_id}", "provider_id": "{provider_id}"}}',
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
def test_cleanup_deletes_orphan_when_both_available_and_scheduled_exist(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Test that AVAILABLE scan is deleted when SCHEDULED also exists."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
|
||||
# Create orphan AVAILABLE scan
|
||||
orphan_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Create SCHEDULED scan (next execution)
|
||||
scheduled_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Execute cleanup
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Verify orphan was deleted
|
||||
assert deleted_count == 1
|
||||
assert not Scan.objects.filter(id=orphan_scan.id).exists()
|
||||
assert Scan.objects.filter(id=scheduled_scan.id).exists()
|
||||
|
||||
def test_cleanup_does_not_delete_when_only_available_exists(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Test that AVAILABLE scan is NOT deleted when no SCHEDULED exists."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
|
||||
# Create only AVAILABLE scan (normal first scan scenario)
|
||||
available_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Execute cleanup
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Verify nothing was deleted
|
||||
assert deleted_count == 0
|
||||
assert Scan.objects.filter(id=available_scan.id).exists()
|
||||
|
||||
def test_cleanup_does_not_delete_when_only_scheduled_exists(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Test that nothing is deleted when only SCHEDULED exists."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
|
||||
# Create only SCHEDULED scan (normal subsequent scan scenario)
|
||||
scheduled_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Execute cleanup
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Verify nothing was deleted
|
||||
assert deleted_count == 0
|
||||
assert Scan.objects.filter(id=scheduled_scan.id).exists()
|
||||
|
||||
def test_cleanup_returns_zero_when_no_scans_exist(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Test that cleanup returns 0 when no scans exist."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
|
||||
# Execute cleanup with no scans
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
assert deleted_count == 0
|
||||
|
||||
def test_cleanup_deletes_multiple_orphan_available_scans(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Test that multiple AVAILABLE orphan scans are all deleted."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
|
||||
# Create multiple orphan AVAILABLE scans
|
||||
orphan_scan_1 = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
orphan_scan_2 = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Create SCHEDULED scan
|
||||
scheduled_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Execute cleanup
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Verify all orphans were deleted
|
||||
assert deleted_count == 2
|
||||
assert not Scan.objects.filter(id=orphan_scan_1.id).exists()
|
||||
assert not Scan.objects.filter(id=orphan_scan_2.id).exists()
|
||||
assert Scan.objects.filter(id=scheduled_scan.id).exists()
|
||||
|
||||
def test_cleanup_does_not_affect_different_provider(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Test that cleanup only affects scans for the specified provider."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider1 = providers_fixture[0]
|
||||
provider2 = providers_fixture[1]
|
||||
periodic_task1 = self._create_periodic_task(provider1.id, tenant.id)
|
||||
periodic_task2 = self._create_periodic_task(provider2.id, tenant.id)
|
||||
|
||||
# Create orphan scenario for provider1
|
||||
orphan_scan_p1 = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider1,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task1.id,
|
||||
)
|
||||
scheduled_scan_p1 = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider1,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=periodic_task1.id,
|
||||
)
|
||||
|
||||
# Create AVAILABLE scan for provider2 (should not be affected)
|
||||
available_scan_p2 = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider2,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task2.id,
|
||||
)
|
||||
|
||||
# Execute cleanup for provider1 only
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider1.id),
|
||||
scheduler_task_id=periodic_task1.id,
|
||||
)
|
||||
|
||||
# Verify only provider1's orphan was deleted
|
||||
assert deleted_count == 1
|
||||
assert not Scan.objects.filter(id=orphan_scan_p1.id).exists()
|
||||
assert Scan.objects.filter(id=scheduled_scan_p1.id).exists()
|
||||
assert Scan.objects.filter(id=available_scan_p2.id).exists()
|
||||
|
||||
def test_cleanup_does_not_affect_manual_scans(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Test that cleanup only affects SCHEDULED trigger scans, not MANUAL."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
|
||||
# Create orphan AVAILABLE scheduled scan
|
||||
orphan_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Create SCHEDULED scan
|
||||
scheduled_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Create AVAILABLE manual scan (should not be affected)
|
||||
manual_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Manual scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
|
||||
# Execute cleanup
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
# Verify only scheduled orphan was deleted
|
||||
assert deleted_count == 1
|
||||
assert not Scan.objects.filter(id=orphan_scan.id).exists()
|
||||
assert Scan.objects.filter(id=scheduled_scan.id).exists()
|
||||
assert Scan.objects.filter(id=manual_scan.id).exists()
|
||||
|
||||
def test_cleanup_does_not_affect_different_scheduler_task(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Test that cleanup only affects scans with the specified scheduler_task_id."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task1 = self._create_periodic_task(provider.id, tenant.id)
|
||||
|
||||
# Create another periodic task
|
||||
interval, _ = IntervalSchedule.objects.get_or_create(every=24, period="hours")
|
||||
periodic_task2 = PeriodicTask.objects.create(
|
||||
name=f"scan-perform-scheduled-other-{provider.id}",
|
||||
task="scan-perform-scheduled",
|
||||
interval=interval,
|
||||
kwargs=f'{{"tenant_id": "{tenant.id}", "provider_id": "{provider.id}"}}',
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
# Create orphan scenario for periodic_task1
|
||||
orphan_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task1.id,
|
||||
)
|
||||
scheduled_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=periodic_task1.id,
|
||||
)
|
||||
|
||||
# Create AVAILABLE scan for periodic_task2 (should not be affected)
|
||||
available_scan_other_task = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task2.id,
|
||||
)
|
||||
|
||||
# Execute cleanup for periodic_task1 only
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
scheduler_task_id=periodic_task1.id,
|
||||
)
|
||||
|
||||
# Verify only periodic_task1's orphan was deleted
|
||||
assert deleted_count == 1
|
||||
assert not Scan.objects.filter(id=orphan_scan.id).exists()
|
||||
assert Scan.objects.filter(id=scheduled_scan.id).exists()
|
||||
assert Scan.objects.filter(id=available_scan_other_task.id).exists()
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 90 KiB |
@@ -0,0 +1,24 @@
|
||||
import warnings
|
||||
|
||||
from dashboard.common_methods import get_section_containers_cis
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
def get_table(data):
|
||||
aux = data[
|
||||
[
|
||||
"REQUIREMENTS_ID",
|
||||
"REQUIREMENTS_DESCRIPTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SECTION",
|
||||
"CHECKID",
|
||||
"STATUS",
|
||||
"REGION",
|
||||
"ACCOUNTID",
|
||||
"RESOURCEID",
|
||||
]
|
||||
].copy()
|
||||
|
||||
return get_section_containers_cis(
|
||||
aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION"
|
||||
)
|
||||
@@ -0,0 +1,28 @@
|
||||
import warnings
|
||||
|
||||
from dashboard.common_methods import get_section_containers_threatscore
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
def get_table(data):
|
||||
aux = data[
|
||||
[
|
||||
"REQUIREMENTS_ID",
|
||||
"REQUIREMENTS_DESCRIPTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SECTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
|
||||
"CHECKID",
|
||||
"STATUS",
|
||||
"REGION",
|
||||
"ACCOUNTID",
|
||||
"RESOURCEID",
|
||||
]
|
||||
].copy()
|
||||
|
||||
return get_section_containers_threatscore(
|
||||
aux,
|
||||
"REQUIREMENTS_ATTRIBUTES_SECTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
|
||||
"REQUIREMENTS_ID",
|
||||
)
|
||||
@@ -0,0 +1,28 @@
|
||||
import warnings
|
||||
|
||||
from dashboard.common_methods import get_section_containers_threatscore
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
def get_table(data):
|
||||
aux = data[
|
||||
[
|
||||
"REQUIREMENTS_ID",
|
||||
"REQUIREMENTS_DESCRIPTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SECTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
|
||||
"CHECKID",
|
||||
"STATUS",
|
||||
"REGION",
|
||||
"ACCOUNTID",
|
||||
"RESOURCEID",
|
||||
]
|
||||
].copy()
|
||||
|
||||
return get_section_containers_threatscore(
|
||||
aux,
|
||||
"REQUIREMENTS_ATTRIBUTES_SECTION",
|
||||
"REQUIREMENTS_ATTRIBUTES_SUBSECTION",
|
||||
"REQUIREMENTS_ID",
|
||||
)
|
||||
@@ -61,6 +61,7 @@ def create_layout_overview(
|
||||
html.Div(className="flex", id="gcp_card", n_clicks=0),
|
||||
html.Div(className="flex", id="k8s_card", n_clicks=0),
|
||||
html.Div(className="flex", id="m365_card", n_clicks=0),
|
||||
html.Div(className="flex", id="alibabacloud_card", n_clicks=0),
|
||||
],
|
||||
className=f"grid gap-x-4 mb-[30px] sm:grid-cols-2 lg:grid-cols-{amount_providers}",
|
||||
),
|
||||
|
||||
@@ -78,6 +78,8 @@ def load_csv_files(csv_files):
|
||||
result = result.replace("_KUBERNETES", " - KUBERNETES")
|
||||
if "M65" in result:
|
||||
result = result.replace("_M65", " - M65")
|
||||
if "ALIBABACLOUD" in result:
|
||||
result = result.replace("_ALIBABACLOUD", " - ALIBABACLOUD")
|
||||
results.append(result)
|
||||
|
||||
unique_results = set(results)
|
||||
@@ -125,7 +127,7 @@ if data is None:
|
||||
)
|
||||
else:
|
||||
|
||||
data["ASSESSMENTDATE"] = pd.to_datetime(data["ASSESSMENTDATE"])
|
||||
data["ASSESSMENTDATE"] = pd.to_datetime(data["ASSESSMENTDATE"], format="mixed")
|
||||
data["ASSESSMENT_TIME"] = data["ASSESSMENTDATE"].dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
data_values = data["ASSESSMENT_TIME"].unique()
|
||||
@@ -278,9 +280,13 @@ def display_data(
|
||||
data["REQUIREMENTS_ATTRIBUTES_PROFILE"] = data[
|
||||
"REQUIREMENTS_ATTRIBUTES_PROFILE"
|
||||
].apply(lambda x: x.split(" - ")[0])
|
||||
|
||||
# Rename the column LOCATION to REGION for Alibaba Cloud
|
||||
if "alibabacloud" in analytics_input:
|
||||
data = data.rename(columns={"LOCATION": "REGION"})
|
||||
# Filter the chosen level of the CIS
|
||||
if is_level_1:
|
||||
data = data[data["REQUIREMENTS_ATTRIBUTES_PROFILE"] == "Level 1"]
|
||||
data = data[data["REQUIREMENTS_ATTRIBUTES_PROFILE"].str.contains("Level 1")]
|
||||
|
||||
# Rename the column PROJECTID to ACCOUNTID for GCP
|
||||
if data.columns.str.contains("PROJECTID").any():
|
||||
@@ -401,9 +407,11 @@ def display_data(
|
||||
compliance_module = importlib.import_module(
|
||||
f"dashboard.compliance.{current}"
|
||||
)
|
||||
data = data.drop_duplicates(
|
||||
subset=["CHECKID", "STATUS", "MUTED", "RESOURCEID", "STATUSEXTENDED"]
|
||||
)
|
||||
# Build subset list based on available columns
|
||||
dedup_columns = ["CHECKID", "STATUS", "RESOURCEID", "STATUSEXTENDED"]
|
||||
if "MUTED" in data.columns:
|
||||
dedup_columns.insert(2, "MUTED")
|
||||
data = data.drop_duplicates(subset=dedup_columns)
|
||||
|
||||
if "threatscore" in analytics_input:
|
||||
data = get_threatscore_mean_by_pillar(data)
|
||||
@@ -646,6 +654,7 @@ def get_table(current_compliance, table):
|
||||
def get_threatscore_mean_by_pillar(df):
|
||||
score_per_pillar = {}
|
||||
max_score_per_pillar = {}
|
||||
counted_findings_per_pillar = {}
|
||||
|
||||
for _, row in df.iterrows():
|
||||
pillar = (
|
||||
@@ -657,6 +666,18 @@ def get_threatscore_mean_by_pillar(df):
|
||||
if pillar not in score_per_pillar:
|
||||
score_per_pillar[pillar] = 0
|
||||
max_score_per_pillar[pillar] = 0
|
||||
counted_findings_per_pillar[pillar] = set()
|
||||
|
||||
# Skip muted findings for score calculation
|
||||
is_muted = "MUTED" in df.columns and row.get("MUTED") == "True"
|
||||
if is_muted:
|
||||
continue
|
||||
|
||||
# Create unique finding identifier to avoid counting duplicates
|
||||
finding_id = f"{row.get('CHECKID', '')}_{row.get('RESOURCEID', '')}"
|
||||
if finding_id in counted_findings_per_pillar[pillar]:
|
||||
continue
|
||||
counted_findings_per_pillar[pillar].add(finding_id)
|
||||
|
||||
level_of_risk = pd.to_numeric(
|
||||
row["REQUIREMENTS_ATTRIBUTES_LEVELOFRISK"], errors="coerce"
|
||||
@@ -700,6 +721,10 @@ def get_table_prowler_threatscore(df):
|
||||
score_per_pillar = {}
|
||||
max_score_per_pillar = {}
|
||||
pillars = {}
|
||||
counted_findings_per_pillar = {}
|
||||
counted_pass = set()
|
||||
counted_fail = set()
|
||||
counted_muted = set()
|
||||
|
||||
df_copy = df.copy()
|
||||
|
||||
@@ -714,6 +739,24 @@ def get_table_prowler_threatscore(df):
|
||||
pillars[pillar] = {"FAIL": 0, "PASS": 0, "MUTED": 0}
|
||||
score_per_pillar[pillar] = 0
|
||||
max_score_per_pillar[pillar] = 0
|
||||
counted_findings_per_pillar[pillar] = set()
|
||||
|
||||
# Create unique finding identifier
|
||||
finding_id = f"{row.get('CHECKID', '')}_{row.get('RESOURCEID', '')}"
|
||||
|
||||
# Check if muted
|
||||
is_muted = "MUTED" in df_copy.columns and row.get("MUTED") == "True"
|
||||
|
||||
# Count muted findings (separate from score calculation)
|
||||
if is_muted and finding_id not in counted_muted:
|
||||
counted_muted.add(finding_id)
|
||||
pillars[pillar]["MUTED"] += 1
|
||||
continue # Skip muted findings for score calculation
|
||||
|
||||
# Skip if already counted for this pillar
|
||||
if finding_id in counted_findings_per_pillar[pillar]:
|
||||
continue
|
||||
counted_findings_per_pillar[pillar].add(finding_id)
|
||||
|
||||
level_of_risk = pd.to_numeric(
|
||||
row["REQUIREMENTS_ATTRIBUTES_LEVELOFRISK"], errors="coerce"
|
||||
@@ -732,13 +775,14 @@ def get_table_prowler_threatscore(df):
|
||||
max_score_per_pillar[pillar] += level_of_risk * weight
|
||||
|
||||
if row["STATUS"] == "PASS":
|
||||
pillars[pillar]["PASS"] += 1
|
||||
if finding_id not in counted_pass:
|
||||
counted_pass.add(finding_id)
|
||||
pillars[pillar]["PASS"] += 1
|
||||
score_per_pillar[pillar] += level_of_risk * weight
|
||||
elif row["STATUS"] == "FAIL":
|
||||
pillars[pillar]["FAIL"] += 1
|
||||
|
||||
if "MUTED" in row and row["MUTED"] == "True":
|
||||
pillars[pillar]["MUTED"] += 1
|
||||
if finding_id not in counted_fail:
|
||||
counted_fail.add(finding_id)
|
||||
pillars[pillar]["FAIL"] += 1
|
||||
|
||||
result_df = []
|
||||
|
||||
|
||||
@@ -79,6 +79,9 @@ ks8_provider_logo = html.Img(
|
||||
m365_provider_logo = html.Img(
|
||||
src="assets/images/providers/m365_provider.png", alt="m365 provider"
|
||||
)
|
||||
alibabacloud_provider_logo = html.Img(
|
||||
src="assets/images/providers/alibabacloud_provider.png", alt="alibabacloud provider"
|
||||
)
|
||||
|
||||
|
||||
def load_csv_files(csv_files):
|
||||
@@ -253,6 +256,8 @@ else:
|
||||
accounts.append(account + " - AWS")
|
||||
if "kubernetes" in list(data[data["ACCOUNT_UID"] == account]["PROVIDER"]):
|
||||
accounts.append(account + " - K8S")
|
||||
if "alibabacloud" in list(data[data["ACCOUNT_UID"] == account]["PROVIDER"]):
|
||||
accounts.append(account + " - ALIBABACLOUD")
|
||||
|
||||
account_dropdown = create_account_dropdown(accounts)
|
||||
|
||||
@@ -298,6 +303,8 @@ else:
|
||||
services.append(service + " - GCP")
|
||||
if "m365" in list(data[data["SERVICE_NAME"] == service]["PROVIDER"]):
|
||||
services.append(service + " - M365")
|
||||
if "alibabacloud" in list(data[data["SERVICE_NAME"] == service]["PROVIDER"]):
|
||||
services.append(service + " - ALIBABACLOUD")
|
||||
|
||||
services = ["All"] + services
|
||||
services = [
|
||||
@@ -520,6 +527,7 @@ else:
|
||||
Output("gcp_card", "children"),
|
||||
Output("k8s_card", "children"),
|
||||
Output("m365_card", "children"),
|
||||
Output("alibabacloud_card", "children"),
|
||||
Output("subscribe_card", "children"),
|
||||
Output("info-file-over", "title"),
|
||||
Output("severity-filter", "value"),
|
||||
@@ -537,6 +545,7 @@ else:
|
||||
Output("gcp_card", "n_clicks"),
|
||||
Output("k8s_card", "n_clicks"),
|
||||
Output("m365_card", "n_clicks"),
|
||||
Output("alibabacloud_card", "n_clicks"),
|
||||
],
|
||||
Input("cloud-account-filter", "value"),
|
||||
Input("region-filter", "value"),
|
||||
@@ -560,6 +569,7 @@ else:
|
||||
Input("sort_button_region", "n_clicks"),
|
||||
Input("sort_button_service", "n_clicks"),
|
||||
Input("sort_button_account", "n_clicks"),
|
||||
Input("alibabacloud_card", "n_clicks"),
|
||||
)
|
||||
def filter_data(
|
||||
cloud_account_values,
|
||||
@@ -584,6 +594,7 @@ def filter_data(
|
||||
sort_button_region,
|
||||
sort_button_service,
|
||||
sort_button_account,
|
||||
alibabacloud_clicks,
|
||||
):
|
||||
# Use n_clicks for vulture
|
||||
n_clicks_csv = n_clicks_csv
|
||||
@@ -599,6 +610,7 @@ def filter_data(
|
||||
gcp_clicks = 0
|
||||
k8s_clicks = 0
|
||||
m365_clicks = 0
|
||||
alibabacloud_clicks = 0
|
||||
if azure_clicks > 0:
|
||||
filtered_data = data.copy()
|
||||
if azure_clicks % 2 != 0 and "azure" in list(data["PROVIDER"]):
|
||||
@@ -607,6 +619,7 @@ def filter_data(
|
||||
gcp_clicks = 0
|
||||
k8s_clicks = 0
|
||||
m365_clicks = 0
|
||||
alibabacloud_clicks = 0
|
||||
if gcp_clicks > 0:
|
||||
filtered_data = data.copy()
|
||||
if gcp_clicks % 2 != 0 and "gcp" in list(data["PROVIDER"]):
|
||||
@@ -615,6 +628,7 @@ def filter_data(
|
||||
azure_clicks = 0
|
||||
k8s_clicks = 0
|
||||
m365_clicks = 0
|
||||
alibabacloud_clicks = 0
|
||||
if k8s_clicks > 0:
|
||||
filtered_data = data.copy()
|
||||
if k8s_clicks % 2 != 0 and "kubernetes" in list(data["PROVIDER"]):
|
||||
@@ -623,6 +637,7 @@ def filter_data(
|
||||
azure_clicks = 0
|
||||
gcp_clicks = 0
|
||||
m365_clicks = 0
|
||||
alibabacloud_clicks = 0
|
||||
if m365_clicks > 0:
|
||||
filtered_data = data.copy()
|
||||
if m365_clicks % 2 != 0 and "m365" in list(data["PROVIDER"]):
|
||||
@@ -631,7 +646,16 @@ def filter_data(
|
||||
azure_clicks = 0
|
||||
gcp_clicks = 0
|
||||
k8s_clicks = 0
|
||||
|
||||
alibabacloud_clicks = 0
|
||||
if alibabacloud_clicks > 0:
|
||||
filtered_data = data.copy()
|
||||
if alibabacloud_clicks % 2 != 0 and "alibabacloud" in list(data["PROVIDER"]):
|
||||
filtered_data = filtered_data[filtered_data["PROVIDER"] == "alibabacloud"]
|
||||
aws_clicks = 0
|
||||
azure_clicks = 0
|
||||
gcp_clicks = 0
|
||||
k8s_clicks = 0
|
||||
m365_clicks = 0
|
||||
# For all the data, we will add to the status column the value 'MUTED (FAIL)' and 'MUTED (PASS)' depending on the value of the column 'STATUS' and 'MUTED'
|
||||
if "MUTED" in filtered_data.columns:
|
||||
filtered_data["STATUS"] = filtered_data.apply(
|
||||
@@ -723,6 +747,8 @@ def filter_data(
|
||||
all_account_ids.append(account)
|
||||
if "kubernetes" in list(data[data["ACCOUNT_UID"] == account]["PROVIDER"]):
|
||||
all_account_ids.append(account)
|
||||
if "alibabacloud" in list(data[data["ACCOUNT_UID"] == account]["PROVIDER"]):
|
||||
all_account_ids.append(account)
|
||||
|
||||
all_account_names = []
|
||||
if "ACCOUNT_NAME" in filtered_data.columns:
|
||||
@@ -745,6 +771,10 @@ def filter_data(
|
||||
cloud_accounts_options.append(item + " - AWS")
|
||||
if "kubernetes" in list(data[data["ACCOUNT_UID"] == item]["PROVIDER"]):
|
||||
cloud_accounts_options.append(item + " - K8S")
|
||||
if "alibabacloud" in list(
|
||||
data[data["ACCOUNT_UID"] == item]["PROVIDER"]
|
||||
):
|
||||
cloud_accounts_options.append(item + " - ALIBABACLOUD")
|
||||
if "ACCOUNT_NAME" in filtered_data.columns:
|
||||
if "azure" in list(data[data["ACCOUNT_NAME"] == item]["PROVIDER"]):
|
||||
cloud_accounts_options.append(item + " - AZURE")
|
||||
@@ -873,6 +903,10 @@ def filter_data(
|
||||
filtered_data[filtered_data["SERVICE_NAME"] == item]["PROVIDER"]
|
||||
):
|
||||
service_filter_options.append(item + " - M365")
|
||||
if "alibabacloud" in list(
|
||||
filtered_data[filtered_data["SERVICE_NAME"] == item]["PROVIDER"]
|
||||
):
|
||||
service_filter_options.append(item + " - ALIBABACLOUD")
|
||||
|
||||
# Filter Service
|
||||
if service_values == ["All"]:
|
||||
@@ -1324,6 +1358,12 @@ def filter_data(
|
||||
filtered_data.loc[
|
||||
filtered_data["ACCOUNT_UID"] == account, "ACCOUNT_UID"
|
||||
] = (account + " - M365")
|
||||
if "alibabacloud" in list(
|
||||
data[data["ACCOUNT_UID"] == account]["PROVIDER"]
|
||||
):
|
||||
filtered_data.loc[
|
||||
filtered_data["ACCOUNT_UID"] == account, "ACCOUNT_UID"
|
||||
] = (account + " - ALIBABACLOUD")
|
||||
|
||||
table_collapsible = []
|
||||
for item in filtered_data.to_dict("records"):
|
||||
@@ -1410,6 +1450,13 @@ def filter_data(
|
||||
else:
|
||||
m365_card = None
|
||||
|
||||
if "alibabacloud" in list(data["PROVIDER"].unique()):
|
||||
alibabacloud_card = create_provider_card(
|
||||
"alibabacloud", alibabacloud_provider_logo, "Accounts", full_filtered_data
|
||||
)
|
||||
else:
|
||||
alibabacloud_card = None
|
||||
|
||||
# Subscribe to Prowler Cloud card
|
||||
subscribe_card = [
|
||||
html.Div(
|
||||
@@ -1454,6 +1501,7 @@ def filter_data(
|
||||
gcp_card,
|
||||
k8s_card,
|
||||
m365_card,
|
||||
alibabacloud_card,
|
||||
subscribe_card,
|
||||
list_files,
|
||||
severity_values,
|
||||
@@ -1469,6 +1517,7 @@ def filter_data(
|
||||
gcp_clicks,
|
||||
k8s_clicks,
|
||||
m365_clicks,
|
||||
alibabacloud_clicks,
|
||||
)
|
||||
else:
|
||||
return (
|
||||
@@ -1487,6 +1536,7 @@ def filter_data(
|
||||
gcp_card,
|
||||
k8s_card,
|
||||
m365_card,
|
||||
alibabacloud_card,
|
||||
subscribe_card,
|
||||
list_files,
|
||||
severity_values,
|
||||
@@ -1504,6 +1554,7 @@ def filter_data(
|
||||
gcp_clicks,
|
||||
k8s_clicks,
|
||||
m365_clicks,
|
||||
alibabacloud_clicks,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+34
-1
@@ -41,6 +41,9 @@ services:
|
||||
volumes:
|
||||
- "./ui:/app"
|
||||
- "/app/node_modules"
|
||||
depends_on:
|
||||
mcp-server:
|
||||
condition: service_healthy
|
||||
|
||||
postgres:
|
||||
image: postgres:16.3-alpine3.20
|
||||
@@ -57,7 +60,11 @@ services:
|
||||
ports:
|
||||
- "${POSTGRES_PORT:-5432}:${POSTGRES_PORT:-5432}"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "sh -c 'pg_isready -U ${POSTGRES_ADMIN_USER} -d ${POSTGRES_DB}'"]
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"sh -c 'pg_isready -U ${POSTGRES_ADMIN_USER} -d ${POSTGRES_DB}'",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -118,6 +125,32 @@ services:
|
||||
- "../docker-entrypoint.sh"
|
||||
- "beat"
|
||||
|
||||
mcp-server:
|
||||
build:
|
||||
context: ./mcp_server
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
- PROWLER_MCP_TRANSPORT_MODE=http
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- ./mcp_server/prowler_mcp_server:/app/prowler_mcp_server
|
||||
- ./mcp_server/pyproject.toml:/app/pyproject.toml
|
||||
- ./mcp_server/entrypoint.sh:/app/entrypoint.sh
|
||||
command: ["uvicorn", "--host", "0.0.0.0", "--port", "8000"]
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"wget -q -O /dev/null http://127.0.0.1:8000/health || exit 1",
|
||||
]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
outputs:
|
||||
driver: local
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
# Production Docker Compose configuration
|
||||
# Uses pre-built images from Docker Hub (prowlercloud/*)
|
||||
#
|
||||
# For development with local builds and hot-reload, use docker-compose-dev.yml instead:
|
||||
# docker compose -f docker-compose-dev.yml up
|
||||
#
|
||||
services:
|
||||
api:
|
||||
hostname: "prowler-api"
|
||||
@@ -26,6 +32,9 @@ services:
|
||||
required: false
|
||||
ports:
|
||||
- ${UI_PORT:-3000}:${UI_PORT:-3000}
|
||||
depends_on:
|
||||
mcp-server:
|
||||
condition: service_healthy
|
||||
|
||||
postgres:
|
||||
image: postgres:16.3-alpine3.20
|
||||
@@ -93,6 +102,22 @@ services:
|
||||
- "../docker-entrypoint.sh"
|
||||
- "beat"
|
||||
|
||||
mcp-server:
|
||||
image: prowlercloud/prowler-mcp:${PROWLER_MCP_VERSION:-stable}
|
||||
environment:
|
||||
- PROWLER_MCP_TRANSPORT_MODE=http
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
ports:
|
||||
- "8000:8000"
|
||||
command: ["uvicorn", "--host", "0.0.0.0", "--port", "8000"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8000/health || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
output:
|
||||
driver: local
|
||||
|
||||
@@ -213,3 +213,5 @@ Also is important to keep all code examples as short as possible, including the
|
||||
| software-supply-chain | Detects or prevents tampering, unauthorized packages, or third-party risks in software supply chain |
|
||||
| e3 | M365-specific controls enabled by or dependent on an E3 license (e.g., baseline security policies, conditional access) |
|
||||
| e5 | M365-specific controls enabled by or dependent on an E5 license (e.g., advanced threat protection, audit, DLP, and eDiscovery) |
|
||||
| privilege-escalation | Detects IAM policies or permissions that allow identities to elevate their privileges beyond their intended scope, potentially gaining administrator or higher-level access through specific action combinations |
|
||||
| ec2-imdsv1 | Identifies EC2 instances using Instance Metadata Service version 1 (IMDSv1), which is vulnerable to SSRF attacks and should be replaced with IMDSv2 for enhanced security |
|
||||
@@ -312,7 +312,8 @@ The type of resource being audited. This field helps categorize and organize fin
|
||||
- **Azure**: Use types from [Azure Resource Graph](https://learn.microsoft.com/en-us/azure/governance/resource-graph/reference/supported-tables-resources), for example: `Microsoft.Storage/storageAccounts`.
|
||||
- **Google Cloud**: Use [Cloud Asset Inventory asset types](https://cloud.google.com/asset-inventory/docs/asset-types), for example: `compute.googleapis.com/Instance`.
|
||||
- **Kubernetes**: Use types shown under `KIND` from `kubectl api-resources`.
|
||||
- **M365 / GitHub**: Leave empty due to lack of standardized types.
|
||||
- **Oracle Cloud Infrastructure**: Use types from [Oracle Cloud Infrastructure documentation](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Search/Tasks/queryingresources_topic-Listing_Supported_Resource_Types.htm).
|
||||
- **M365 / GitHub / MongoDB Atlas**: Leave empty due to lack of standardized types.
|
||||
|
||||
#### Description
|
||||
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
---
|
||||
title: 'End-2-End Tests for Prowler App'
|
||||
---
|
||||
|
||||
End-to-end (E2E) tests validate complete user flows in Prowler App (UI + API). These tests are implemented with [Playwright](https://playwright.dev/) under the `ui/tests` folder and are designed to run against a Prowler App environment.
|
||||
|
||||
## General Recommendations
|
||||
|
||||
When adding or maintaining E2E tests for Prowler App, follow these guidelines:
|
||||
|
||||
1. **Test real user journeys**
|
||||
Focus on full workflows (for example, sign-up → login → add provider → launch scan) instead of low-level UI details already covered by unit or integration tests.
|
||||
|
||||
2. **Group tests by entity or feature area**
|
||||
- Organize E2E tests by entity or feature area (for example, `providers.spec.ts`, `scans.spec.ts`, `invitations.spec.ts`, `sign-up.spec.ts`).
|
||||
- Each entity should have its own test file and corresponding page model class (for example, `ProvidersPage`, `ScansPage`, `InvitationsPage`).
|
||||
- Related tests for the same entity should be grouped together in the same test file to improve maintainability and make it easier to find and update tests for a specific feature.
|
||||
|
||||
3. **Use a Page Model (Page Object Model)**
|
||||
- Encapsulate selectors and common actions in page classes instead of repeating them in each test.
|
||||
- Leverage and extend the existing Playwright page models in `ui/tests`—such as `ProvidersPage`, `ScansPage`, and others—which are all based on the shared `BasePage`.
|
||||
- Page models for Prowler App pages should be placed in their respective entity folders (for example, `ui/tests/providers/providers-page.ts`).
|
||||
- Page models for external pages (not part of Prowler App) should be grouped in the `external` folder (for example, `ui/tests/external/github-page.ts`).
|
||||
- This approach improves readability, reduces duplication, and makes refactors safer.
|
||||
|
||||
4. **Reuse authentication states (StorageState)**
|
||||
- Multiple authentication setup projects are available that generate pre-authenticated state files stored in `playwright/.auth/`. Each project requires specific environment variables:
|
||||
- `admin.auth.setup` – Admin users with full system permissions (requires `E2E_ADMIN_USER` / `E2E_ADMIN_PASSWORD`)
|
||||
- `manage-scans.auth.setup` – Users with scan management permissions (requires `E2E_MANAGE_SCANS_USER` / `E2E_MANAGE_SCANS_PASSWORD`)
|
||||
- `manage-integrations.auth.setup` – Users with integration management permissions (requires `E2E_MANAGE_INTEGRATIONS_USER` / `E2E_MANAGE_INTEGRATIONS_PASSWORD`)
|
||||
- `manage-account.auth.setup` – Users with account management permissions (requires `E2E_MANAGE_ACCOUNT_USER` / `E2E_MANAGE_ACCOUNT_PASSWORD`)
|
||||
- `manage-cloud-providers.auth.setup` – Users with cloud provider management permissions (requires `E2E_MANAGE_CLOUD_PROVIDERS_USER` / `E2E_MANAGE_CLOUD_PROVIDERS_PASSWORD`)
|
||||
- `unlimited-visibility.auth.setup` – Users with unlimited visibility permissions (requires `E2E_UNLIMITED_VISIBILITY_USER` / `E2E_UNLIMITED_VISIBILITY_PASSWORD`)
|
||||
- `invite-and-manage-users.auth.setup` – Users with user invitation and management permissions (requires `E2E_INVITE_AND_MANAGE_USERS_USER` / `E2E_INVITE_AND_MANAGE_USERS_PASSWORD`)
|
||||
<Note>
|
||||
If fixtures have been applied (fixtures are used to populate the database with initial development data), you can use the user `e2e@prowler.com` with password `Thisisapassword123@` to configure the Admin credentials by setting `E2E_ADMIN_USER=e2e@prowler.com` and `E2E_ADMIN_PASSWORD=Thisisapassword123@`.
|
||||
</Note>
|
||||
|
||||
- Within test files, use `test.use({ storageState: "playwright/.auth/admin_user.json" })` to load the pre-authenticated state, avoiding redundant authentication steps in each test. This must be placed at the test level (not inside the test function) to apply the authentication state to all tests in that scope. This approach is preferred over declaring dependencies in `playwright.config.ts` because it provides more control over which authentication states are used in specific tests.
|
||||
|
||||
**Example:**
|
||||
|
||||
```typescript
|
||||
// Use admin authentication state for all tests in this scope
|
||||
test.use({ storageState: "playwright/.auth/admin_user.json" });
|
||||
|
||||
test("should perform admin action", async ({ page }) => {
|
||||
// Test implementation
|
||||
});
|
||||
```
|
||||
|
||||
5. **Tag and document scenarios**
|
||||
- Follow the existing naming convention for suites and test cases (for example, `SCANS-E2E-001`, `PROVIDER-E2E-003`) and use tags such as `@e2e`, `@serial` and feature tags (for example, `@providers`, `@scans`,`@aws`) to filter and organize tests.
|
||||
|
||||
**Example:**
|
||||
```typescript
|
||||
test(
|
||||
"should add a new AWS provider with static credentials",
|
||||
{
|
||||
tag: [
|
||||
"@critical",
|
||||
"@e2e",
|
||||
"@providers",
|
||||
"@aws",
|
||||
"@serial",
|
||||
"@PROVIDER-E2E-001",
|
||||
],
|
||||
},
|
||||
async ({ page }) => {
|
||||
// Test implementation
|
||||
}
|
||||
);
|
||||
```
|
||||
- Document each one in the Markdown files under `ui/tests`, including **Priority**, **Tags**, **Description**, **Preconditions**, **Flow steps**, **Expected results**,**Key verification points** and **Notes**.
|
||||
|
||||
**Example**
|
||||
```Markdown
|
||||
## Test Case: `SCANS-E2E-001` - Execute On-Demand Scan
|
||||
|
||||
**Priority:** `critical`
|
||||
|
||||
**Tags:**
|
||||
|
||||
- type → @e2e, @serial
|
||||
- feature → @scans
|
||||
|
||||
**Description/Objective:** Validates the complete flow to execute an on-demand scan selecting a provider by UID and confirming success on the Scans page.
|
||||
|
||||
**Preconditions:**
|
||||
|
||||
- Admin user authentication required (admin.auth.setup setup)
|
||||
- Environment variables configured for : E2E_AWS_PROVIDER_ACCOUNT_ID,E2E_AWS_PROVIDER_ACCESS_KEY and E2E_AWS_PROVIDER_SECRET_KEY
|
||||
- Remove any existing AWS provider with the same Account ID before starting the test
|
||||
- This test must be run serially and never in parallel with other tests, as it requires the Account ID Provider to be already registered.
|
||||
|
||||
### Flow Steps:
|
||||
|
||||
1. Navigate to Scans page
|
||||
2. Open provider selector and choose the entry whose text contains E2E_AWS_PROVIDER_ACCOUNT_ID
|
||||
3. Optionally fill scan label (alias)
|
||||
4. Click "Start now" to launch the scan
|
||||
5. Verify the success toast appears
|
||||
6. Verify a row in the Scans table contains the provided scan label (or shows the new scan entry)
|
||||
|
||||
### Expected Result:
|
||||
|
||||
- Scan is launched successfully
|
||||
- Success toast is displayed to the user
|
||||
- Scans table displays the new scan entry (including the alias when provided)
|
||||
|
||||
### Key verification points:
|
||||
|
||||
- Scans page loads correctly
|
||||
- Provider select is available and lists the configured provider UID
|
||||
- "Start now" button is rendered and enabled when form is valid
|
||||
- Success toast message: "The scan was launched successfully."
|
||||
- Table contains a row with the scan label or new scan state (queued/available/executing)
|
||||
|
||||
### Notes:
|
||||
|
||||
- The table may take a short time to reflect the new scan; assertions look for a row containing the alias.
|
||||
- Provider cleanup performed before each test to ensure clean state
|
||||
- Tests should run serially to avoid state conflicts.
|
||||
|
||||
```
|
||||
|
||||
6. **Use environment variables for secrets and dynamic data**
|
||||
Credentials, provider identifiers, secrets, tokens must come from environment variables (for example, `E2E_AWS_PROVIDER_ACCOUNT_ID`, `E2E_AWS_PROVIDER_ACCESS_KEY`, `E2E_AWS_PROVIDER_SECRET_KEY`, `E2E_GCP_PROJECT_ID`).
|
||||
|
||||
<Warning>
|
||||
Never commit real secrets, tokens, or account IDs to the repository.
|
||||
</Warning>
|
||||
|
||||
7. **Keep tests deterministic and isolated**
|
||||
- Use Playwright's `test.beforeEach()` and `test.afterEach()` hooks to manage test state:
|
||||
- **`test.beforeEach()`**: Execute cleanup or setup logic before each test runs (for example, delete existing providers with a specific account ID to ensure a clean state).
|
||||
- **`test.afterEach()`**: Execute cleanup logic after each test completes (for example, remove test data created during the test execution to prevent interference with subsequent tests).
|
||||
- Define tests as serial using `test.describe.serial()` when they share state or resources that could interfere with parallel execution (for example, tests that use the same provider account ID or create dependent resources). This ensures tests within the serial group run sequentially, preventing race conditions and data conflicts.
|
||||
- Use unique identifiers (for example, random suffixes for emails or labels) to prevent data collisions.
|
||||
|
||||
8. **Use explicit waiting strategies**
|
||||
- Avoid using `waitForLoadState('networkidle')` as it is unreliable and can lead to flaky tests or unnecessary delays.
|
||||
- Leverage Playwright's auto-waiting capabilities by waiting for specific elements to be actionable (for example, `locator.click()`, `locator.fill()`, `locator.waitFor()`).
|
||||
- **Prioritize selector strategies**: Prefer `page.getByRole()` over other approaches like `page.getByText()`. `getByRole()` is more resilient to UI changes, aligns with accessibility best practices, and better reflects how users interact with the application (by role and accessible name rather than implementation details).
|
||||
- For dynamic content, wait for specific UI elements that indicate the page is ready (for example, button becoming enabled, a specific text appearing, etc).
|
||||
- This approach makes tests more reliable, faster, and aligned with how users actually interact with the application.
|
||||
|
||||
**Common waiting patterns used in Prowler E2E tests:**
|
||||
|
||||
- **Element visibility assertions**: Use `expect(locator).toBeVisible()` or `expect(locator).not.toBeVisible()` to wait for elements to appear or disappear (Playwright automatically waits for these conditions).
|
||||
|
||||
- **URL changes**: Use `expect(page).toHaveURL(url)` or `page.waitForURL(url)` to wait for navigation to complete.
|
||||
|
||||
- **Element states**: Use `locator.waitFor({ state: "visible" })` or `locator.waitFor({ state: "hidden" })` when you need explicit state control.
|
||||
|
||||
- **Text content**: Use `expect(locator).toHaveText(text)` or `expect(locator).toContainText(text)` to wait for specific text to appear.
|
||||
|
||||
- **Element attributes**: Use `expect(locator).toHaveAttribute(name, value)` to wait for attributes like `aria-disabled="false"` indicating a button is enabled.
|
||||
|
||||
- **Custom conditions**: Use `page.waitForFunction(() => condition)` for complex conditions that cannot be expressed with locators (for example, checking DOM element dimensions or computed styles).
|
||||
|
||||
- **Retryable assertions**: Use `expect(async () => { ... }).toPass({ timeout })` for conditions that may take time to stabilize (for example, waiting for table rows to filter after a server request).
|
||||
|
||||
- **Scroll into view**: Use `locator.scrollIntoViewIfNeeded()` before interacting with elements that may be outside the viewport.
|
||||
|
||||
**Example from Prowler tests:**
|
||||
|
||||
```typescript
|
||||
// Wait for page to load by checking main content is visible
|
||||
await expect(page.locator("main")).toBeVisible();
|
||||
|
||||
// Wait for URL change after form submission
|
||||
await expect(page).toHaveURL("/providers");
|
||||
|
||||
// Wait for button to become enabled
|
||||
await expect(submitButton).toHaveAttribute("aria-disabled", "false");
|
||||
|
||||
// Wait for loading spinner to disappear
|
||||
await expect(page.getByText("Loading")).not.toBeVisible();
|
||||
|
||||
// Wait for custom condition
|
||||
await page.waitForFunction(() => {
|
||||
const main = document.querySelector("main");
|
||||
return main && main.offsetHeight > 0;
|
||||
});
|
||||
|
||||
// Wait for retryable condition (e.g., table filtering)
|
||||
await expect(async () => {
|
||||
const rowCount = await tableRows.count();
|
||||
expect(rowCount).toBeLessThanOrEqual(1);
|
||||
}).toPass({ timeout: 20000 });
|
||||
```
|
||||
|
||||
## Running Prowler Tests
|
||||
|
||||
E2E tests for Prowler App run from the `ui` project using Playwright. The Playwright configuration lives in `ui/playwright.config.ts` and defines:
|
||||
|
||||
- `testDir: "./tests"` – location of E2E test files (relative to the `ui` project root, so `ui/tests`).
|
||||
- `webServer` – how to start the Next.js development server and connect to Prowler API.
|
||||
- `use.baseURL` – base URL for browser interactions (defaults to `http://localhost:3000` or `AUTH_URL` if set).
|
||||
- `reporter: [["list"]]` – uses the list reporter to display test results in a concise format in the terminal. Other reporter options are available (for example, `html`, `json`, `junit`, `github`), and multiple reporters can be configured simultaneously. See the [Playwright reporter documentation](https://playwright.dev/docs/test-reporters) for all available options.
|
||||
- `expect.timeout: 20000` – timeout for assertions (20 seconds). This is the maximum time Playwright will wait for an assertion to pass before considering it failed.
|
||||
- **Test artifacts** (in `use` configuration): By default, `trace`, `screenshot`, and `video` are set to `"off"` to minimize resource usage. To review test failures or debug issues, these can be enabled in `playwright.config.ts` by changing them to `"on"`, `"on-first-retry"`, or `"retain-on-failure"` depending on your needs.
|
||||
- `outputDir: "/tmp/playwright-tests"` – directory where Playwright stores test artifacts (screenshots, videos, traces) during test execution.
|
||||
- **CI-specific configuration**: The configuration uses different settings when running in CI environments (detected via `process.env.CI`):
|
||||
- **Retries**: `2` retries in CI (to handle flaky tests), `0` retries locally (for faster feedback during development).
|
||||
- **Workers**: `1` worker in CI (sequential execution for stability), `undefined` locally (parallel execution by default for faster test runs).
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Before running E2E tests:
|
||||
|
||||
- **Install root and UI dependencies**
|
||||
- Follow the [developer guide introduction](/developer-guide/introduction#getting-the-code-and-installing-all-dependencies) to clone the repository and install core dependencies.
|
||||
- From the `ui` directory, install frontend dependencies:
|
||||
|
||||
```bash
|
||||
cd ui
|
||||
pnpm install
|
||||
pnpm run test:e2e:install # Install Playwright browsers
|
||||
```
|
||||
|
||||
- **Ensure Prowler API is available**
|
||||
- By default, Playwright uses `NEXT_PUBLIC_API_BASE_URL=http://localhost:8080/api/v1` (configured in `playwright.config.ts`).
|
||||
- Start Prowler API so it is reachable on that URL (for example, via `docker-compose-dev.yml` or the development orchestration used locally).
|
||||
- If a different API URL is required, set `NEXT_PUBLIC_API_BASE_URL` accordingly before running the tests.
|
||||
|
||||
- **Ensure Prowler App UI is available**
|
||||
- Playwright automatically starts the Next.js server through the `webServer` block in `playwright.config.ts` (`pnpm run dev` by default).
|
||||
- If the UI is already running on `http://localhost:3000`, Playwright will reuse the existing server when `reuseExistingServer` is `true`.
|
||||
|
||||
- **Configure E2E environment variables**
|
||||
- Suite-specific variables (for example, provider account IDs, credentials, and E2E user data) must be provided before running tests.
|
||||
- They can be defined either:
|
||||
- As exported environment variables in the shell before executing the Playwright commands, or
|
||||
- In a `.env.local` or `.env` file under `ui/`, and then loaded into the shell before running tests, for example:
|
||||
|
||||
```bash
|
||||
cd ui
|
||||
set -a
|
||||
source .env.local # or .env
|
||||
set +a
|
||||
```
|
||||
- Refer to the Markdown documentation files in `ui/tests` for each E2E suite (for example, the `*.md` files that describe sign-up, providers, scans, invitations, and other flows) to see the exact list of required variables and their meaning.
|
||||
- Each E2E test suite explicitly checks that its required environment variables are defined at runtime and will fail with a clear error message if any mandatory variable is missing, making misconfiguration easy to detect.
|
||||
|
||||
### Executing Tests
|
||||
|
||||
To execute E2E tests for Prowler App:
|
||||
|
||||
1. **Run the full E2E suite (headless)**
|
||||
|
||||
From the `ui` directory:
|
||||
|
||||
```bash
|
||||
pnpm run test:e2e
|
||||
```
|
||||
|
||||
This command runs Playwright with the configured projects
|
||||
|
||||
2. **Run E2E tests with the Playwright UI runner**
|
||||
|
||||
```bash
|
||||
pnpm run test:e2e:ui
|
||||
```
|
||||
|
||||
This opens the Playwright test runner UI to inspect, debug, and rerun specific tests or projects.
|
||||
|
||||
3. **Debug E2E tests interactively**
|
||||
|
||||
```bash
|
||||
pnpm run test:e2e:debug
|
||||
```
|
||||
|
||||
Use this mode to step through flows, inspect selectors, and adjust timings. It runs tests in headed mode with debugging tools enabled.
|
||||
|
||||
4. **Run tests in headed mode without debugger**
|
||||
|
||||
```bash
|
||||
pnpm run test:e2e:headed
|
||||
```
|
||||
|
||||
This is useful to visually confirm flows while still running the full suite.
|
||||
|
||||
5. **View previous test reports**
|
||||
|
||||
```bash
|
||||
pnpm run test:e2e:report
|
||||
```
|
||||
|
||||
This opens the latest Playwright HTML report, including traces and screenshots when enabled.
|
||||
|
||||
6. **Run specific tests or subsets**
|
||||
|
||||
In addition to the predefined scripts, Playwright allows filtering which tests run. These examples use the Playwright CLI directly through `pnpm`:
|
||||
|
||||
- **By test ID (`@ID` in the test metadata or description)**
|
||||
|
||||
To run a single test case identified by its ID (for example, `@PROVIDER-E2E-001` or `@SCANS-E2E-001`):
|
||||
|
||||
```bash
|
||||
pnpm playwright test --grep @PROVIDER-E2E-001
|
||||
```
|
||||
|
||||
- **By tags**
|
||||
|
||||
To run all tests that share a common tag (for example, all provider E2E tests tagged with `@providers`):
|
||||
|
||||
```bash
|
||||
pnpm playwright test --grep @providers
|
||||
```
|
||||
|
||||
This is useful to focus on a specific feature area such as providers, scans, invitations, or sign-up.
|
||||
|
||||
- **By Playwright project**
|
||||
|
||||
To run only the tests associated with a given project defined in `playwright.config.ts` (for example, `providers` or `scans`):
|
||||
|
||||
```bash
|
||||
pnpm playwright test --project=providers
|
||||
```
|
||||
|
||||
Combining project and grep filters is also supported, enabling very narrow runs (for example, a single test ID within the `providers` project). For additional CLI options and combinations, see the [Playwright command line documentation](https://playwright.dev/docs/test-cli).
|
||||
|
||||
<Note>
|
||||
For detailed flows, preconditions, and environment variable requirements per feature, always refer to the Markdown files in `ui/tests`. Those documents are the single source of truth for business expectations and validation points in each E2E suite.
|
||||
</Note>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user