mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-05-20 03:02:43 +00:00
Compare commits
99 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8e40c2ceb1 | |||
| 71aec6aede | |||
| 9761651f8d | |||
| 406aace585 | |||
| ebd5814112 | |||
| 42e816081e | |||
| 741217ce80 | |||
| 5f9ab68bd9 | |||
| fba2854f65 | |||
| 8794515318 | |||
| 335db928dc | |||
| 046baa8eb9 | |||
| ef60ea99c3 | |||
| 1483efa18e | |||
| b74744b135 | |||
| e80eed6baf | |||
| 1ba22f6f45 | |||
| da6b7b89cb | |||
| cc9aa7f7ee | |||
| ecf749fce8 | |||
| 1a7f52fc9c | |||
| b630234cdf | |||
| d6685eec1f | |||
| 86cff92d1f | |||
| 28e81783ef | |||
| 13266b8743 | |||
| 4e143cf013 | |||
| 5cfe140b7b | |||
| c7d7ec9a3b | |||
| 155a1813cc | |||
| 71e444d4ae | |||
| 42b7f0f1a9 | |||
| 5b3f0fbd7f | |||
| 06eb69e455 | |||
| 338a11eaaf | |||
| 8814a0710a | |||
| 76a55cdb54 | |||
| 736badb284 | |||
| 37f77bb778 | |||
| 7e5e48c588 | |||
| 5f0017046f | |||
| 612d867838 | |||
| 8c2668ebe4 | |||
| be4b1bd99b | |||
| 502525eff1 | |||
| 09b5afe9c3 | |||
| 9a4fc784db | |||
| 04177db648 | |||
| 2408dbf855 | |||
| 9c4a8782e4 | |||
| 0d549ea39e | |||
| 0060081cad | |||
| 0c2d06dd9a | |||
| 14b9be4c47 | |||
| 6bac5650e6 | |||
| 6170462a61 | |||
| 2ad5926b13 | |||
| a6ddc85e4c | |||
| aceff35f29 | |||
| 3ae96c3aa6 | |||
| 0dcaaa9083 | |||
| 323a7f0349 | |||
| 736cbea862 | |||
| d3e290978e | |||
| 9c91cfcb7d | |||
| e279f7fcfd | |||
| a555cffebe | |||
| 49f5435392 | |||
| a087dd9b85 | |||
| 6e89c301b2 | |||
| d5dac448a6 | |||
| 00e6eb35f1 | |||
| cdb455b2b1 | |||
| 837c65ba23 | |||
| 035293b612 | |||
| 250b5df836 | |||
| ec59dbc6ee | |||
| 4d5676f00e | |||
| 2a4b62527a | |||
| ec0341c696 | |||
| 2e5f3a5a66 | |||
| 231a5fab86 | |||
| 10319ea69d | |||
| 53bb5aff22 | |||
| 52a5fff61f | |||
| f28754b883 | |||
| 6fce797ca2 | |||
| a1fd315104 | |||
| a91f0ac8b5 | |||
| 2c96df05f4 | |||
| b57788c7b9 | |||
| 7431bab2a7 | |||
| a52697bfdf | |||
| 9dc2199381 | |||
| 89db760b89 | |||
| 4356c1e186 | |||
| e32cebc553 | |||
| 23e1cc281d | |||
| 48d3fb4fe3 |
@@ -29,6 +29,12 @@ POSTGRES_ADMIN_PASSWORD=postgres
|
||||
POSTGRES_USER=prowler
|
||||
POSTGRES_PASSWORD=postgres
|
||||
POSTGRES_DB=prowler_db
|
||||
# Read replica settings (optional)
|
||||
# POSTGRES_REPLICA_HOST=postgres-db
|
||||
# POSTGRES_REPLICA_PORT=5432
|
||||
# POSTGRES_REPLICA_USER=prowler
|
||||
# POSTGRES_REPLICA_PASSWORD=postgres
|
||||
# POSTGRES_REPLICA_DB=prowler_db
|
||||
|
||||
# Celery-Prowler task settings
|
||||
TASK_RETRY_DELAY_SECONDS=0.1
|
||||
@@ -99,7 +105,7 @@ SENTRY_ENVIRONMENT=local
|
||||
SENTRY_RELEASE=local
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.10.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.12.2
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Set short git commit SHA
|
||||
id: vars
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
|
||||
- name: Trigger deployment
|
||||
if: github.event_name == 'push'
|
||||
uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -44,16 +44,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5
|
||||
uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/api-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5
|
||||
uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
|
||||
@@ -76,20 +76,20 @@ jobs:
|
||||
--health-retries 5
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Test if changes are in not ignored paths
|
||||
id: are-non-ignored-files-changed
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
.github/workflows/api-pull-request.yml
|
||||
files_ignore: ${{ env.IGNORE_FILES }}
|
||||
|
||||
- name: Replace @master with current branch in pyproject.toml
|
||||
- name: Replace @master with current branch in pyproject.toml - Only for pull requests to `master`
|
||||
working-directory: ./api
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' && github.event_name == 'pull_request' && github.base_ref == 'master'
|
||||
run: |
|
||||
BRANCH_NAME="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
|
||||
echo "Using branch: $BRANCH_NAME"
|
||||
@@ -104,7 +104,7 @@ jobs:
|
||||
|
||||
- name: Update SDK's poetry.lock resolved_reference to latest commit - Only for push events to `master`
|
||||
working-directory: ./api
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' && github.event_name == 'push'
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
run: |
|
||||
# Get the latest commit hash from the prowler-cloud/prowler repository
|
||||
LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha')
|
||||
@@ -127,17 +127,11 @@ jobs:
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: "poetry"
|
||||
|
||||
- name: Install system dependencies for xmlsec
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libxml2-dev libxmlsec1-dev libxmlsec1-openssl pkg-config
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./api
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
||||
@@ -208,7 +202,7 @@ jobs:
|
||||
|
||||
- name: Upload coverage reports to Codecov
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -216,11 +210,11 @@ jobs:
|
||||
test-container-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Test if changes are in not ignored paths
|
||||
id: are-non-ignored-files-changed
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: api/**
|
||||
files_ignore: ${{ env.IGNORE_FILES }}
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Find existing documentation comment
|
||||
uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
|
||||
uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0
|
||||
id: find-comment
|
||||
with:
|
||||
issue-number: ${{ env.PR_NUMBER }}
|
||||
|
||||
@@ -20,4 +20,5 @@ jobs:
|
||||
id: conventional-commit-check
|
||||
uses: agenthunt/conventional-commit-checker-action@9e552d650d0e205553ec7792d447929fc78e012b # v2.0.0
|
||||
with:
|
||||
pr-title-regex: '^([^\s(]+)(?:\(([^)]+)\))?: (.+)'
|
||||
pr-title-regex: '^(feat|fix|docs|style|refactor|perf|test|chore|build|ci|revert)(\([^)]+\))?!?: .+'
|
||||
|
||||
@@ -7,11 +7,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: TruffleHog OSS
|
||||
uses: trufflesecurity/trufflehog@a05cf0859455b5b16317ee22d809887a4043cdf0 # v3.90.2
|
||||
uses: trufflesecurity/trufflehog@466da5b0bb161144f6afca9afe5d57975828c410 # v3.90.8
|
||||
with:
|
||||
path: ./
|
||||
base: ${{ github.event.repository.default_branch }}
|
||||
|
||||
@@ -14,4 +14,4 @@ jobs:
|
||||
pull-requests: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0
|
||||
- uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
name: MCP Server - Build and Push containers
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
paths:
|
||||
- "mcp_server/**"
|
||||
- ".github/workflows/mcp-server-build-push-containers.yml"
|
||||
|
||||
# Uncomment the below code to test this action on PRs
|
||||
# pull_request:
|
||||
# branches:
|
||||
# - "master"
|
||||
# paths:
|
||||
# - "mcp_server/**"
|
||||
# - ".github/workflows/mcp-server-build-push-containers.yml"
|
||||
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
env:
|
||||
# Tags
|
||||
LATEST_TAG: latest
|
||||
|
||||
WORKING_DIRECTORY: ./mcp_server
|
||||
|
||||
# Container Registries
|
||||
PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud
|
||||
PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-mcp
|
||||
|
||||
jobs:
|
||||
repository-check:
|
||||
name: Repository check
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
is_repo: ${{ steps.repository_check.outputs.is_repo }}
|
||||
steps:
|
||||
- name: Repository check
|
||||
id: repository_check
|
||||
working-directory: /tmp
|
||||
run: |
|
||||
if [[ ${{ github.repository }} == "prowler-cloud/prowler" ]]
|
||||
then
|
||||
echo "is_repo=true" >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
echo "This action only runs for prowler-cloud/prowler"
|
||||
echo "is_repo=false" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
|
||||
container-build-push:
|
||||
needs: repository-check
|
||||
if: needs.repository-check.outputs.is_repo == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Set short git commit SHA
|
||||
id: vars
|
||||
run: |
|
||||
shortSha=$(git rev-parse --short ${{ github.sha }})
|
||||
echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build and push container image (latest)
|
||||
# Comment the following line for testing
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
# Set push: false for testing
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }}
|
||||
${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.SHORT_SHA }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -9,14 +9,15 @@ on:
|
||||
branches:
|
||||
- "master"
|
||||
- "v5.*"
|
||||
pull_request_target:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
branches:
|
||||
- "master"
|
||||
- "v5.*"
|
||||
# Leaving this commented until we find a way to run it for forks but in Prowler's context
|
||||
# pull_request_target:
|
||||
# types:
|
||||
# - opened
|
||||
# - synchronize
|
||||
# - reopened
|
||||
# branches:
|
||||
# - "master"
|
||||
# - "v5.*"
|
||||
|
||||
jobs:
|
||||
conflict-checker:
|
||||
@@ -28,11 +29,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
**
|
||||
@@ -70,7 +71,7 @@ jobs:
|
||||
|
||||
- name: Add conflict label
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true'
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
script: |
|
||||
@@ -96,7 +97,7 @@ jobs:
|
||||
|
||||
- name: Remove conflict label
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'false'
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
script: |
|
||||
@@ -118,7 +119,7 @@ jobs:
|
||||
|
||||
- name: Find existing conflict comment
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true'
|
||||
uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
|
||||
uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0
|
||||
id: find-comment
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
@@ -148,7 +149,7 @@ jobs:
|
||||
|
||||
- name: Find existing conflict comment when resolved
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'false'
|
||||
uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
|
||||
uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0
|
||||
id: find-resolved-comment
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
@@ -166,3 +167,9 @@ jobs:
|
||||
✅ **Conflict Markers Resolved**
|
||||
|
||||
All conflict markers have been successfully resolved in this pull request.
|
||||
|
||||
- name: Fail workflow if conflicts detected
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true'
|
||||
run: |
|
||||
echo "::error::Workflow failed due to conflict markers in files: ${{ steps.conflict-check.outputs.conflict_files }}"
|
||||
exit 1
|
||||
|
||||
@@ -22,13 +22,13 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
@@ -59,25 +59,157 @@ jobs:
|
||||
BRANCH_NAME="v${MAJOR_VERSION}.${MINOR_VERSION}"
|
||||
echo "BRANCH_NAME=${BRANCH_NAME}" >> "${GITHUB_ENV}"
|
||||
|
||||
# Calculate UI version (1.X.X format - matches Prowler minor version)
|
||||
UI_VERSION="1.${MINOR_VERSION}.${PATCH_VERSION}"
|
||||
echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}"
|
||||
# Function to extract the latest version from changelog
|
||||
extract_latest_version() {
|
||||
local changelog_file="$1"
|
||||
if [ -f "$changelog_file" ]; then
|
||||
# Extract the first version entry (most recent) from changelog
|
||||
# Format: ## [version] (1.2.3) or ## [vversion] (v1.2.3)
|
||||
local version=$(grep -m 1 '^## \[' "$changelog_file" | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]')
|
||||
echo "$version"
|
||||
else
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
# Calculate API version (1.X.X format - one minor version ahead)
|
||||
API_MINOR_VERSION=$((MINOR_VERSION + 1))
|
||||
API_VERSION="1.${API_MINOR_VERSION}.${PATCH_VERSION}"
|
||||
# Read actual versions from changelogs (source of truth)
|
||||
UI_VERSION=$(extract_latest_version "ui/CHANGELOG.md")
|
||||
API_VERSION=$(extract_latest_version "api/CHANGELOG.md")
|
||||
SDK_VERSION=$(extract_latest_version "prowler/CHANGELOG.md")
|
||||
|
||||
echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}"
|
||||
echo "SDK_VERSION=${SDK_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
if [ -n "$UI_VERSION" ]; then
|
||||
echo "Read UI version from changelog: $UI_VERSION"
|
||||
else
|
||||
echo "Warning: No UI version found in ui/CHANGELOG.md"
|
||||
fi
|
||||
|
||||
if [ -n "$API_VERSION" ]; then
|
||||
echo "Read API version from changelog: $API_VERSION"
|
||||
else
|
||||
echo "Warning: No API version found in api/CHANGELOG.md"
|
||||
fi
|
||||
|
||||
if [ -n "$SDK_VERSION" ]; then
|
||||
echo "Read SDK version from changelog: $SDK_VERSION"
|
||||
else
|
||||
echo "Warning: No SDK version found in prowler/CHANGELOG.md"
|
||||
fi
|
||||
|
||||
echo "Prowler version: $PROWLER_VERSION"
|
||||
echo "Branch name: $BRANCH_NAME"
|
||||
echo "UI version: $UI_VERSION"
|
||||
echo "API version: $API_VERSION"
|
||||
echo "SDK version: $SDK_VERSION"
|
||||
echo "Is minor release: $([ $PATCH_VERSION -eq 0 ] && echo 'true' || echo 'false')"
|
||||
else
|
||||
echo "Invalid version syntax: '$PROWLER_VERSION' (must be N.N.N)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Extract changelog entries
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Function to extract changelog for a specific version
|
||||
extract_changelog() {
|
||||
local file="$1"
|
||||
local version="$2"
|
||||
local output_file="$3"
|
||||
|
||||
if [ ! -f "$file" ]; then
|
||||
echo "Warning: $file not found, skipping..."
|
||||
touch "$output_file"
|
||||
return
|
||||
fi
|
||||
|
||||
# Extract changelog section for this version
|
||||
awk -v version="$version" '
|
||||
/^## \[v?'"$version"'\]/ { found=1; next }
|
||||
found && /^## \[v?[0-9]+\.[0-9]+\.[0-9]+\]/ { found=0 }
|
||||
found && !/^## \[v?'"$version"'\]/ { print }
|
||||
' "$file" > "$output_file"
|
||||
|
||||
# Remove --- separators
|
||||
sed -i '/^---$/d' "$output_file"
|
||||
|
||||
# Remove trailing empty lines
|
||||
sed -i '/^$/d' "$output_file"
|
||||
}
|
||||
|
||||
# Calculate expected versions for this release
|
||||
if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
EXPECTED_UI_VERSION="1.${BASH_REMATCH[2]}.${BASH_REMATCH[3]}"
|
||||
EXPECTED_API_VERSION="1.$((${BASH_REMATCH[2]} + 1)).${BASH_REMATCH[3]}"
|
||||
|
||||
echo "Expected UI version for this release: $EXPECTED_UI_VERSION"
|
||||
echo "Expected API version for this release: $EXPECTED_API_VERSION"
|
||||
fi
|
||||
|
||||
# Determine if components have changes for this specific release
|
||||
# UI has changes if its current version matches what we expect for this release
|
||||
if [ -n "$UI_VERSION" ] && [ "$UI_VERSION" = "$EXPECTED_UI_VERSION" ]; then
|
||||
echo "HAS_UI_CHANGES=true" >> $GITHUB_ENV
|
||||
echo "✓ UI changes detected - version matches expected: $UI_VERSION"
|
||||
extract_changelog "ui/CHANGELOG.md" "$UI_VERSION" "ui_changelog.md"
|
||||
else
|
||||
echo "HAS_UI_CHANGES=false" >> $GITHUB_ENV
|
||||
echo "ℹ No UI changes for this release (current: $UI_VERSION, expected: $EXPECTED_UI_VERSION)"
|
||||
touch "ui_changelog.md"
|
||||
fi
|
||||
|
||||
# API has changes if its current version matches what we expect for this release
|
||||
if [ -n "$API_VERSION" ] && [ "$API_VERSION" = "$EXPECTED_API_VERSION" ]; then
|
||||
echo "HAS_API_CHANGES=true" >> $GITHUB_ENV
|
||||
echo "✓ API changes detected - version matches expected: $API_VERSION"
|
||||
extract_changelog "api/CHANGELOG.md" "$API_VERSION" "api_changelog.md"
|
||||
else
|
||||
echo "HAS_API_CHANGES=false" >> $GITHUB_ENV
|
||||
echo "ℹ No API changes for this release (current: $API_VERSION, expected: $EXPECTED_API_VERSION)"
|
||||
touch "api_changelog.md"
|
||||
fi
|
||||
|
||||
# SDK has changes if its current version matches the input version
|
||||
if [ -n "$SDK_VERSION" ] && [ "$SDK_VERSION" = "$PROWLER_VERSION" ]; then
|
||||
echo "HAS_SDK_CHANGES=true" >> $GITHUB_ENV
|
||||
echo "✓ SDK changes detected - version matches input: $SDK_VERSION"
|
||||
extract_changelog "prowler/CHANGELOG.md" "$PROWLER_VERSION" "prowler_changelog.md"
|
||||
else
|
||||
echo "HAS_SDK_CHANGES=false" >> $GITHUB_ENV
|
||||
echo "ℹ No SDK changes for this release (current: $SDK_VERSION, input: $PROWLER_VERSION)"
|
||||
touch "prowler_changelog.md"
|
||||
fi
|
||||
|
||||
# Combine changelogs in order: UI, API, SDK
|
||||
> combined_changelog.md
|
||||
|
||||
if [ "$HAS_UI_CHANGES" = "true" ] && [ -s "ui_changelog.md" ]; then
|
||||
echo "## UI" >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
cat ui_changelog.md >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
fi
|
||||
|
||||
if [ "$HAS_API_CHANGES" = "true" ] && [ -s "api_changelog.md" ]; then
|
||||
echo "## API" >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
cat api_changelog.md >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
fi
|
||||
|
||||
if [ "$HAS_SDK_CHANGES" = "true" ] && [ -s "prowler_changelog.md" ]; then
|
||||
echo "## SDK" >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
cat prowler_changelog.md >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
fi
|
||||
|
||||
echo "Combined changelog preview:"
|
||||
cat combined_changelog.md
|
||||
|
||||
- name: Checkout existing branch for patch release
|
||||
if: ${{ env.PATCH_VERSION != '0' }}
|
||||
run: |
|
||||
@@ -114,6 +246,7 @@ jobs:
|
||||
echo "✓ prowler/config/config.py version: $CURRENT_VERSION"
|
||||
|
||||
- name: Verify version in api/pyproject.toml
|
||||
if: ${{ env.HAS_API_CHANGES == 'true' }}
|
||||
run: |
|
||||
CURRENT_API_VERSION=$(grep '^version = ' api/pyproject.toml | sed -E 's/version = "([^"]+)"/\1/' | tr -d '[:space:]')
|
||||
API_VERSION_TRIMMED=$(echo "$API_VERSION" | tr -d '[:space:]')
|
||||
@@ -124,7 +257,7 @@ jobs:
|
||||
echo "✓ api/pyproject.toml version: $CURRENT_API_VERSION"
|
||||
|
||||
- name: Verify prowler dependency in api/pyproject.toml
|
||||
if: ${{ env.PATCH_VERSION != '0' }}
|
||||
if: ${{ env.PATCH_VERSION != '0' && env.HAS_API_CHANGES == 'true' }}
|
||||
run: |
|
||||
CURRENT_PROWLER_REF=$(grep 'prowler @ git+https://github.com/prowler-cloud/prowler.git@' api/pyproject.toml | sed -E 's/.*@([^"]+)".*/\1/' | tr -d '[:space:]')
|
||||
BRANCH_NAME_TRIMMED=$(echo "$BRANCH_NAME" | tr -d '[:space:]')
|
||||
@@ -135,6 +268,7 @@ jobs:
|
||||
echo "✓ api/pyproject.toml prowler dependency: $CURRENT_PROWLER_REF"
|
||||
|
||||
- name: Verify version in api/src/backend/api/v1/views.py
|
||||
if: ${{ env.HAS_API_CHANGES == 'true' }}
|
||||
run: |
|
||||
CURRENT_API_VERSION=$(grep 'spectacular_settings.VERSION = ' api/src/backend/api/v1/views.py | sed -E 's/.*spectacular_settings.VERSION = "([^"]+)".*/\1/' | tr -d '[:space:]')
|
||||
API_VERSION_TRIMMED=$(echo "$API_VERSION" | tr -d '[:space:]')
|
||||
@@ -162,12 +296,11 @@ jobs:
|
||||
CURRENT_PROWLER_REF=$(grep 'prowler @ git+https://github.com/prowler-cloud/prowler.git@' api/pyproject.toml | sed -E 's/.*@([^"]+)".*/\1/' | tr -d '[:space:]')
|
||||
BRANCH_NAME_TRIMMED=$(echo "$BRANCH_NAME" | tr -d '[:space:]')
|
||||
|
||||
# Create a temporary branch for the PR
|
||||
# Create a temporary branch for the PR from the minor version branch
|
||||
TEMP_BRANCH="update-api-dependency-$BRANCH_NAME_TRIMMED-$(date +%s)"
|
||||
echo "TEMP_BRANCH=$TEMP_BRANCH" >> $GITHUB_ENV
|
||||
|
||||
# Switch back to master and create temp branch
|
||||
git checkout master
|
||||
# Create temp branch from the current minor version branch
|
||||
git checkout -b "$TEMP_BRANCH"
|
||||
|
||||
# Minor release: update the dependency to use the release branch
|
||||
@@ -221,77 +354,14 @@ jobs:
|
||||
component/api
|
||||
no-changelog
|
||||
|
||||
- name: Extract changelog entries
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Function to extract changelog for a specific version
|
||||
extract_changelog() {
|
||||
local file="$1"
|
||||
local version="$2"
|
||||
local output_file="$3"
|
||||
|
||||
if [ ! -f "$file" ]; then
|
||||
echo "Warning: $file not found, skipping..."
|
||||
touch "$output_file"
|
||||
return
|
||||
fi
|
||||
|
||||
# Extract changelog section for this version
|
||||
awk -v version="$version" '
|
||||
/^## \[v?'"$version"'\]/ { found=1; next }
|
||||
found && /^## \[v?[0-9]+\.[0-9]+\.[0-9]+\]/ { found=0 }
|
||||
found && !/^## \[v?'"$version"'\]/ { print }
|
||||
' "$file" > "$output_file"
|
||||
|
||||
# Remove --- separators
|
||||
sed -i '/^---$/d' "$output_file"
|
||||
|
||||
# Remove trailing empty lines
|
||||
sed -i '/^$/d' "$output_file"
|
||||
}
|
||||
|
||||
# Extract changelogs
|
||||
echo "Extracting changelog entries..."
|
||||
extract_changelog "prowler/CHANGELOG.md" "$PROWLER_VERSION" "prowler_changelog.md"
|
||||
extract_changelog "api/CHANGELOG.md" "$API_VERSION" "api_changelog.md"
|
||||
extract_changelog "ui/CHANGELOG.md" "$UI_VERSION" "ui_changelog.md"
|
||||
|
||||
# Combine changelogs in order: UI, API, SDK
|
||||
> combined_changelog.md
|
||||
|
||||
if [ -s "ui_changelog.md" ]; then
|
||||
echo "## UI" >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
cat ui_changelog.md >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
fi
|
||||
|
||||
if [ -s "api_changelog.md" ]; then
|
||||
echo "## API" >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
cat api_changelog.md >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
fi
|
||||
|
||||
if [ -s "prowler_changelog.md" ]; then
|
||||
echo "## SDK" >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
cat prowler_changelog.md >> combined_changelog.md
|
||||
echo "" >> combined_changelog.md
|
||||
fi
|
||||
|
||||
echo "Combined changelog preview:"
|
||||
cat combined_changelog.md
|
||||
|
||||
- name: Create draft release
|
||||
uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2
|
||||
uses: softprops/action-gh-release@6cbd405e2c4e67a21c47fa9e383d020e4e28b836 # v2.3.3
|
||||
with:
|
||||
tag_name: ${{ env.PROWLER_VERSION }}
|
||||
name: Prowler ${{ env.PROWLER_VERSION }}
|
||||
body_path: combined_changelog.md
|
||||
draft: true
|
||||
target_commitish: ${{ env.PATCH_VERSION == '0' && 'master' || env.BRANCH_NAME }}
|
||||
target_commitish: ${{ env.BRANCH_NAME }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
MONITORED_FOLDERS: "api ui prowler dashboard"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
- name: Find existing changelog comment
|
||||
if: github.event.pull_request.head.repo.full_name == github.repository
|
||||
id: find_comment
|
||||
uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e #v3.1.0
|
||||
uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad #v4.0.0
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
comment-author: 'github-actions[bot]'
|
||||
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
if: github.event.pull_request.merged == true && github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.merge_commit_sha }}
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV
|
||||
|
||||
- name: Trigger pull request
|
||||
uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -59,10 +59,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
@@ -108,13 +108,13 @@ jobs:
|
||||
esac
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
name: Bump Version
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Get Prowler version
|
||||
shell: bash
|
||||
|
||||
@@ -52,16 +52,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5
|
||||
uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/sdk-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5
|
||||
uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
|
||||
@@ -21,11 +21,11 @@ jobs:
|
||||
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Test if changes are in not ignored paths
|
||||
id: are-non-ignored-files-changed
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: ./**
|
||||
files_ignore: |
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: "poetry"
|
||||
@@ -104,7 +104,7 @@ jobs:
|
||||
|
||||
- name: Dockerfile - Check if Dockerfile has changed
|
||||
id: dockerfile-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
Dockerfile
|
||||
@@ -117,7 +117,7 @@ jobs:
|
||||
# Test AWS
|
||||
- name: AWS - Check if any file has changed
|
||||
id: aws-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/aws/**
|
||||
@@ -132,7 +132,7 @@ jobs:
|
||||
# Test Azure
|
||||
- name: Azure - Check if any file has changed
|
||||
id: azure-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/azure/**
|
||||
@@ -147,7 +147,7 @@ jobs:
|
||||
# Test GCP
|
||||
- name: GCP - Check if any file has changed
|
||||
id: gcp-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/gcp/**
|
||||
@@ -162,7 +162,7 @@ jobs:
|
||||
# Test Kubernetes
|
||||
- name: Kubernetes - Check if any file has changed
|
||||
id: kubernetes-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/kubernetes/**
|
||||
@@ -177,7 +177,7 @@ jobs:
|
||||
# Test GitHub
|
||||
- name: GitHub - Check if any file has changed
|
||||
id: github-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/github/**
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
# Test NHN
|
||||
- name: NHN - Check if any file has changed
|
||||
id: nhn-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/nhn/**
|
||||
@@ -207,7 +207,7 @@ jobs:
|
||||
# Test M365
|
||||
- name: M365 - Check if any file has changed
|
||||
id: m365-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/m365/**
|
||||
@@ -222,7 +222,7 @@ jobs:
|
||||
# Test IaC
|
||||
- name: IaC - Check if any file has changed
|
||||
id: iac-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/iac/**
|
||||
@@ -237,7 +237,7 @@ jobs:
|
||||
# Test MongoDB Atlas
|
||||
- name: MongoDB Atlas - Check if any file has changed
|
||||
id: mongodb-atlas-changed-files
|
||||
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
|
||||
uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/mongodbatlas/**
|
||||
@@ -263,7 +263,7 @@ jobs:
|
||||
# Codecov
|
||||
- name: Upload coverage reports to Codecov
|
||||
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
|
||||
@@ -64,14 +64,14 @@ jobs:
|
||||
;;
|
||||
esac
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pipx install poetry==2.1.1
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
# cache: ${{ env.CACHE }}
|
||||
|
||||
@@ -23,12 +23,12 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
ref: ${{ env.GITHUB_BRANCH }}
|
||||
|
||||
- name: setup python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: 3.9 #install the python needed
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
pip install boto3
|
||||
|
||||
- name: Configure AWS Credentials -- DEV
|
||||
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
|
||||
uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0
|
||||
with:
|
||||
aws-region: ${{ env.AWS_REGION_DEV }}
|
||||
role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }}
|
||||
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
- name: Set short git commit SHA
|
||||
id: vars
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
|
||||
- name: Trigger deployment
|
||||
if: github.event_name == 'push'
|
||||
uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
|
||||
@@ -44,16 +44,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5
|
||||
uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/ui-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5
|
||||
uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
NEXT_PUBLIC_API_BASE_URL: 'http://localhost:8080/api/v1'
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
- name: Fix API data directory permissions
|
||||
run: docker run --rm -v $(pwd)/_data/api:/data alpine chown -R 1000:1000 /data
|
||||
- name: Start API services
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
echo "All database fixtures loaded successfully!"
|
||||
'
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version: '20.x'
|
||||
cache: 'npm'
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
working-directory: ./ui
|
||||
run: npm run build
|
||||
- name: Cache Playwright browsers
|
||||
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
id: playwright-cache
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
|
||||
@@ -27,11 +27,11 @@ jobs:
|
||||
node-version: [20.x]
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Setup Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
cache: 'npm'
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
test-container-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
- name: Build Container
|
||||
|
||||
@@ -80,9 +80,6 @@ _data/
|
||||
# Claude
|
||||
CLAUDE.md
|
||||
|
||||
# LLM's (Until we have a standard one)
|
||||
AGENTS.md
|
||||
|
||||
# MCP Server
|
||||
mcp_server/prowler_mcp_server/prowler_app/server.py
|
||||
mcp_server/prowler_mcp_server/prowler_app/utils/schema.yaml
|
||||
|
||||
@@ -6,6 +6,7 @@ repos:
|
||||
- id: check-merge-conflict
|
||||
- id: check-yaml
|
||||
args: ["--unsafe"]
|
||||
exclude: prowler/config/llm_config.yaml
|
||||
- id: check-json
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
# Repository Guidelines
|
||||
|
||||
## How to Use This Guide
|
||||
|
||||
- Start here for cross-project norms, Prowler is a monorepo with several components. Every component should have an `AGENTS.md` file that contains the guidelines for the agents in that component. The file is located beside the code you are touching (e.g. `api/AGENTS.md`, `ui/AGENTS.md`, `prowler/AGENTS.md`).
|
||||
- Follow the stricter rule when guidance conflicts; component docs override this file for their scope.
|
||||
- Keep instructions synchronized. When you add new workflows or scripts, update both, the relevant component `AGENTS.md` and this file if they apply broadly.
|
||||
|
||||
## Project Overview
|
||||
|
||||
Prowler is an open-source cloud security assessment tool that supports multiple cloud providers (AWS, Azure, GCP, Kubernetes, GitHub, M365, etc.). The project consists in a monorepo with the following main components:
|
||||
|
||||
- **Prowler SDK**: Python SDK, includes the Prowler CLI, providers, services, checks, compliances, config, etc. (`prowler/`)
|
||||
- **Prowler API**: Django-based REST API backend (`api/`)
|
||||
- **Prowler UI**: Next.js frontend application (`ui/`)
|
||||
- **Prowler MCP Server**: Model Context Protocol server that gives access to the entire Prowler ecosystem for LLMs (`mcp_server/`)
|
||||
- **Prowler Dashboard**: Prowler CLI feature that allows to visualize the results of the scans in a simple dashboard (`dashboard/`)
|
||||
|
||||
### Project Structure (Key Folders & Files)
|
||||
|
||||
- `prowler/`: Main source code for Prowler SDK (CLI, providers, services, checks, compliances, config, etc.)
|
||||
- `api/`: Django-based REST API backend components
|
||||
- `ui/`: Next.js frontend application
|
||||
- `mcp_server/`: Model Context Protocol server that gives access to the entire Prowler ecosystem for LLMs
|
||||
- `dashboard/`: Prowler CLI feature that allows to visualize the results of the scans in a simple dashboard
|
||||
- `docs/`: Documentation
|
||||
- `examples/`: Example output formats for providers and scripts
|
||||
- `permissions/`: Permission-related files and policies
|
||||
- `contrib/`: Community-contributed scripts or modules
|
||||
- `tests/`: Prowler SDK test suite
|
||||
- `docker-compose.yml`: Docker compose file to run the Prowler App (API + UI) production environment
|
||||
- `docker-compose-dev.yml`: Docker compose file to run the Prowler App (API + UI) development environment
|
||||
- `pyproject.toml`: Poetry Prowler SDK project file
|
||||
- `.pre-commit-config.yaml`: Pre-commit hooks configuration
|
||||
- `Makefile`: Makefile to run the project
|
||||
- `LICENSE`: License file
|
||||
- `README.md`: README file
|
||||
- `CONTRIBUTING.md`: Contributing guide
|
||||
|
||||
## Python Development
|
||||
|
||||
Most of the code is written in Python, so the main files in the root are focused on Python code.
|
||||
|
||||
### Poetry Dev Environment
|
||||
|
||||
For developing in Python we recommend using `poetry` to manage the dependencies. The minimal version is `2.1.1`. So it is recommended to run all commands using `poetry run ...`.
|
||||
|
||||
To install the core dependencies to develop it is needed to run `poetry install --with dev`.
|
||||
|
||||
### Pre-commit hooks
|
||||
|
||||
The project has pre-commit hooks to lint and format the code. They are installed by running `poetry run pre-commit install`.
|
||||
|
||||
When commiting a change, the hooks will be run automatically. Some of them are:
|
||||
|
||||
- Code formatting (black, isort)
|
||||
- Linting (flake8, pylint)
|
||||
- Security checks (bandit, safety, trufflehog)
|
||||
- YAML/JSON validation
|
||||
- Poetry lock file validation
|
||||
|
||||
|
||||
### Linting and Formatting
|
||||
|
||||
We use the following tools to lint and format the code:
|
||||
|
||||
- `flake8`: for linting the code
|
||||
- `black`: for formatting the code
|
||||
- `pylint`: for linting the code
|
||||
|
||||
You can run all using the `make` command:
|
||||
```bash
|
||||
poetry run make lint
|
||||
poetry run make format
|
||||
```
|
||||
|
||||
Or they will be run automatically when you commit your changes using pre-commit hooks.
|
||||
|
||||
## Commit & Pull Request Guidelines
|
||||
|
||||
For the commit messages and pull requests name follow the conventional-commit style.
|
||||
|
||||
Befire creating a pull request, complete the checklist in `.github/pull_request_template.md`. Summaries should explain deployment impact, highlight review steps, and note changelog or permission updates. Run all relevant tests and linters before requesting review and link screenshots for UI or dashboard changes.
|
||||
|
||||
### Conventional Commit Style
|
||||
|
||||
The Conventional Commits specification is a lightweight convention on top of commit messages. It provides an easy set of rules for creating an explicit commit history; which makes it easier to write automated tools on top of.
|
||||
|
||||
The commit message should be structured as follows:
|
||||
|
||||
```
|
||||
<type>[optional scope]: <description>
|
||||
<BLANK LINE>
|
||||
[optional body]
|
||||
<BLANK LINE>
|
||||
[optional footer(s)]
|
||||
```
|
||||
|
||||
Any line of the commit message cannot be longer 100 characters! This allows the message to be easier to read on GitHub as well as in various git tools
|
||||
|
||||
#### Commit Types
|
||||
|
||||
- **feat**: code change introuce new functionality to the application
|
||||
- **fix**: code change that solve a bug in the codebase
|
||||
- **docs**: documentation only changes
|
||||
- **chore**: changes related to the build process or auxiliary tools and libraries, that do not affect the application's functionality
|
||||
- **perf**: code change that improves performance
|
||||
- **refactor**: code change that neither fixes a bug nor adds a feature
|
||||
- **style**: changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
|
||||
- **test**: adding missing tests or correcting existing tests
|
||||
@@ -45,3 +45,7 @@ pypi-upload: ## Upload package
|
||||
help: ## Show this help.
|
||||
@echo "Prowler Makefile"
|
||||
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
|
||||
|
||||
##@ Development Environment
|
||||
run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, and workers
|
||||
docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat --build
|
||||
|
||||
@@ -90,6 +90,7 @@ prowler dashboard
|
||||
| M365 | 70 | 7 | 3 | 2 | Official | Stable | UI, API, CLI |
|
||||
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | Beta | CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | Beta | CLI |
|
||||
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | Beta | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | Beta | CLI |
|
||||
|
||||
> [!Note]
|
||||
|
||||
@@ -7,11 +7,26 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
### Added
|
||||
- Default JWT keys are generated and stored if they are missing from configuration [(#8655)](https://github.com/prowler-cloud/prowler/pull/8655)
|
||||
- `compliance_name` for each compliance [(#7920)](https://github.com/prowler-cloud/prowler/pull/7920)
|
||||
- Support for M365 Certificate authentication [(#8538)](https://github.com/prowler-cloud/prowler/pull/8538)
|
||||
- API Key support [(#8805)](https://github.com/prowler-cloud/prowler/pull/8805)
|
||||
- SAML role mapping protection for single-admin tenants to prevent accidental lockout [(#8882)](https://github.com/prowler-cloud/prowler/pull/8882)
|
||||
- Support for `passed_findings` and `total_findings` fields in compliance requirement overview for accurate Prowler ThreatScore calculation [(#8582)](https://github.com/prowler-cloud/prowler/pull/8582)
|
||||
- Database read replica support [(#8869)](https://github.com/prowler-cloud/prowler/pull/8869)
|
||||
|
||||
### Changed
|
||||
- Now the MANAGE_ACCOUNT permission is required to modify or read user permissions instead of MANAGE_USERS [(#8281)](https://github.com/prowler-cloud/prowler/pull/8281)
|
||||
- Now at least one user with MANAGE_ACCOUNT permission is required in the tenant [(#8729)](https://github.com/prowler-cloud/prowler/pull/8729)
|
||||
|
||||
### Security
|
||||
- Django updated to the latest 5.1 security release, 5.1.13, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/104) and [directory traversals](https://github.com/prowler-cloud/prowler/security/dependabot/103) [(#8842)](https://github.com/prowler-cloud/prowler/pull/8842)
|
||||
|
||||
---
|
||||
|
||||
## [1.13.2] (Prowler 5.12.3)
|
||||
|
||||
### Fixed
|
||||
- 500 error when deleting user [(#8731)](https://github.com/prowler-cloud/prowler/pull/8731)
|
||||
|
||||
---
|
||||
|
||||
## [1.13.1] (Prowler 5.12.2)
|
||||
|
||||
+3
-1
@@ -18,10 +18,12 @@ Valkey exposes a Redis 7.2 compliant API. Any service that exposes the Redis API
|
||||
|
||||
# Modify environment variables
|
||||
|
||||
Under the root path of the project, you can find a file called `.env.example`. This file shows all the environment variables that the project uses. You *must* create a new file called `.env` and set the values for the variables.
|
||||
Under the root path of the project, you can find a file called `.env`. This file shows all the environment variables that the project uses. You should review it and set the values for the variables you want to change.
|
||||
|
||||
If you don’t set `DJANGO_TOKEN_SIGNING_KEY` or `DJANGO_TOKEN_VERIFYING_KEY`, the API will generate them at `~/.config/prowler-api/` with `0600` and `0644` permissions; back up these files to persist identity across redeploys.
|
||||
|
||||
**Important note**: Every Prowler version (or repository branches and tags) could have different variables set in its `.env` file. Please use the `.env` file that corresponds with each version.
|
||||
|
||||
## Local deployment
|
||||
Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the Poetry interpreter, not before. Otherwise, variables will not be loaded properly.
|
||||
|
||||
|
||||
Generated
+87
-9
@@ -273,14 +273,14 @@ tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" a
|
||||
|
||||
[[package]]
|
||||
name = "authlib"
|
||||
version = "1.6.1"
|
||||
version = "1.6.4"
|
||||
description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients."
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["dev"]
|
||||
files = [
|
||||
{file = "authlib-1.6.1-py2.py3-none-any.whl", hash = "sha256:e9d2031c34c6309373ab845afc24168fe9e93dc52d252631f52642f21f5ed06e"},
|
||||
{file = "authlib-1.6.1.tar.gz", hash = "sha256:4dffdbb1460ba6ec8c17981a4c67af7d8af131231b5a36a88a1e8c80c111cdfd"},
|
||||
{file = "authlib-1.6.4-py2.py3-none-any.whl", hash = "sha256:39313d2a2caac3ecf6d8f95fbebdfd30ae6ea6ae6a6db794d976405fdd9aa796"},
|
||||
{file = "authlib-1.6.4.tar.gz", hash = "sha256:104b0442a43061dc8bc23b133d1d06a2b0a9c2e3e33f34c4338929e816287649"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@@ -383,6 +383,24 @@ cryptography = ">=2.1.4"
|
||||
isodate = ">=0.6.1"
|
||||
typing-extensions = ">=4.0.1"
|
||||
|
||||
[[package]]
|
||||
name = "azure-mgmt-apimanagement"
|
||||
version = "5.0.0"
|
||||
description = "Microsoft Azure API Management Client Library for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "azure_mgmt_apimanagement-5.0.0-py3-none-any.whl", hash = "sha256:b88c42a392333b60722fb86f15d092dfc19a8d67510dccd15c217381dff4e6ec"},
|
||||
{file = "azure_mgmt_apimanagement-5.0.0.tar.gz", hash = "sha256:0ab7fe17e70fe3154cd840ff47d19d7a4610217003eaa7c21acf3511a6e57999"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
azure-common = ">=1.1"
|
||||
azure-mgmt-core = ">=1.3.2"
|
||||
isodate = ">=0.6.1"
|
||||
typing-extensions = ">=4.6.0"
|
||||
|
||||
[[package]]
|
||||
name = "azure-mgmt-applicationinsights"
|
||||
version = "4.1.0"
|
||||
@@ -540,6 +558,23 @@ azure-mgmt-core = ">=1.3.2"
|
||||
isodate = ">=0.6.1"
|
||||
typing-extensions = ">=4.6.0"
|
||||
|
||||
[[package]]
|
||||
name = "azure-mgmt-loganalytics"
|
||||
version = "12.0.0"
|
||||
description = "Microsoft Azure Log Analytics Management Client Library for Python"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "azure-mgmt-loganalytics-12.0.0.zip", hash = "sha256:da128a7e0291be7fa2063848df92a9180cf5c16d42adc09d2bc2efd711536bfb"},
|
||||
{file = "azure_mgmt_loganalytics-12.0.0-py2.py3-none-any.whl", hash = "sha256:75ac1d47dd81179905c40765be8834643d8994acff31056ddc1863017f3faa02"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
azure-common = ">=1.1,<2.0"
|
||||
azure-mgmt-core = ">=1.2.0,<2.0.0"
|
||||
msrest = ">=0.6.21"
|
||||
|
||||
[[package]]
|
||||
name = "azure-mgmt-monitor"
|
||||
version = "6.0.2"
|
||||
@@ -750,6 +785,23 @@ azure-mgmt-core = ">=1.3.2"
|
||||
isodate = ">=0.6.1"
|
||||
typing-extensions = ">=4.6.0"
|
||||
|
||||
[[package]]
|
||||
name = "azure-monitor-query"
|
||||
version = "2.0.0"
|
||||
description = "Microsoft Corporation Azure Monitor Query Client Library for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "azure_monitor_query-2.0.0-py3-none-any.whl", hash = "sha256:8f52d581271d785e12f49cd5aaa144b8910fb843db2373855a7ef94c7fc462ea"},
|
||||
{file = "azure_monitor_query-2.0.0.tar.gz", hash = "sha256:7b05f2fcac4fb67fc9f77a7d4c5d98a0f3099fb73b57c69ec1b080773994671b"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
azure-core = ">=1.30.0"
|
||||
isodate = ">=0.6.1"
|
||||
typing-extensions = ">=4.6.0"
|
||||
|
||||
[[package]]
|
||||
name = "azure-storage-blob"
|
||||
version = "12.24.1"
|
||||
@@ -1511,14 +1563,14 @@ with-social = ["django-allauth[socialaccount] (>=64.0.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "django"
|
||||
version = "5.1.12"
|
||||
version = "5.1.13"
|
||||
description = "A high-level Python web framework that encourages rapid development and clean, pragmatic design."
|
||||
optional = false
|
||||
python-versions = ">=3.10"
|
||||
groups = ["main", "dev"]
|
||||
files = [
|
||||
{file = "django-5.1.12-py3-none-any.whl", hash = "sha256:9eb695636cea3601b65690f1596993c042206729afb320ca0960b55f8ed4477b"},
|
||||
{file = "django-5.1.12.tar.gz", hash = "sha256:8a8991b1ec052ef6a44fefd1ef336ab8daa221287bcb91a4a17d5e1abec5bbcc"},
|
||||
{file = "django-5.1.13-py3-none-any.whl", hash = "sha256:06f257f79dc4c17f3f9e23b106a4c5ed1335abecbe731e83c598c941d14fbeed"},
|
||||
{file = "django-5.1.13.tar.gz", hash = "sha256:543ff21679f15e80edfc01fe7ea35f8291b6d4ea589433882913626a7c1cf929"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@@ -1872,6 +1924,27 @@ files = [
|
||||
Django = ">=4.2"
|
||||
djangorestframework = ">=3.15.0"
|
||||
|
||||
[[package]]
|
||||
name = "drf-simple-apikey"
|
||||
version = "2.2.1"
|
||||
description = "API Key authentication and permissions for Django REST."
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "drf_simple_apikey-2.2.1-py2.py3-none-any.whl", hash = "sha256:2a60b35676d14f907c47dee179dd0fa7425a84c34d6ff5b48d08d3b87ff32809"},
|
||||
{file = "drf_simple_apikey-2.2.1.tar.gz", hash = "sha256:e5a52804bbac12c8db80c10a3d51a8514fc59fc8385b5e751099a2bc944ad25d"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
cryptography = ">=38.0.4"
|
||||
django = ">=4.2"
|
||||
djangorestframework = ">=3.14.0"
|
||||
|
||||
[package.extras]
|
||||
test = ["coverage", "pytest", "pytest-django"]
|
||||
tooling = ["black (==22.3.0)", "bump2version", "pylint"]
|
||||
|
||||
[[package]]
|
||||
name = "drf-spectacular"
|
||||
version = "0.27.2"
|
||||
@@ -4003,7 +4076,7 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler"
|
||||
version = "5.11.0"
|
||||
version = "5.13.0"
|
||||
description = "Prowler is an Open Source security tool to perform AWS, GCP and Azure security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMP, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme) and your custom security frameworks."
|
||||
optional = false
|
||||
python-versions = ">3.9.1,<3.13"
|
||||
@@ -4016,6 +4089,7 @@ alive-progress = "3.3.0"
|
||||
awsipranges = "0.3.3"
|
||||
azure-identity = "1.21.0"
|
||||
azure-keyvault-keys = "4.10.0"
|
||||
azure-mgmt-apimanagement = "5.0.0"
|
||||
azure-mgmt-applicationinsights = "4.1.0"
|
||||
azure-mgmt-authorization = "4.0.0"
|
||||
azure-mgmt-compute = "34.0.0"
|
||||
@@ -4024,6 +4098,7 @@ azure-mgmt-containerservice = "34.1.0"
|
||||
azure-mgmt-cosmosdb = "9.7.0"
|
||||
azure-mgmt-databricks = "2.0.0"
|
||||
azure-mgmt-keyvault = "10.3.1"
|
||||
azure-mgmt-loganalytics = "12.0.0"
|
||||
azure-mgmt-monitor = "6.0.2"
|
||||
azure-mgmt-network = "28.1.0"
|
||||
azure-mgmt-rdbms = "10.1.0"
|
||||
@@ -4036,6 +4111,7 @@ azure-mgmt-sql = "3.0.1"
|
||||
azure-mgmt-storage = "22.1.1"
|
||||
azure-mgmt-subscription = "3.1.1"
|
||||
azure-mgmt-web = "8.0.0"
|
||||
azure-monitor-query = "2.0.0"
|
||||
azure-storage-blob = "12.24.1"
|
||||
boto3 = "1.39.15"
|
||||
botocore = "1.39.15"
|
||||
@@ -4047,8 +4123,10 @@ detect-secrets = "1.5.0"
|
||||
dulwich = "0.23.0"
|
||||
google-api-python-client = "2.163.0"
|
||||
google-auth-httplib2 = ">=0.1,<0.3"
|
||||
h2 = "4.3.0"
|
||||
jsonschema = "4.23.0"
|
||||
kubernetes = "32.0.1"
|
||||
markdown = "3.9.0"
|
||||
microsoft-kiota-abstractions = "1.9.2"
|
||||
msgraph-sdk = "1.23.0"
|
||||
numpy = "2.0.2"
|
||||
@@ -4069,7 +4147,7 @@ tzlocal = "5.3.1"
|
||||
type = "git"
|
||||
url = "https://github.com/prowler-cloud/prowler.git"
|
||||
reference = "master"
|
||||
resolved_reference = "525f152e51f82de2110ed158c8dc489e42c289cf"
|
||||
resolved_reference = "a52697bfdfee83d14a49c11dcbe96888b5cd767e"
|
||||
|
||||
[[package]]
|
||||
name = "psutil"
|
||||
@@ -6181,4 +6259,4 @@ type = ["pytest-mypy"]
|
||||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.11,<3.13"
|
||||
content-hash = "91058a14382b76136a82f45624a30aece7a6d77c8b36c290bb4c40ea60c8850b"
|
||||
content-hash = "03442fd4673006c5a74374f90f53621fd1c9d117279fe6cc0355ef833eb7f9bb"
|
||||
|
||||
+3
-2
@@ -7,7 +7,7 @@ authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}]
|
||||
dependencies = [
|
||||
"celery[pytest] (>=5.4.0,<6.0.0)",
|
||||
"dj-rest-auth[with_social,jwt] (==7.0.1)",
|
||||
"django (==5.1.12)",
|
||||
"django (==5.1.13)",
|
||||
"django-allauth[saml] (>=65.8.0,<66.0.0)",
|
||||
"django-celery-beat (>=2.7.0,<3.0.0)",
|
||||
"django-celery-results (>=2.5.1,<3.0.0)",
|
||||
@@ -32,7 +32,8 @@ dependencies = [
|
||||
"openai (>=1.82.0,<2.0.0)",
|
||||
"xmlsec==1.3.14",
|
||||
"h2 (==4.3.0)",
|
||||
"markdown (>=3.9,<4.0)"
|
||||
"markdown (>=3.9,<4.0)",
|
||||
"drf-simple-apikey (==2.2.1)"
|
||||
]
|
||||
description = "Prowler's API (Django/DRF)"
|
||||
license = "Apache-2.0"
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
from pathlib import Path
|
||||
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
@@ -30,6 +28,7 @@ class ApiConfig(AppConfig):
|
||||
name = "api"
|
||||
|
||||
def ready(self):
|
||||
from api import schema_extensions # noqa: F401
|
||||
from api import signals # noqa: F401
|
||||
from api.compliance import load_prowler_compliance
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
from typing import Optional, Tuple
|
||||
from uuid import UUID
|
||||
|
||||
from cryptography.fernet import InvalidToken
|
||||
from django.utils import timezone
|
||||
from drf_simple_apikey.backends import APIKeyAuthentication as BaseAPIKeyAuth
|
||||
from drf_simple_apikey.crypto import get_crypto
|
||||
from rest_framework.authentication import BaseAuthentication
|
||||
from rest_framework.exceptions import AuthenticationFailed
|
||||
from rest_framework.request import Request
|
||||
from rest_framework_simplejwt.authentication import JWTAuthentication
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.models import TenantAPIKey, TenantAPIKeyManager
|
||||
|
||||
|
||||
class TenantAPIKeyAuthentication(BaseAPIKeyAuth):
|
||||
model = TenantAPIKey
|
||||
|
||||
def __init__(self):
|
||||
self.key_crypto = get_crypto()
|
||||
|
||||
def _authenticate_credentials(self, request, key):
|
||||
"""
|
||||
Override to use admin connection, bypassing RLS during authentication.
|
||||
Delegates to parent after temporarily routing model queries to admin DB.
|
||||
"""
|
||||
# Temporarily point the model's manager to admin database
|
||||
original_objects = self.model.objects
|
||||
self.model.objects = self.model.objects.using(MainRouter.admin_db)
|
||||
|
||||
try:
|
||||
# Call parent method which will now use admin database
|
||||
return super()._authenticate_credentials(request, key)
|
||||
finally:
|
||||
# Restore original manager
|
||||
self.model.objects = original_objects
|
||||
|
||||
def authenticate(self, request: Request):
|
||||
prefixed_key = self.get_key(request)
|
||||
|
||||
# Split prefix from key (format: pk_xxxxxxxx.encrypted_key)
|
||||
try:
|
||||
prefix, key = prefixed_key.split(TenantAPIKeyManager.separator, 1)
|
||||
except ValueError:
|
||||
raise AuthenticationFailed("Invalid API Key.")
|
||||
|
||||
try:
|
||||
entity, _ = self._authenticate_credentials(request, key)
|
||||
except InvalidToken:
|
||||
raise AuthenticationFailed("Invalid API Key.")
|
||||
|
||||
# Get the API key instance to update last_used_at and retrieve tenant info
|
||||
# We need to decrypt again to get the pk (already validated by _authenticate_credentials)
|
||||
payload = self.key_crypto.decrypt(key)
|
||||
api_key_pk = payload["_pk"]
|
||||
|
||||
# Convert string UUID back to UUID object for lookup
|
||||
if isinstance(api_key_pk, str):
|
||||
api_key_pk = UUID(api_key_pk)
|
||||
|
||||
try:
|
||||
api_key_instance = TenantAPIKey.objects.using(MainRouter.admin_db).get(
|
||||
id=api_key_pk, prefix=prefix
|
||||
)
|
||||
except TenantAPIKey.DoesNotExist:
|
||||
raise AuthenticationFailed("Invalid API Key.")
|
||||
|
||||
# Update last_used_at
|
||||
api_key_instance.last_used_at = timezone.now()
|
||||
api_key_instance.save(update_fields=["last_used_at"], using=MainRouter.admin_db)
|
||||
|
||||
return entity, {
|
||||
"tenant_id": str(api_key_instance.tenant_id),
|
||||
"sub": str(api_key_instance.entity.id),
|
||||
"api_key_prefix": prefix,
|
||||
}
|
||||
|
||||
|
||||
class CombinedJWTOrAPIKeyAuthentication(BaseAuthentication):
|
||||
jwt_auth = JWTAuthentication()
|
||||
api_key_auth = TenantAPIKeyAuthentication()
|
||||
|
||||
def authenticate(self, request: Request) -> Optional[Tuple[object, dict]]:
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
|
||||
# Prioritize JWT authentication if both are present
|
||||
if auth_header.startswith("Bearer "):
|
||||
return self.jwt_auth.authenticate(request)
|
||||
|
||||
if auth_header.startswith("Api-Key "):
|
||||
return self.api_key_auth.authenticate(request)
|
||||
|
||||
# Default fallback
|
||||
return self.jwt_auth.authenticate(request)
|
||||
@@ -1,13 +1,15 @@
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.db import transaction
|
||||
from rest_framework import permissions
|
||||
from rest_framework.exceptions import NotAuthenticated
|
||||
from rest_framework.filters import SearchFilter
|
||||
from rest_framework.permissions import SAFE_METHODS
|
||||
from rest_framework_json_api import filters
|
||||
from rest_framework_json_api.views import ModelViewSet
|
||||
from rest_framework_simplejwt.authentication import JWTAuthentication
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.authentication import CombinedJWTOrAPIKeyAuthentication
|
||||
from api.db_router import MainRouter, reset_read_db_alias, set_read_db_alias
|
||||
from api.db_utils import POSTGRES_USER_VAR, rls_transaction
|
||||
from api.filters import CustomDjangoFilterBackend
|
||||
from api.models import Role, Tenant
|
||||
@@ -15,7 +17,7 @@ from api.rbac.permissions import HasPermissions
|
||||
|
||||
|
||||
class BaseViewSet(ModelViewSet):
|
||||
authentication_classes = [JWTAuthentication]
|
||||
authentication_classes = [CombinedJWTOrAPIKeyAuthentication]
|
||||
required_permissions = []
|
||||
permission_classes = [permissions.IsAuthenticated, HasPermissions]
|
||||
filter_backends = [
|
||||
@@ -31,6 +33,20 @@ class BaseViewSet(ModelViewSet):
|
||||
ordering_fields = "__all__"
|
||||
ordering = ["id"]
|
||||
|
||||
def _get_request_db_alias(self, request):
|
||||
if request is None:
|
||||
return MainRouter.default_db
|
||||
|
||||
read_alias = (
|
||||
MainRouter.replica_db
|
||||
if request.method in SAFE_METHODS
|
||||
and MainRouter.replica_db in settings.DATABASES
|
||||
else None
|
||||
)
|
||||
if read_alias:
|
||||
return read_alias
|
||||
return MainRouter.default_db
|
||||
|
||||
def initial(self, request, *args, **kwargs):
|
||||
"""
|
||||
Sets required_permissions before permissions are checked.
|
||||
@@ -48,8 +64,21 @@ class BaseViewSet(ModelViewSet):
|
||||
|
||||
class BaseRLSViewSet(BaseViewSet):
|
||||
def dispatch(self, request, *args, **kwargs):
|
||||
with transaction.atomic():
|
||||
return super().dispatch(request, *args, **kwargs)
|
||||
self.db_alias = self._get_request_db_alias(request)
|
||||
alias_token = None
|
||||
try:
|
||||
if self.db_alias != MainRouter.default_db:
|
||||
alias_token = set_read_db_alias(self.db_alias)
|
||||
|
||||
if request is not None:
|
||||
request.db_alias = self.db_alias
|
||||
|
||||
with transaction.atomic(using=self.db_alias):
|
||||
return super().dispatch(request, *args, **kwargs)
|
||||
finally:
|
||||
if alias_token is not None:
|
||||
reset_read_db_alias(alias_token)
|
||||
self.db_alias = MainRouter.default_db
|
||||
|
||||
def initial(self, request, *args, **kwargs):
|
||||
# Ideally, this logic would be in the `.setup()` method but DRF view sets don't call it
|
||||
@@ -61,7 +90,9 @@ class BaseRLSViewSet(BaseViewSet):
|
||||
if tenant_id is None:
|
||||
raise NotAuthenticated("Tenant ID is not present in token")
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
with rls_transaction(
|
||||
tenant_id, using=getattr(self, "db_alias", MainRouter.default_db)
|
||||
):
|
||||
self.request.tenant_id = tenant_id
|
||||
return super().initial(request, *args, **kwargs)
|
||||
|
||||
@@ -73,18 +104,33 @@ class BaseRLSViewSet(BaseViewSet):
|
||||
|
||||
class BaseTenantViewset(BaseViewSet):
|
||||
def dispatch(self, request, *args, **kwargs):
|
||||
with transaction.atomic():
|
||||
tenant = super().dispatch(request, *args, **kwargs)
|
||||
|
||||
self.db_alias = self._get_request_db_alias(request)
|
||||
alias_token = None
|
||||
try:
|
||||
# If the request is a POST, create the admin role
|
||||
if request.method == "POST":
|
||||
isinstance(tenant, dict) and self._create_admin_role(tenant.data["id"])
|
||||
except Exception as e:
|
||||
self._handle_creation_error(e, tenant)
|
||||
raise
|
||||
if self.db_alias != MainRouter.default_db:
|
||||
alias_token = set_read_db_alias(self.db_alias)
|
||||
|
||||
return tenant
|
||||
if request is not None:
|
||||
request.db_alias = self.db_alias
|
||||
|
||||
with transaction.atomic(using=self.db_alias):
|
||||
tenant = super().dispatch(request, *args, **kwargs)
|
||||
|
||||
try:
|
||||
# If the request is a POST, create the admin role
|
||||
if request.method == "POST":
|
||||
isinstance(tenant, dict) and self._create_admin_role(
|
||||
tenant.data["id"]
|
||||
)
|
||||
except Exception as e:
|
||||
self._handle_creation_error(e, tenant)
|
||||
raise
|
||||
|
||||
return tenant
|
||||
finally:
|
||||
if alias_token is not None:
|
||||
reset_read_db_alias(alias_token)
|
||||
self.db_alias = MainRouter.default_db
|
||||
|
||||
def _create_admin_role(self, tenant_id):
|
||||
Role.objects.using(MainRouter.admin_db).create(
|
||||
@@ -117,14 +163,31 @@ class BaseTenantViewset(BaseViewSet):
|
||||
raise NotAuthenticated("Tenant ID is not present in token")
|
||||
|
||||
user_id = str(request.user.id)
|
||||
with rls_transaction(value=user_id, parameter=POSTGRES_USER_VAR):
|
||||
with rls_transaction(
|
||||
value=user_id,
|
||||
parameter=POSTGRES_USER_VAR,
|
||||
using=getattr(self, "db_alias", MainRouter.default_db),
|
||||
):
|
||||
return super().initial(request, *args, **kwargs)
|
||||
|
||||
|
||||
class BaseUserViewset(BaseViewSet):
|
||||
def dispatch(self, request, *args, **kwargs):
|
||||
with transaction.atomic():
|
||||
return super().dispatch(request, *args, **kwargs)
|
||||
self.db_alias = self._get_request_db_alias(request)
|
||||
alias_token = None
|
||||
try:
|
||||
if self.db_alias != MainRouter.default_db:
|
||||
alias_token = set_read_db_alias(self.db_alias)
|
||||
|
||||
if request is not None:
|
||||
request.db_alias = self.db_alias
|
||||
|
||||
with transaction.atomic(using=self.db_alias):
|
||||
return super().dispatch(request, *args, **kwargs)
|
||||
finally:
|
||||
if alias_token is not None:
|
||||
reset_read_db_alias(alias_token)
|
||||
self.db_alias = MainRouter.default_db
|
||||
|
||||
def initial(self, request, *args, **kwargs):
|
||||
# TODO refactor after improving RLS on users
|
||||
@@ -137,6 +200,8 @@ class BaseUserViewset(BaseViewSet):
|
||||
if tenant_id is None:
|
||||
raise NotAuthenticated("Tenant ID is not present in token")
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
with rls_transaction(
|
||||
tenant_id, using=getattr(self, "db_alias", MainRouter.default_db)
|
||||
):
|
||||
self.request.tenant_id = tenant_id
|
||||
return super().initial(request, *args, **kwargs)
|
||||
|
||||
@@ -150,12 +150,16 @@ def generate_scan_compliance(
|
||||
requirement["checks"][check_id] = status
|
||||
requirement["checks_status"][status.lower()] += 1
|
||||
|
||||
if requirement["status"] != "FAIL" and any(
|
||||
value == "FAIL" for value in requirement["checks"].values()
|
||||
):
|
||||
requirement["status"] = "FAIL"
|
||||
compliance_overview[compliance_id]["requirements_status"]["passed"] -= 1
|
||||
compliance_overview[compliance_id]["requirements_status"]["failed"] += 1
|
||||
if requirement["status"] != "FAIL" and any(
|
||||
value == "FAIL" for value in requirement["checks"].values()
|
||||
):
|
||||
requirement["status"] = "FAIL"
|
||||
compliance_overview[compliance_id]["requirements_status"][
|
||||
"passed"
|
||||
] -= 1
|
||||
compliance_overview[compliance_id]["requirements_status"][
|
||||
"failed"
|
||||
] += 1
|
||||
|
||||
|
||||
def generate_compliance_overview_template(prowler_compliance: dict):
|
||||
|
||||
@@ -1,9 +1,31 @@
|
||||
from contextvars import ContextVar
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
ALLOWED_APPS = ("django", "socialaccount", "account", "authtoken", "silk")
|
||||
|
||||
_read_db_alias = ContextVar("read_db_alias", default=None)
|
||||
|
||||
|
||||
def set_read_db_alias(alias: str | None):
|
||||
if not alias:
|
||||
return None
|
||||
return _read_db_alias.set(alias)
|
||||
|
||||
|
||||
def get_read_db_alias() -> str | None:
|
||||
return _read_db_alias.get()
|
||||
|
||||
|
||||
def reset_read_db_alias(token) -> None:
|
||||
if token is not None:
|
||||
_read_db_alias.reset(token)
|
||||
|
||||
|
||||
class MainRouter:
|
||||
default_db = "default"
|
||||
admin_db = "admin"
|
||||
replica_db = "replica"
|
||||
|
||||
def db_for_read(self, model, **hints): # noqa: F841
|
||||
model_table_name = model._meta.db_table
|
||||
@@ -11,6 +33,9 @@ class MainRouter:
|
||||
model_table_name.startswith(f"{app}_") for app in ALLOWED_APPS
|
||||
):
|
||||
return self.admin_db
|
||||
read_alias = get_read_db_alias()
|
||||
if read_alias:
|
||||
return read_alias
|
||||
return None
|
||||
|
||||
def db_for_write(self, model, **hints): # noqa: F841
|
||||
@@ -27,3 +52,8 @@ class MainRouter:
|
||||
if {obj1._state.db, obj2._state.db} <= {self.default_db, self.admin_db}:
|
||||
return True
|
||||
return None
|
||||
|
||||
|
||||
READ_REPLICA_ALIAS = (
|
||||
MainRouter.replica_db if MainRouter.replica_db in settings.DATABASES else None
|
||||
)
|
||||
|
||||
@@ -6,12 +6,14 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import BaseUserManager
|
||||
from django.db import connection, models, transaction
|
||||
from django.db import DEFAULT_DB_ALIAS, connection, connections, models, transaction
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from psycopg2 import connect as psycopg2_connect
|
||||
from psycopg2.extensions import AsIs, new_type, register_adapter, register_type
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
|
||||
from api.db_router import get_read_db_alias, reset_read_db_alias, set_read_db_alias
|
||||
|
||||
DB_USER = settings.DATABASES["default"]["USER"] if not settings.TESTING else "test"
|
||||
DB_PASSWORD = (
|
||||
settings.DATABASES["default"]["PASSWORD"] if not settings.TESTING else "test"
|
||||
@@ -49,7 +51,11 @@ def psycopg_connection(database_alias: str):
|
||||
|
||||
|
||||
@contextmanager
|
||||
def rls_transaction(value: str, parameter: str = POSTGRES_TENANT_VAR):
|
||||
def rls_transaction(
|
||||
value: str,
|
||||
parameter: str = POSTGRES_TENANT_VAR,
|
||||
using: str | None = None,
|
||||
):
|
||||
"""
|
||||
Creates a new database transaction setting the given configuration value for Postgres RLS. It validates the
|
||||
if the value is a valid UUID.
|
||||
@@ -57,16 +63,32 @@ def rls_transaction(value: str, parameter: str = POSTGRES_TENANT_VAR):
|
||||
Args:
|
||||
value (str): Database configuration parameter value.
|
||||
parameter (str): Database configuration parameter name, by default is 'api.tenant_id'.
|
||||
using (str | None): Optional database alias to run the transaction against. Defaults to the
|
||||
active read alias (if any) or Django's default connection.
|
||||
"""
|
||||
with transaction.atomic():
|
||||
with connection.cursor() as cursor:
|
||||
try:
|
||||
# just in case the value is an UUID object
|
||||
uuid.UUID(str(value))
|
||||
except ValueError:
|
||||
raise ValidationError("Must be a valid UUID")
|
||||
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
|
||||
yield cursor
|
||||
requested_alias = using or get_read_db_alias()
|
||||
db_alias = requested_alias or DEFAULT_DB_ALIAS
|
||||
if db_alias not in connections:
|
||||
db_alias = DEFAULT_DB_ALIAS
|
||||
|
||||
router_token = None
|
||||
try:
|
||||
if db_alias != DEFAULT_DB_ALIAS:
|
||||
router_token = set_read_db_alias(db_alias)
|
||||
|
||||
with transaction.atomic(using=db_alias):
|
||||
conn = connections[db_alias]
|
||||
with conn.cursor() as cursor:
|
||||
try:
|
||||
# just in case the value is a UUID object
|
||||
uuid.UUID(str(value))
|
||||
except ValueError:
|
||||
raise ValidationError("Must be a valid UUID")
|
||||
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
|
||||
yield cursor
|
||||
finally:
|
||||
if router_token is not None:
|
||||
reset_read_db_alias(router_token)
|
||||
|
||||
|
||||
class CustomUserManager(BaseUserManager):
|
||||
@@ -434,6 +456,12 @@ def drop_index_on_partitions(
|
||||
schema_editor.execute(sql)
|
||||
|
||||
|
||||
def generate_api_key_prefix():
|
||||
"""Generate a random 8-character prefix for API keys (e.g., 'pk_abc123de')."""
|
||||
random_chars = generate_random_token(length=8)
|
||||
return f"pk_{random_chars}"
|
||||
|
||||
|
||||
# Postgres enum definition for member role
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from django.core.exceptions import ValidationError as django_validation_error
|
||||
from rest_framework import status
|
||||
from rest_framework.exceptions import APIException
|
||||
from rest_framework.exceptions import (
|
||||
APIException,
|
||||
AuthenticationFailed,
|
||||
NotAuthenticated,
|
||||
)
|
||||
from rest_framework_json_api.exceptions import exception_handler
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
from rest_framework_simplejwt.exceptions import InvalidToken, TokenError
|
||||
@@ -68,15 +72,18 @@ def custom_exception_handler(exc, context):
|
||||
exc = ValidationError(exc.message_dict)
|
||||
else:
|
||||
exc = ValidationError(detail=exc.messages[0], code=exc.code)
|
||||
elif isinstance(exc, (TokenError, InvalidToken)):
|
||||
if (
|
||||
hasattr(exc, "detail")
|
||||
and isinstance(exc.detail, dict)
|
||||
and "messages" in exc.detail
|
||||
):
|
||||
exc.detail["messages"] = [
|
||||
message_item["message"] for message_item in exc.detail["messages"]
|
||||
]
|
||||
# Force 401 status for AuthenticationFailed exceptions regardless of the authentication backend
|
||||
elif isinstance(exc, (AuthenticationFailed, NotAuthenticated, TokenError)):
|
||||
exc.status_code = status.HTTP_401_UNAUTHORIZED
|
||||
if isinstance(exc, (TokenError, InvalidToken)):
|
||||
if (
|
||||
hasattr(exc, "detail")
|
||||
and isinstance(exc.detail, dict)
|
||||
and "messages" in exc.detail
|
||||
):
|
||||
exc.detail["messages"] = [
|
||||
message_item["message"] for message_item in exc.detail["messages"]
|
||||
]
|
||||
return exception_handler(exc, context)
|
||||
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ from api.models import (
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
Task,
|
||||
TenantAPIKey,
|
||||
User,
|
||||
)
|
||||
from api.rls import Tenant
|
||||
@@ -219,10 +220,31 @@ class MembershipFilter(FilterSet):
|
||||
|
||||
|
||||
class ProviderFilter(FilterSet):
|
||||
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
|
||||
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
|
||||
connected = BooleanFilter()
|
||||
inserted_at = DateFilter(
|
||||
field_name="inserted_at",
|
||||
lookup_expr="date",
|
||||
help_text="""Filter by date when the provider was added
|
||||
(format: YYYY-MM-DD)""",
|
||||
)
|
||||
updated_at = DateFilter(
|
||||
field_name="updated_at",
|
||||
lookup_expr="date",
|
||||
help_text="""Filter by date when the provider was updated
|
||||
(format: YYYY-MM-DD)""",
|
||||
)
|
||||
connected = BooleanFilter(
|
||||
help_text="""Filter by connection status. Set to True to return only
|
||||
connected providers, or False to return only providers with failed
|
||||
connections. If not specified, both connected and failed providers are
|
||||
included. Providers with no connection attempt (status is null) are
|
||||
excluded from this filter."""
|
||||
)
|
||||
provider = ChoiceFilter(choices=Provider.ProviderChoices.choices)
|
||||
provider__in = ChoiceInFilter(
|
||||
field_name="provider",
|
||||
choices=Provider.ProviderChoices.choices,
|
||||
lookup_expr="in",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = Provider
|
||||
@@ -648,8 +670,16 @@ class LatestFindingFilter(CommonFindingFilters):
|
||||
|
||||
|
||||
class ProviderSecretFilter(FilterSet):
|
||||
inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date")
|
||||
updated_at = DateFilter(field_name="updated_at", lookup_expr="date")
|
||||
inserted_at = DateFilter(
|
||||
field_name="inserted_at",
|
||||
lookup_expr="date",
|
||||
help_text="Filter by date when the secret was added (format: YYYY-MM-DD)",
|
||||
)
|
||||
updated_at = DateFilter(
|
||||
field_name="updated_at",
|
||||
lookup_expr="date",
|
||||
help_text="Filter by date when the secret was updated (format: YYYY-MM-DD)",
|
||||
)
|
||||
provider = UUIDFilter(field_name="provider__id", lookup_expr="exact")
|
||||
|
||||
class Meta:
|
||||
@@ -880,3 +910,20 @@ class IntegrationJiraFindingsFilter(FilterSet):
|
||||
}
|
||||
)
|
||||
return super().filter_queryset(queryset)
|
||||
|
||||
|
||||
class TenantApiKeyFilter(FilterSet):
|
||||
inserted_at = DateFilter(field_name="created", lookup_expr="date")
|
||||
inserted_at__gte = DateFilter(field_name="created", lookup_expr="gte")
|
||||
inserted_at__lte = DateFilter(field_name="created", lookup_expr="lte")
|
||||
expires_at = DateFilter(field_name="expiry_date", lookup_expr="date")
|
||||
expires_at__gte = DateFilter(field_name="expiry_date", lookup_expr="gte")
|
||||
expires_at__lte = DateFilter(field_name="expiry_date", lookup_expr="lte")
|
||||
|
||||
class Meta:
|
||||
model = TenantAPIKey
|
||||
fields = {
|
||||
"prefix": ["exact", "icontains"],
|
||||
"revoked": ["exact"],
|
||||
"name": ["exact", "icontains"],
|
||||
}
|
||||
|
||||
@@ -8,9 +8,14 @@ def extract_auth_info(request) -> dict:
|
||||
if getattr(request, "auth", None) is not None:
|
||||
tenant_id = request.auth.get("tenant_id", "N/A")
|
||||
user_id = request.auth.get("sub", "N/A")
|
||||
api_key_prefix = request.auth.get("api_key_prefix", "N/A")
|
||||
else:
|
||||
tenant_id, user_id = "N/A", "N/A"
|
||||
return {"tenant_id": tenant_id, "user_id": user_id}
|
||||
tenant_id, user_id, api_key_prefix = "N/A", "N/A", "N/A"
|
||||
return {
|
||||
"tenant_id": tenant_id,
|
||||
"user_id": user_id,
|
||||
"api_key_prefix": api_key_prefix,
|
||||
}
|
||||
|
||||
|
||||
class APILoggingMiddleware:
|
||||
@@ -38,6 +43,7 @@ class APILoggingMiddleware:
|
||||
extra={
|
||||
"user_id": auth_info["user_id"],
|
||||
"tenant_id": auth_info["tenant_id"],
|
||||
"api_key_prefix": auth_info["api_key_prefix"],
|
||||
"method": request.method,
|
||||
"path": request.path,
|
||||
"query_params": request.GET.dict(),
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# Generated by Django 5.1.12 on 2025-09-30 13:10
|
||||
|
||||
import uuid
|
||||
|
||||
import django.core.validators
|
||||
import django.db.models.deletion
|
||||
import drf_simple_apikey.models
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.db_utils
|
||||
import api.rls
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0047_remove_integration_unique_configuration_per_tenant"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="TenantAPIKey",
|
||||
fields=[
|
||||
(
|
||||
"name",
|
||||
models.CharField(
|
||||
max_length=255,
|
||||
validators=[django.core.validators.MinLengthValidator(3)],
|
||||
),
|
||||
),
|
||||
(
|
||||
"expiry_date",
|
||||
models.DateTimeField(
|
||||
default=drf_simple_apikey.models._expiry_date,
|
||||
help_text="Once API key expires, entities cannot use it anymore.",
|
||||
verbose_name="Expires",
|
||||
),
|
||||
),
|
||||
(
|
||||
"revoked",
|
||||
models.BooleanField(
|
||||
blank=True,
|
||||
default=False,
|
||||
help_text="If the API key is revoked, entities cannot use it anymore. (This cannot be undone.)",
|
||||
),
|
||||
),
|
||||
("created", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"whitelisted_ips",
|
||||
models.JSONField(
|
||||
blank=True,
|
||||
help_text="List of allowed IP addresses for this API key.",
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
(
|
||||
"blacklisted_ips",
|
||||
models.JSONField(
|
||||
blank=True,
|
||||
help_text="List of denied IP addresses for this API key.",
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
(
|
||||
"prefix",
|
||||
models.CharField(
|
||||
default=api.db_utils.generate_api_key_prefix,
|
||||
editable=False,
|
||||
help_text="Unique prefix to identify the API key",
|
||||
max_length=11,
|
||||
unique=True,
|
||||
),
|
||||
),
|
||||
(
|
||||
"last_used_at",
|
||||
models.DateTimeField(
|
||||
blank=True,
|
||||
help_text="Last time this API key was used for authentication",
|
||||
null=True,
|
||||
),
|
||||
),
|
||||
(
|
||||
"entity",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="user_api_keys",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
(
|
||||
"tenant",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"db_table": "api_keys",
|
||||
"abstract": False,
|
||||
"indexes": [
|
||||
models.Index(
|
||||
fields=["tenant_id", "prefix"],
|
||||
name="api_keys_tenant_prefix_idx",
|
||||
)
|
||||
],
|
||||
"constraints": [
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "prefix"), name="unique_api_key_prefixes"
|
||||
),
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "name"),
|
||||
name="unique_api_key_name_per_tenant",
|
||||
),
|
||||
],
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="tenantapikey",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_tenantapikey",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
]
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
# Generated by Django 5.1.12 on 2025-10-07 10:41
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0048_api_key"),
|
||||
]
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="compliancerequirementoverview",
|
||||
name="passed_findings",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="compliancerequirementoverview",
|
||||
name="total_findings",
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
]
|
||||
@@ -22,6 +22,8 @@ from django.db.models import Q
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from django_celery_results.models import TaskResult
|
||||
from drf_simple_apikey.crypto import get_crypto
|
||||
from drf_simple_apikey.models import AbstractAPIKey, AbstractAPIKeyManager
|
||||
from psqlextra.manager import PostgresManager
|
||||
from psqlextra.models import PostgresPartitionedModel
|
||||
from psqlextra.types import PostgresPartitioningMethod
|
||||
@@ -42,6 +44,7 @@ from api.db_utils import (
|
||||
StateEnumField,
|
||||
StatusEnumField,
|
||||
enum_to_choices,
|
||||
generate_api_key_prefix,
|
||||
generate_random_token,
|
||||
one_week_from_now,
|
||||
)
|
||||
@@ -125,6 +128,17 @@ class ActiveProviderPartitionedManager(PostgresManager, ActiveProviderManager):
|
||||
return super().get_queryset().filter(self.active_provider_filter())
|
||||
|
||||
|
||||
class TenantAPIKeyManager(AbstractAPIKeyManager):
|
||||
separator = "."
|
||||
|
||||
def assign_api_key(self, obj) -> str:
|
||||
payload = {"_pk": str(obj.pk), "_exp": obj.expiry_date.timestamp()}
|
||||
key = get_crypto().generate(payload)
|
||||
|
||||
prefixed_key = f"{obj.prefix}{self.separator}{key}"
|
||||
return prefixed_key
|
||||
|
||||
|
||||
class User(AbstractBaseUser):
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
name = models.CharField(max_length=150, validators=[MinLengthValidator(3)])
|
||||
@@ -204,6 +218,60 @@ class Membership(models.Model):
|
||||
resource_name = "memberships"
|
||||
|
||||
|
||||
class TenantAPIKey(AbstractAPIKey, RowLevelSecurityProtectedModel):
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
name = models.CharField(max_length=100, validators=[MinLengthValidator(3)])
|
||||
prefix = models.CharField(
|
||||
max_length=11,
|
||||
unique=True,
|
||||
default=generate_api_key_prefix,
|
||||
editable=False,
|
||||
help_text="Unique prefix to identify the API key",
|
||||
)
|
||||
last_used_at = models.DateTimeField(
|
||||
null=True,
|
||||
blank=True,
|
||||
help_text="Last time this API key was used for authentication",
|
||||
)
|
||||
entity = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.SET_NULL,
|
||||
null=True,
|
||||
blank=True,
|
||||
related_name="user_api_keys",
|
||||
)
|
||||
|
||||
objects = TenantAPIKeyManager()
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "api_keys"
|
||||
|
||||
constraints = [
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "prefix"),
|
||||
name="unique_api_key_prefixes",
|
||||
),
|
||||
models.UniqueConstraint(
|
||||
fields=("tenant_id", "name"),
|
||||
name="unique_api_key_name_per_tenant",
|
||||
),
|
||||
]
|
||||
|
||||
indexes = [
|
||||
models.Index(
|
||||
fields=["tenant_id", "prefix"], name="api_keys_tenant_prefix_idx"
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "api-keys"
|
||||
|
||||
|
||||
class Provider(RowLevelSecurityProtectedModel):
|
||||
objects = ActiveProviderManager()
|
||||
all_objects = models.Manager()
|
||||
@@ -1230,6 +1298,8 @@ class ComplianceRequirementOverview(RowLevelSecurityProtectedModel):
|
||||
passed_checks = models.IntegerField(default=0)
|
||||
failed_checks = models.IntegerField(default=0)
|
||||
total_checks = models.IntegerField(default=0)
|
||||
passed_findings = models.IntegerField(default=0)
|
||||
total_findings = models.IntegerField(default=0)
|
||||
|
||||
scan = models.ForeignKey(
|
||||
Scan,
|
||||
|
||||
@@ -11,11 +11,12 @@ class APIJSONRenderer(JSONRenderer):
|
||||
def render(self, data, accepted_media_type=None, renderer_context=None):
|
||||
request = renderer_context.get("request")
|
||||
tenant_id = getattr(request, "tenant_id", None) if request else None
|
||||
db_alias = getattr(request, "db_alias", None) if request else None
|
||||
include_param_present = "include" in request.query_params if request else False
|
||||
|
||||
# Use rls_transaction if needed for included resources, otherwise do nothing
|
||||
context_manager = (
|
||||
rls_transaction(tenant_id)
|
||||
rls_transaction(tenant_id, using=db_alias)
|
||||
if tenant_id and include_param_present
|
||||
else nullcontext()
|
||||
)
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
from drf_spectacular.extensions import OpenApiAuthenticationExtension
|
||||
from drf_spectacular.openapi import AutoSchema
|
||||
|
||||
|
||||
class CombinedJWTOrAPIKeyAuthenticationScheme(OpenApiAuthenticationExtension):
|
||||
target_class = "api.authentication.CombinedJWTOrAPIKeyAuthentication"
|
||||
name = "JWT or API Key"
|
||||
|
||||
def get_security_definition(self, auto_schema: AutoSchema): # noqa: F841
|
||||
return {
|
||||
"type": "http",
|
||||
"scheme": "bearer",
|
||||
"bearerFormat": "JWT",
|
||||
"description": "Supports both JWT Bearer tokens and API Key authentication. "
|
||||
"Use `Bearer <token>` for JWT or `Api-Key <key>` for API keys.",
|
||||
}
|
||||
@@ -1,12 +1,12 @@
|
||||
from celery import states
|
||||
from celery.signals import before_task_publish
|
||||
from config.celery import celery_app
|
||||
from django.db.models.signals import post_delete
|
||||
from django.db.models.signals import post_delete, pre_delete
|
||||
from django.dispatch import receiver
|
||||
from django_celery_results.backends.database import DatabaseBackend
|
||||
|
||||
from api.db_utils import delete_related_daily_task
|
||||
from api.models import Provider
|
||||
from api.models import Membership, Provider, TenantAPIKey, User
|
||||
|
||||
|
||||
def create_task_result_on_publish(sender=None, headers=None, **kwargs): # noqa: F841
|
||||
@@ -32,3 +32,27 @@ before_task_publish.connect(
|
||||
def delete_provider_scan_task(sender, instance, **kwargs): # noqa: F841
|
||||
# Delete the associated periodic task when the provider is deleted
|
||||
delete_related_daily_task(instance.id)
|
||||
|
||||
|
||||
@receiver(pre_delete, sender=User)
|
||||
def revoke_user_api_keys(sender, instance, **kwargs): # noqa: F841
|
||||
"""
|
||||
Revoke all API keys associated with a user before deletion.
|
||||
|
||||
The entity field will be set to NULL by on_delete=SET_NULL,
|
||||
but we explicitly revoke the keys to prevent further use.
|
||||
"""
|
||||
TenantAPIKey.objects.filter(entity=instance).update(revoked=True)
|
||||
|
||||
|
||||
@receiver(post_delete, sender=Membership)
|
||||
def revoke_membership_api_keys(sender, instance, **kwargs): # noqa: F841
|
||||
"""
|
||||
Revoke all API keys when a user is removed from a tenant.
|
||||
|
||||
When a membership is deleted, all API keys created by that user
|
||||
in that tenant should be revoked to prevent further access.
|
||||
"""
|
||||
TenantAPIKey.objects.filter(
|
||||
entity=instance.user, tenant_id=instance.tenant.id
|
||||
).update(revoked=True)
|
||||
|
||||
+959
-114
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,384 @@
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from django.test import RequestFactory
|
||||
from rest_framework.exceptions import AuthenticationFailed
|
||||
|
||||
from api.authentication import TenantAPIKeyAuthentication
|
||||
from api.db_router import MainRouter
|
||||
from api.models import TenantAPIKey
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestTenantAPIKeyAuthentication:
|
||||
@pytest.fixture
|
||||
def auth_backend(self):
|
||||
"""Create an instance of TenantAPIKeyAuthentication."""
|
||||
return TenantAPIKeyAuthentication()
|
||||
|
||||
@pytest.fixture
|
||||
def request_factory(self):
|
||||
"""Create a Django request factory."""
|
||||
return RequestFactory()
|
||||
|
||||
def test_authenticate_credentials_uses_admin_database(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test that _authenticate_credentials routes queries to admin database."""
|
||||
api_key = api_keys_fixture[0]
|
||||
raw_key = api_key._raw_key
|
||||
|
||||
# Extract the encrypted key part (after the prefix and separator)
|
||||
_, encrypted_key = raw_key.split(TenantAPIKey.objects.separator, 1)
|
||||
|
||||
# Create a mock request
|
||||
request = request_factory.get("/")
|
||||
|
||||
# Call the method
|
||||
entity, auth_dict = auth_backend._authenticate_credentials(
|
||||
request, encrypted_key
|
||||
)
|
||||
|
||||
# Verify that the entity is the user associated with the API key
|
||||
assert entity == api_key.entity
|
||||
assert entity.id == api_key.entity.id
|
||||
|
||||
def test_authenticate_credentials_restores_manager_on_success(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test that the manager is restored after successful authentication."""
|
||||
api_key = api_keys_fixture[0]
|
||||
raw_key = api_key._raw_key
|
||||
_, encrypted_key = raw_key.split(TenantAPIKey.objects.separator, 1)
|
||||
|
||||
# Store the original manager
|
||||
original_manager = TenantAPIKey.objects
|
||||
|
||||
request = request_factory.get("/")
|
||||
|
||||
# Call the method
|
||||
auth_backend._authenticate_credentials(request, encrypted_key)
|
||||
|
||||
# Verify the manager was restored
|
||||
assert TenantAPIKey.objects == original_manager
|
||||
|
||||
def test_authenticate_credentials_restores_manager_on_exception(
|
||||
self, auth_backend, request_factory
|
||||
):
|
||||
"""Test that the manager is restored even when an exception occurs."""
|
||||
# Store the original manager
|
||||
original_manager = TenantAPIKey.objects
|
||||
|
||||
request = request_factory.get("/")
|
||||
|
||||
# Try to authenticate with an invalid key that will raise an exception
|
||||
with pytest.raises(Exception):
|
||||
auth_backend._authenticate_credentials(request, "invalid_encrypted_key")
|
||||
|
||||
# Verify the manager was restored despite the exception
|
||||
assert TenantAPIKey.objects == original_manager
|
||||
|
||||
def test_authenticate_valid_api_key(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test successful authentication with a valid API key."""
|
||||
api_key = api_keys_fixture[0]
|
||||
raw_key = api_key._raw_key
|
||||
|
||||
# Create a request with the API key in the Authorization header
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key}"
|
||||
|
||||
# Authenticate
|
||||
entity, auth_dict = auth_backend.authenticate(request)
|
||||
|
||||
# Verify the entity and auth dict
|
||||
assert entity == api_key.entity
|
||||
assert auth_dict["tenant_id"] == str(api_key.tenant_id)
|
||||
assert auth_dict["sub"] == str(api_key.entity.id)
|
||||
assert auth_dict["api_key_prefix"] == api_key.prefix
|
||||
|
||||
# Verify that last_used_at was updated
|
||||
api_key.refresh_from_db()
|
||||
assert api_key.last_used_at is not None
|
||||
assert (datetime.now(timezone.utc) - api_key.last_used_at).seconds < 5
|
||||
|
||||
def test_authenticate_valid_api_key_uses_admin_database(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test that authenticate uses admin database for API key lookup."""
|
||||
api_key = api_keys_fixture[0]
|
||||
raw_key = api_key._raw_key
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key}"
|
||||
|
||||
# Mock the manager's using method to verify it's called with admin_db
|
||||
with patch.object(
|
||||
TenantAPIKey.objects, "using", wraps=TenantAPIKey.objects.using
|
||||
) as mock_using:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
# Verify that .using('admin') was called
|
||||
mock_using.assert_called_with(MainRouter.admin_db)
|
||||
|
||||
def test_authenticate_invalid_key_format_missing_separator(
|
||||
self, auth_backend, request_factory
|
||||
):
|
||||
"""Test authentication fails with invalid API key format (no separator)."""
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = "Api-Key invalid_key_no_separator"
|
||||
|
||||
with pytest.raises(AuthenticationFailed) as exc_info:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
assert str(exc_info.value.detail) == "Invalid API Key."
|
||||
|
||||
def test_authenticate_invalid_key_format_empty_prefix(
|
||||
self, auth_backend, request_factory
|
||||
):
|
||||
"""Test authentication fails with empty prefix."""
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = "Api-Key .encrypted_part"
|
||||
|
||||
with pytest.raises(AuthenticationFailed) as exc_info:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
assert str(exc_info.value.detail) == "Invalid API Key."
|
||||
|
||||
def test_authenticate_invalid_encrypted_key(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test authentication fails with invalid encrypted key."""
|
||||
api_key = api_keys_fixture[0]
|
||||
|
||||
# Create an invalid key with valid prefix but invalid encryption
|
||||
invalid_key = f"{api_key.prefix}.invalid_encrypted_data"
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {invalid_key}"
|
||||
|
||||
with pytest.raises(AuthenticationFailed) as exc_info:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
assert str(exc_info.value.detail) == "Invalid API Key."
|
||||
|
||||
def test_authenticate_revoked_api_key(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test authentication fails with a revoked API key."""
|
||||
# Use the revoked API key (index 2 from fixture)
|
||||
api_key = api_keys_fixture[2]
|
||||
raw_key = api_key._raw_key
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key}"
|
||||
|
||||
# The revoked key should fail during credential validation
|
||||
with pytest.raises(AuthenticationFailed) as exc_info:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
assert str(exc_info.value.detail) == "This API Key has been revoked."
|
||||
|
||||
def test_authenticate_expired_api_key(
|
||||
self, auth_backend, create_test_user, tenants_fixture, request_factory
|
||||
):
|
||||
"""Test authentication fails with an expired API key."""
|
||||
tenant = tenants_fixture[0]
|
||||
user = create_test_user
|
||||
|
||||
# Create an expired API key
|
||||
api_key, raw_key = TenantAPIKey.objects.create_api_key(
|
||||
name="Expired API Key",
|
||||
tenant_id=tenant.id,
|
||||
entity=user,
|
||||
expiry_date=datetime.now(timezone.utc) - timedelta(days=1),
|
||||
)
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key}"
|
||||
|
||||
with pytest.raises(AuthenticationFailed) as exc_info:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
assert str(exc_info.value.detail) == "API Key has already expired."
|
||||
|
||||
def test_authenticate_nonexistent_api_key(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test authentication fails when API key doesn't exist in database."""
|
||||
# Create a valid-looking encrypted key with a non-existent UUID
|
||||
api_key = api_keys_fixture[0]
|
||||
non_existent_uuid = str(uuid4())
|
||||
|
||||
# Manually create an encrypted key with a non-existent ID
|
||||
payload = {
|
||||
"_pk": non_existent_uuid,
|
||||
"_exp": (datetime.now(timezone.utc) + timedelta(days=30)).timestamp(),
|
||||
}
|
||||
encrypted_key = auth_backend.key_crypto.generate(payload)
|
||||
fake_key = f"{api_key.prefix}.{encrypted_key}"
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {fake_key}"
|
||||
|
||||
with pytest.raises(AuthenticationFailed) as exc_info:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
assert str(exc_info.value.detail) == "No entity matching this api key."
|
||||
|
||||
def test_authenticate_updates_last_used_at(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test that last_used_at is updated on successful authentication."""
|
||||
api_key = api_keys_fixture[0]
|
||||
raw_key = api_key._raw_key
|
||||
|
||||
# Store the original last_used_at
|
||||
original_last_used = api_key.last_used_at
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key}"
|
||||
|
||||
# Authenticate
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
# Refresh from database
|
||||
api_key.refresh_from_db()
|
||||
|
||||
# Verify last_used_at was updated
|
||||
assert api_key.last_used_at is not None
|
||||
if original_last_used:
|
||||
assert api_key.last_used_at > original_last_used
|
||||
|
||||
def test_authenticate_saves_to_admin_database(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test that the API key save operation uses admin database."""
|
||||
api_key = api_keys_fixture[0]
|
||||
raw_key = api_key._raw_key
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key}"
|
||||
|
||||
# Mock the save method to verify it's called with using='admin'
|
||||
with patch.object(TenantAPIKey, "save") as mock_save:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
# Verify save was called with using=admin_db
|
||||
mock_save.assert_called_once_with(
|
||||
update_fields=["last_used_at"], using=MainRouter.admin_db
|
||||
)
|
||||
|
||||
def test_authenticate_returns_correct_auth_dict(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test that the auth dict contains all required fields."""
|
||||
api_key = api_keys_fixture[0]
|
||||
raw_key = api_key._raw_key
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key}"
|
||||
|
||||
entity, auth_dict = auth_backend.authenticate(request)
|
||||
|
||||
# Verify all required fields are present
|
||||
assert "tenant_id" in auth_dict
|
||||
assert "sub" in auth_dict
|
||||
assert "api_key_prefix" in auth_dict
|
||||
|
||||
# Verify values are correct
|
||||
assert auth_dict["tenant_id"] == str(api_key.tenant_id)
|
||||
assert auth_dict["sub"] == str(api_key.entity.id)
|
||||
assert auth_dict["api_key_prefix"] == api_key.prefix
|
||||
|
||||
def test_authenticate_with_multiple_api_keys_same_tenant(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test that authentication works correctly with multiple API keys for the same tenant."""
|
||||
# Test with first API key
|
||||
api_key1 = api_keys_fixture[0]
|
||||
raw_key1 = api_key1._raw_key
|
||||
|
||||
request1 = request_factory.get("/")
|
||||
request1.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key1}"
|
||||
|
||||
entity1, auth_dict1 = auth_backend.authenticate(request1)
|
||||
|
||||
assert entity1 == api_key1.entity
|
||||
assert auth_dict1["api_key_prefix"] == api_key1.prefix
|
||||
|
||||
# Test with second API key
|
||||
api_key2 = api_keys_fixture[1]
|
||||
raw_key2 = api_key2._raw_key
|
||||
|
||||
request2 = request_factory.get("/")
|
||||
request2.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key2}"
|
||||
|
||||
entity2, auth_dict2 = auth_backend.authenticate(request2)
|
||||
|
||||
assert entity2 == api_key2.entity
|
||||
assert auth_dict2["api_key_prefix"] == api_key2.prefix
|
||||
|
||||
# Verify they're different keys but same tenant
|
||||
assert auth_dict1["api_key_prefix"] != auth_dict2["api_key_prefix"]
|
||||
assert auth_dict1["tenant_id"] == auth_dict2["tenant_id"]
|
||||
|
||||
def test_authenticate_with_wrong_prefix_in_db(
|
||||
self, auth_backend, api_keys_fixture, request_factory
|
||||
):
|
||||
"""Test authentication fails when prefix doesn't match database."""
|
||||
api_key = api_keys_fixture[0]
|
||||
raw_key = api_key._raw_key
|
||||
|
||||
# Extract the encrypted part and combine with wrong prefix
|
||||
_, encrypted_part = raw_key.split(TenantAPIKey.objects.separator, 1)
|
||||
wrong_key = f"pk_wrong123.{encrypted_part}"
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {wrong_key}"
|
||||
|
||||
with pytest.raises(AuthenticationFailed) as exc_info:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
assert str(exc_info.value.detail) == "Invalid API Key."
|
||||
|
||||
def test_authenticate_credentials_exception_handling(
|
||||
self, auth_backend, request_factory
|
||||
):
|
||||
"""Test that exceptions in _authenticate_credentials are properly handled."""
|
||||
request = request_factory.get("/")
|
||||
|
||||
# Test with completely invalid data that will cause InvalidToken
|
||||
with pytest.raises(Exception):
|
||||
auth_backend._authenticate_credentials(request, "completely_invalid")
|
||||
|
||||
def test_authenticate_with_expired_timestamp(
|
||||
self, auth_backend, create_test_user, tenants_fixture, request_factory
|
||||
):
|
||||
"""Test that expired timestamp in encrypted key causes authentication failure."""
|
||||
tenant = tenants_fixture[0]
|
||||
user = create_test_user
|
||||
|
||||
# Create an API key with a very short expiry
|
||||
api_key, raw_key = TenantAPIKey.objects.create_api_key(
|
||||
name="Short-lived API Key",
|
||||
tenant_id=tenant.id,
|
||||
entity=user,
|
||||
expiry_date=datetime.now(timezone.utc) + timedelta(seconds=1),
|
||||
)
|
||||
|
||||
# Wait for the key to expire
|
||||
time.sleep(2)
|
||||
|
||||
request = request_factory.get("/")
|
||||
request.META["HTTP_AUTHORIZATION"] = f"Api-Key {raw_key}"
|
||||
|
||||
# Should fail with expired key
|
||||
with pytest.raises(AuthenticationFailed) as exc_info:
|
||||
auth_backend.authenticate(request)
|
||||
|
||||
assert str(exc_info.value.detail) == "API Key has already expired."
|
||||
@@ -11,6 +11,7 @@ from api.db_utils import (
|
||||
batch_delete,
|
||||
create_objects_in_batches,
|
||||
enum_to_choices,
|
||||
generate_api_key_prefix,
|
||||
generate_random_token,
|
||||
one_week_from_now,
|
||||
update_objects_in_batches,
|
||||
@@ -313,3 +314,28 @@ class TestUpdateObjectsInBatches:
|
||||
|
||||
qs = Provider.objects.filter(tenant=tenant, uid__endswith="_upd")
|
||||
assert qs.count() == total
|
||||
|
||||
|
||||
class TestGenerateApiKeyPrefix:
|
||||
def test_prefix_format(self):
|
||||
"""Test that generated prefix starts with 'pk_'."""
|
||||
prefix = generate_api_key_prefix()
|
||||
assert prefix.startswith("pk_")
|
||||
|
||||
def test_prefix_length(self):
|
||||
"""Test that prefix has correct length (pk_ + 8 random chars = 11)."""
|
||||
prefix = generate_api_key_prefix()
|
||||
assert len(prefix) == 11
|
||||
|
||||
def test_prefix_uniqueness(self):
|
||||
"""Test that multiple generations produce unique prefixes."""
|
||||
prefixes = {generate_api_key_prefix() for _ in range(100)}
|
||||
assert len(prefixes) == 100
|
||||
|
||||
def test_prefix_character_set(self):
|
||||
"""Test that random part uses only allowed characters."""
|
||||
allowed_chars = "23456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||
for _ in range(50):
|
||||
prefix = generate_api_key_prefix()
|
||||
random_part = prefix[3:] # Strip 'pk_'
|
||||
assert all(char in allowed_chars for char in random_part)
|
||||
|
||||
@@ -24,6 +24,7 @@ def test_api_logging_middleware_logging(mock_logger):
|
||||
mock_extract_auth_info.return_value = {
|
||||
"user_id": "user123",
|
||||
"tenant_id": "tenant456",
|
||||
"api_key_prefix": "pk_test",
|
||||
}
|
||||
|
||||
with patch("api.middleware.logging.getLogger") as mock_get_logger:
|
||||
@@ -44,6 +45,7 @@ def test_api_logging_middleware_logging(mock_logger):
|
||||
expected_extra = {
|
||||
"user_id": "user123",
|
||||
"tenant_id": "tenant456",
|
||||
"api_key_prefix": "pk_test",
|
||||
"method": "GET",
|
||||
"path": "/test-path",
|
||||
"query_params": {"param1": "value1", "param2": "value2"},
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -115,6 +115,40 @@ from rest_framework_json_api import serializers
|
||||
"description": "The Azure tenant ID, representing the directory where the application is "
|
||||
"registered.",
|
||||
},
|
||||
"user": {
|
||||
"type": "email",
|
||||
"description": "Deprecated: User microsoft email address.",
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
"description": "Deprecated: User password.",
|
||||
},
|
||||
},
|
||||
"required": [
|
||||
"client_id",
|
||||
"client_secret",
|
||||
"tenant_id",
|
||||
"user",
|
||||
"password",
|
||||
],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"title": "M365 Certificate Credentials",
|
||||
"properties": {
|
||||
"client_id": {
|
||||
"type": "string",
|
||||
"description": "The Azure application (client) ID for authentication in Azure AD.",
|
||||
},
|
||||
"tenant_id": {
|
||||
"type": "string",
|
||||
"description": "The Azure tenant ID, representing the directory where the application is "
|
||||
"registered.",
|
||||
},
|
||||
"certificate_content": {
|
||||
"type": "string",
|
||||
"description": "The certificate content in base64 format for certificate-based authentication.",
|
||||
},
|
||||
"user": {
|
||||
"type": "email",
|
||||
"description": "User microsoft email address.",
|
||||
@@ -126,8 +160,8 @@ from rest_framework_json_api import serializers
|
||||
},
|
||||
"required": [
|
||||
"client_id",
|
||||
"client_secret",
|
||||
"tenant_id",
|
||||
"certificate_content",
|
||||
"user",
|
||||
"password",
|
||||
],
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import base64
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
@@ -39,6 +40,7 @@ from api.models import (
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
Task,
|
||||
TenantAPIKey,
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
@@ -316,6 +318,23 @@ class UserSerializer(BaseSerializerV1):
|
||||
)
|
||||
|
||||
|
||||
class UserIncludeSerializer(UserSerializer):
|
||||
class Meta:
|
||||
model = User
|
||||
fields = [
|
||||
"id",
|
||||
"name",
|
||||
"email",
|
||||
"company_name",
|
||||
"date_joined",
|
||||
"roles",
|
||||
]
|
||||
|
||||
included_serializers = {
|
||||
"roles": "api.v1.serializers.RoleIncludeSerializer",
|
||||
}
|
||||
|
||||
|
||||
class UserCreateSerializer(BaseWriteSerializer):
|
||||
password = serializers.CharField(write_only=True)
|
||||
company_name = serializers.CharField(required=False)
|
||||
@@ -908,6 +927,17 @@ class ProviderCreateSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
"uid",
|
||||
# "scanner_args"
|
||||
]
|
||||
extra_kwargs = {
|
||||
"alias": {
|
||||
"help_text": "Human readable name to identify the provider, e.g. 'Production AWS Account', 'Dev Environment'",
|
||||
},
|
||||
"provider": {
|
||||
"help_text": "Type of provider to create.",
|
||||
},
|
||||
"uid": {
|
||||
"help_text": "Unique identifier for the provider, set by the provider, e.g. AWS account ID, Azure subscription ID, GCP project ID, etc.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ProviderUpdateSerializer(BaseWriteSerializer):
|
||||
@@ -922,6 +952,11 @@ class ProviderUpdateSerializer(BaseWriteSerializer):
|
||||
"alias",
|
||||
# "scanner_args"
|
||||
]
|
||||
extra_kwargs = {
|
||||
"alias": {
|
||||
"help_text": "Human readable name to identify the provider, e.g. 'Production AWS Account', 'Dev Environment'",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Scans
|
||||
@@ -1361,10 +1396,38 @@ class AzureProviderSecret(serializers.Serializer):
|
||||
|
||||
class M365ProviderSecret(serializers.Serializer):
|
||||
client_id = serializers.CharField()
|
||||
client_secret = serializers.CharField()
|
||||
client_secret = serializers.CharField(required=False)
|
||||
tenant_id = serializers.CharField()
|
||||
user = serializers.EmailField(required=False)
|
||||
password = serializers.CharField(required=False)
|
||||
certificate_content = serializers.CharField(required=False)
|
||||
|
||||
def validate(self, attrs):
|
||||
if attrs.get("client_secret") and attrs.get("certificate_content"):
|
||||
raise serializers.ValidationError(
|
||||
"You cannot provide both client_secret and certificate_content."
|
||||
)
|
||||
if not attrs.get("client_secret") and not attrs.get("certificate_content"):
|
||||
raise serializers.ValidationError(
|
||||
"You must provide either client_secret or certificate_content."
|
||||
)
|
||||
return super().validate(attrs)
|
||||
|
||||
def validate_certificate_content(self, certificate_content):
|
||||
"""Validate that M365 certificate content is valid base64 encoded data."""
|
||||
if certificate_content:
|
||||
try:
|
||||
base64.b64decode(certificate_content, validate=True)
|
||||
except Exception as e:
|
||||
raise ValidationError(
|
||||
{
|
||||
"certificate_content": [
|
||||
f"The provided certificate content is not valid base64 encoded data: {str(e)}"
|
||||
]
|
||||
},
|
||||
code="m365-certificate-content",
|
||||
)
|
||||
return certificate_content
|
||||
|
||||
class Meta:
|
||||
resource_name = "provider-secrets"
|
||||
@@ -1957,6 +2020,17 @@ class ComplianceOverviewDetailSerializer(serializers.Serializer):
|
||||
resource_name = "compliance-requirements-details"
|
||||
|
||||
|
||||
class ComplianceOverviewDetailThreatscoreSerializer(ComplianceOverviewDetailSerializer):
|
||||
"""
|
||||
Serializer for detailed compliance requirement information for Threatscore.
|
||||
|
||||
Includes additional fields specific to the Threatscore framework.
|
||||
"""
|
||||
|
||||
passed_findings = serializers.IntegerField()
|
||||
total_findings = serializers.IntegerField()
|
||||
|
||||
|
||||
class ComplianceOverviewAttributesSerializer(serializers.Serializer):
|
||||
id = serializers.CharField()
|
||||
compliance_name = serializers.CharField()
|
||||
@@ -2735,3 +2809,125 @@ class LighthouseConfigUpdateSerializer(BaseWriteSerializer):
|
||||
instance.api_key_decoded = api_key
|
||||
instance.save()
|
||||
return instance
|
||||
|
||||
|
||||
# API Keys
|
||||
|
||||
|
||||
class TenantApiKeySerializer(RLSSerializer):
|
||||
"""
|
||||
Serializer for the TenantApiKey model.
|
||||
"""
|
||||
|
||||
# Map database field names to API field names for consistency
|
||||
expires_at = serializers.DateTimeField(source="expiry_date", read_only=True)
|
||||
inserted_at = serializers.DateTimeField(source="created", read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = TenantAPIKey
|
||||
fields = [
|
||||
"id",
|
||||
"name",
|
||||
"prefix",
|
||||
"expires_at",
|
||||
"revoked",
|
||||
"inserted_at",
|
||||
"last_used_at",
|
||||
"entity",
|
||||
]
|
||||
|
||||
included_serializers = {
|
||||
"entity": "api.v1.serializers.UserIncludeSerializer",
|
||||
}
|
||||
|
||||
|
||||
class TenantApiKeyCreateSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
"""Serializer for creating new API keys."""
|
||||
|
||||
# Map database field names to API field names for consistency
|
||||
expires_at = serializers.DateTimeField(source="expiry_date", required=False)
|
||||
inserted_at = serializers.DateTimeField(source="created", read_only=True)
|
||||
api_key = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = TenantAPIKey
|
||||
fields = [
|
||||
"id",
|
||||
"name",
|
||||
"prefix",
|
||||
"expires_at",
|
||||
"revoked",
|
||||
"entity",
|
||||
"inserted_at",
|
||||
"last_used_at",
|
||||
"api_key",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
"prefix": {"read_only": True},
|
||||
"revoked": {"read_only": True},
|
||||
"entity": {"read_only": True},
|
||||
"inserted_at": {"read_only": True},
|
||||
"last_used_at": {"read_only": True},
|
||||
"api_key": {"read_only": True},
|
||||
}
|
||||
|
||||
def validate_name(self, value):
|
||||
"""Validate that the name is unique within the tenant."""
|
||||
tenant_id = self.context.get("tenant_id")
|
||||
if TenantAPIKey.objects.filter(tenant_id=tenant_id, name=value).exists():
|
||||
raise ValidationError("An API key with this name already exists.")
|
||||
return value
|
||||
|
||||
def get_api_key(self, obj):
|
||||
"""Return the raw API key if it was stored during creation."""
|
||||
return getattr(obj, "_raw_api_key", None)
|
||||
|
||||
def create(self, validated_data):
|
||||
instance, raw_api_key = TenantAPIKey.objects.create_api_key(
|
||||
**validated_data,
|
||||
tenant_id=self.context.get("tenant_id"),
|
||||
entity=self.context.get("request").user,
|
||||
)
|
||||
# Store the raw API key temporarily on the instance for the serializer
|
||||
instance._raw_api_key = raw_api_key
|
||||
return instance
|
||||
|
||||
|
||||
class TenantApiKeyUpdateSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
"""Serializer for updating API keys - only allows changing the name."""
|
||||
|
||||
# Map database field names to API field names for consistency
|
||||
expires_at = serializers.DateTimeField(source="expiry_date", read_only=True)
|
||||
inserted_at = serializers.DateTimeField(source="created", read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = TenantAPIKey
|
||||
fields = [
|
||||
"id",
|
||||
"name",
|
||||
"prefix",
|
||||
"expires_at",
|
||||
"entity",
|
||||
"inserted_at",
|
||||
"last_used_at",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
"prefix": {"read_only": True},
|
||||
"entity": {"read_only": True},
|
||||
"expires_at": {"read_only": True},
|
||||
"inserted_at": {"read_only": True},
|
||||
"last_used_at": {"read_only": True},
|
||||
}
|
||||
|
||||
def validate_name(self, value):
|
||||
"""Validate that the name is unique within the tenant, excluding current instance."""
|
||||
tenant_id = self.context.get("tenant_id")
|
||||
if (
|
||||
TenantAPIKey.objects.filter(tenant_id=tenant_id, name=value)
|
||||
.exclude(id=self.instance.id)
|
||||
.exists()
|
||||
):
|
||||
raise ValidationError("An API key with this name already exists.")
|
||||
return value
|
||||
|
||||
@@ -39,6 +39,7 @@ from api.v1.views import (
|
||||
TenantViewSet,
|
||||
UserRoleRelationshipView,
|
||||
UserViewSet,
|
||||
TenantApiKeyViewSet,
|
||||
)
|
||||
|
||||
router = routers.DefaultRouter(trailing_slash=False)
|
||||
@@ -65,6 +66,7 @@ router.register(
|
||||
LighthouseConfigViewSet,
|
||||
basename="lighthouseconfiguration",
|
||||
)
|
||||
router.register(r"api-keys", TenantApiKeyViewSet, basename="api-key")
|
||||
|
||||
tenants_router = routers.NestedSimpleRouter(router, r"tenants", lookup="tenant")
|
||||
tenants_router.register(
|
||||
|
||||
+156
-38
@@ -95,6 +95,7 @@ from api.filters import (
|
||||
ScanSummarySeverityFilter,
|
||||
ServiceOverviewFilter,
|
||||
TaskFilter,
|
||||
TenantApiKeyFilter,
|
||||
TenantFilter,
|
||||
UserFilter,
|
||||
)
|
||||
@@ -124,6 +125,7 @@ from api.models import (
|
||||
SeverityChoices,
|
||||
StateChoices,
|
||||
Task,
|
||||
TenantAPIKey,
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
@@ -140,6 +142,7 @@ from api.v1.mixins import PaginateByPkMixin, TaskManagementMixin
|
||||
from api.v1.serializers import (
|
||||
ComplianceOverviewAttributesSerializer,
|
||||
ComplianceOverviewDetailSerializer,
|
||||
ComplianceOverviewDetailThreatscoreSerializer,
|
||||
ComplianceOverviewMetadataSerializer,
|
||||
ComplianceOverviewSerializer,
|
||||
FindingDynamicFilterSerializer,
|
||||
@@ -189,6 +192,9 @@ from api.v1.serializers import (
|
||||
ScanUpdateSerializer,
|
||||
ScheduleDailyCreateSerializer,
|
||||
TaskSerializer,
|
||||
TenantApiKeyCreateSerializer,
|
||||
TenantApiKeySerializer,
|
||||
TenantApiKeyUpdateSerializer,
|
||||
TenantSerializer,
|
||||
TokenRefreshSerializer,
|
||||
TokenSerializer,
|
||||
@@ -387,6 +393,11 @@ class SchemaView(SpectacularAPIView):
|
||||
"description": "Endpoints for Single Sign-On authentication management via SAML for seamless user "
|
||||
"authentication.",
|
||||
},
|
||||
{
|
||||
"name": "API Keys",
|
||||
"description": "Endpoints for API keys management. These can be used as an alternative to JWT "
|
||||
"authorization.",
|
||||
},
|
||||
]
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
@@ -655,36 +666,48 @@ class TenantFinishACSView(FinishACSView):
|
||||
.get(email_domain=email_domain)
|
||||
.tenant
|
||||
)
|
||||
role_name = (
|
||||
extra.get("userType", ["no_permissions"])[0].strip()
|
||||
if extra.get("userType")
|
||||
else "no_permissions"
|
||||
|
||||
# Check if tenant has only one user with MANAGE_ACCOUNT role
|
||||
users_with_manage_account = (
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db)
|
||||
.filter(role__manage_account=True, tenant_id=tenant.id)
|
||||
.values("user")
|
||||
.distinct()
|
||||
.count()
|
||||
)
|
||||
try:
|
||||
role = Role.objects.using(MainRouter.admin_db).get(
|
||||
name=role_name, tenant=tenant
|
||||
|
||||
# Only apply role mapping from userType if tenant does NOT have exactly one user with MANAGE_ACCOUNT
|
||||
if users_with_manage_account != 1:
|
||||
role_name = (
|
||||
extra.get("userType", ["no_permissions"])[0].strip()
|
||||
if extra.get("userType")
|
||||
else "no_permissions"
|
||||
)
|
||||
except Role.DoesNotExist:
|
||||
role = Role.objects.using(MainRouter.admin_db).create(
|
||||
name=role_name,
|
||||
tenant=tenant,
|
||||
manage_users=False,
|
||||
manage_account=False,
|
||||
manage_billing=False,
|
||||
manage_providers=False,
|
||||
manage_integrations=False,
|
||||
manage_scans=False,
|
||||
unlimited_visibility=False,
|
||||
try:
|
||||
role = Role.objects.using(MainRouter.admin_db).get(
|
||||
name=role_name, tenant=tenant
|
||||
)
|
||||
except Role.DoesNotExist:
|
||||
role = Role.objects.using(MainRouter.admin_db).create(
|
||||
name=role_name,
|
||||
tenant=tenant,
|
||||
manage_users=False,
|
||||
manage_account=False,
|
||||
manage_billing=False,
|
||||
manage_providers=False,
|
||||
manage_integrations=False,
|
||||
manage_scans=False,
|
||||
unlimited_visibility=False,
|
||||
)
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).filter(
|
||||
user=user,
|
||||
tenant_id=tenant.id,
|
||||
).delete()
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
user=user,
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).filter(
|
||||
user=user,
|
||||
tenant_id=tenant.id,
|
||||
).delete()
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
user=user,
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
membership, _ = Membership.objects.using(MainRouter.admin_db).get_or_create(
|
||||
user=user,
|
||||
tenant=tenant,
|
||||
@@ -811,7 +834,9 @@ class UserViewSet(BaseUserViewset):
|
||||
if kwargs["pk"] != str(self.request.user.id):
|
||||
raise ValidationError("Only the current user can be deleted.")
|
||||
|
||||
return super().destroy(request, *args, **kwargs)
|
||||
user = self.get_object()
|
||||
user.delete(using=MainRouter.admin_db)
|
||||
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
@extend_schema(
|
||||
parameters=[
|
||||
@@ -3424,15 +3449,16 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
)
|
||||
filtered_queryset = self.filter_queryset(self.get_queryset())
|
||||
|
||||
all_requirements = (
|
||||
filtered_queryset.values(
|
||||
"requirement_id", "framework", "version", "description"
|
||||
)
|
||||
.distinct()
|
||||
.annotate(
|
||||
total_instances=Count("id"),
|
||||
manual_count=Count("id", filter=Q(requirement_status="MANUAL")),
|
||||
)
|
||||
all_requirements = filtered_queryset.values(
|
||||
"requirement_id",
|
||||
"framework",
|
||||
"version",
|
||||
"description",
|
||||
).annotate(
|
||||
total_instances=Count("id"),
|
||||
manual_count=Count("id", filter=Q(requirement_status="MANUAL")),
|
||||
passed_findings_sum=Sum("passed_findings"),
|
||||
total_findings_sum=Sum("total_findings"),
|
||||
)
|
||||
|
||||
passed_instances = (
|
||||
@@ -3451,6 +3477,8 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
total_instances = requirement["total_instances"]
|
||||
passed_count = passed_counts.get(requirement_id, 0)
|
||||
is_manual = requirement["manual_count"] == total_instances
|
||||
passed_findings = requirement["passed_findings_sum"] or 0
|
||||
total_findings = requirement["total_findings_sum"] or 0
|
||||
if is_manual:
|
||||
requirement_status = "MANUAL"
|
||||
elif passed_count == total_instances:
|
||||
@@ -3465,10 +3493,19 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin):
|
||||
"version": requirement["version"],
|
||||
"description": requirement["description"],
|
||||
"status": requirement_status,
|
||||
"passed_findings": passed_findings,
|
||||
"total_findings": total_findings,
|
||||
}
|
||||
)
|
||||
|
||||
serializer = self.get_serializer(requirements_summary, many=True)
|
||||
# Use different serializer for threatscore framework
|
||||
if "threatscore" not in compliance_id:
|
||||
serializer = self.get_serializer(requirements_summary, many=True)
|
||||
else:
|
||||
serializer = ComplianceOverviewDetailThreatscoreSerializer(
|
||||
requirements_summary, many=True
|
||||
)
|
||||
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="attributes")
|
||||
@@ -4188,3 +4225,84 @@ class ProcessorViewSet(BaseRLSViewSet):
|
||||
elif self.action == "partial_update":
|
||||
return ProcessorUpdateSerializer
|
||||
return super().get_serializer_class()
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
tags=["API Keys"],
|
||||
summary="List API keys",
|
||||
description="Retrieve a list of API keys for the tenant, with filtering support.",
|
||||
),
|
||||
retrieve=extend_schema(
|
||||
tags=["API Keys"],
|
||||
summary="Retrieve API key details",
|
||||
description="Fetch detailed information about a specific API key by its ID.",
|
||||
),
|
||||
create=extend_schema(
|
||||
tags=["API Keys"],
|
||||
summary="Create a new API key",
|
||||
description="Create a new API key for the tenant.",
|
||||
),
|
||||
partial_update=extend_schema(
|
||||
tags=["API Keys"],
|
||||
summary="Partially update an API key",
|
||||
description="Modify certain fields of an existing API key without affecting other settings.",
|
||||
),
|
||||
revoke=extend_schema(
|
||||
tags=["API Keys"],
|
||||
summary="Revoke an API key",
|
||||
description="Revoke an API key by its ID. This action is irreversible and will prevent the key from being "
|
||||
"used.",
|
||||
request=None,
|
||||
responses={
|
||||
200: OpenApiResponse(
|
||||
response=TenantApiKeySerializer,
|
||||
description="API key was successfully revoked",
|
||||
)
|
||||
},
|
||||
),
|
||||
)
|
||||
class TenantApiKeyViewSet(BaseRLSViewSet):
|
||||
queryset = TenantAPIKey.objects.all()
|
||||
serializer_class = TenantApiKeySerializer
|
||||
filterset_class = TenantApiKeyFilter
|
||||
http_method_names = ["get", "post", "patch", "delete"]
|
||||
ordering = ["revoked", "-created"]
|
||||
ordering_fields = ["name", "prefix", "revoked", "inserted_at", "expires_at"]
|
||||
# RBAC required permissions
|
||||
required_permissions = [Permissions.MANAGE_ACCOUNT]
|
||||
|
||||
def get_queryset(self):
|
||||
queryset = TenantAPIKey.objects.filter(
|
||||
tenant_id=self.request.tenant_id
|
||||
).annotate(inserted_at=F("created"), expires_at=F("expiry_date"))
|
||||
return queryset
|
||||
|
||||
def get_serializer_class(self):
|
||||
if self.action == "create":
|
||||
return TenantApiKeyCreateSerializer
|
||||
elif self.action == "partial_update":
|
||||
return TenantApiKeyUpdateSerializer
|
||||
return super().get_serializer_class()
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
def destroy(self, request, *args, **kwargs):
|
||||
raise MethodNotAllowed(method="DESTROY")
|
||||
|
||||
@action(detail=True, methods=["delete"])
|
||||
def revoke(self, request, *args, **kwargs):
|
||||
instance = self.get_object()
|
||||
|
||||
# Check if already revoked
|
||||
if instance.revoked:
|
||||
raise ValidationError(
|
||||
{
|
||||
"detail": "API key is already revoked",
|
||||
}
|
||||
)
|
||||
|
||||
TenantAPIKey.objects.revoke_api_key(instance.pk)
|
||||
instance.refresh_from_db()
|
||||
|
||||
serializer = self.get_serializer(instance)
|
||||
return Response(data=serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@@ -48,6 +48,10 @@ class NDJSONFormatter(logging.Formatter):
|
||||
log_record["user_id"] = record.user_id
|
||||
if hasattr(record, "tenant_id"):
|
||||
log_record["tenant_id"] = record.tenant_id
|
||||
if hasattr(record, "api_key_prefix"):
|
||||
log_record["api_key_prefix"] = (
|
||||
record.api_key_prefix if record.api_key_prefix != "N/A" else None
|
||||
)
|
||||
if hasattr(record, "method"):
|
||||
log_record["method"] = record.method
|
||||
if hasattr(record, "path"):
|
||||
@@ -90,6 +94,9 @@ class HumanReadableFormatter(logging.Formatter):
|
||||
# Add REST API extra fields
|
||||
if hasattr(record, "user_id"):
|
||||
log_components.append(f"({record.user_id})")
|
||||
if hasattr(record, "api_key_prefix"):
|
||||
if record.api_key_prefix != "N/A":
|
||||
log_components.append(f"(API-Key {record.api_key_prefix})")
|
||||
if hasattr(record, "tenant_id"):
|
||||
log_components.append(f"[{record.tenant_id}]")
|
||||
if hasattr(record, "method"):
|
||||
|
||||
@@ -43,6 +43,7 @@ INSTALLED_APPS = [
|
||||
"allauth.socialaccount.providers.saml",
|
||||
"dj_rest_auth.registration",
|
||||
"rest_framework.authtoken",
|
||||
"drf_simple_apikey",
|
||||
]
|
||||
|
||||
MIDDLEWARE = [
|
||||
@@ -84,7 +85,7 @@ TEMPLATES = [
|
||||
REST_FRAMEWORK = {
|
||||
"DEFAULT_SCHEMA_CLASS": "drf_spectacular_jsonapi.schemas.openapi.JsonApiAutoSchema",
|
||||
"DEFAULT_AUTHENTICATION_CLASSES": (
|
||||
"rest_framework_simplejwt.authentication.JWTAuthentication",
|
||||
"api.authentication.CombinedJWTOrAPIKeyAuthentication",
|
||||
),
|
||||
"PAGE_SIZE": 10,
|
||||
"EXCEPTION_HANDLER": "api.exceptions.custom_exception_handler",
|
||||
@@ -220,7 +221,8 @@ SIMPLE_JWT = {
|
||||
"JTI_CLAIM": "jti",
|
||||
"USER_ID_FIELD": "id",
|
||||
"USER_ID_CLAIM": "sub",
|
||||
# Issuer and Audience claims, for the moment we will keep these values as default values, they may change in the future.
|
||||
# Issuer and Audience claims, for the moment we will keep these values as default values, they may change in the
|
||||
# future.
|
||||
"AUDIENCE": env.str("DJANGO_JWT_AUDIENCE", "https://api.prowler.com"),
|
||||
"ISSUER": env.str("DJANGO_JWT_ISSUER", "https://api.prowler.com"),
|
||||
# Additional security settings
|
||||
@@ -229,6 +231,13 @@ SIMPLE_JWT = {
|
||||
|
||||
SECRETS_ENCRYPTION_KEY = env.str("DJANGO_SECRETS_ENCRYPTION_KEY", "")
|
||||
|
||||
# DRF Simple API Key settings
|
||||
DRF_API_KEY = {
|
||||
"FERNET_SECRET": SECRETS_ENCRYPTION_KEY,
|
||||
"API_KEY_LIFETIME": 365,
|
||||
"AUTHENTICATION_KEYWORD_HEADER": "Api-Key",
|
||||
}
|
||||
|
||||
# Internationalization
|
||||
# https://docs.djangoproject.com/en/5.0/topics/i18n/
|
||||
|
||||
|
||||
@@ -5,24 +5,39 @@ DEBUG = env.bool("DJANGO_DEBUG", default=True)
|
||||
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"])
|
||||
|
||||
# Database
|
||||
default_db_name = env("POSTGRES_DB", default="prowler_db")
|
||||
default_db_user = env("POSTGRES_USER", default="prowler_user")
|
||||
default_db_password = env("POSTGRES_PASSWORD", default="prowler")
|
||||
default_db_host = env("POSTGRES_HOST", default="postgres-db")
|
||||
default_db_port = env("POSTGRES_PORT", default="5432")
|
||||
|
||||
DATABASES = {
|
||||
"prowler_user": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"NAME": env("POSTGRES_DB", default="prowler_db"),
|
||||
"USER": env("POSTGRES_USER", default="prowler_user"),
|
||||
"PASSWORD": env("POSTGRES_PASSWORD", default="prowler"),
|
||||
"HOST": env("POSTGRES_HOST", default="postgres-db"),
|
||||
"PORT": env("POSTGRES_PORT", default="5432"),
|
||||
"NAME": default_db_name,
|
||||
"USER": default_db_user,
|
||||
"PASSWORD": default_db_password,
|
||||
"HOST": default_db_host,
|
||||
"PORT": default_db_port,
|
||||
},
|
||||
"admin": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"NAME": env("POSTGRES_DB", default="prowler_db"),
|
||||
"NAME": default_db_name,
|
||||
"USER": env("POSTGRES_ADMIN_USER", default="prowler"),
|
||||
"PASSWORD": env("POSTGRES_ADMIN_PASSWORD", default="S3cret"),
|
||||
"HOST": env("POSTGRES_HOST", default="postgres-db"),
|
||||
"PORT": env("POSTGRES_PORT", default="5432"),
|
||||
"HOST": default_db_host,
|
||||
"PORT": default_db_port,
|
||||
},
|
||||
"replica": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"NAME": env("POSTGRES_REPLICA_DB", default=default_db_name),
|
||||
"USER": env("POSTGRES_REPLICA_USER", default=default_db_user),
|
||||
"PASSWORD": env("POSTGRES_REPLICA_PASSWORD", default=default_db_password),
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] = tuple( # noqa: F405
|
||||
|
||||
@@ -6,22 +6,37 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.
|
||||
|
||||
# Database
|
||||
# TODO Use Django database routers https://docs.djangoproject.com/en/5.0/topics/db/multi-db/#automatic-database-routing
|
||||
default_db_name = env("POSTGRES_DB")
|
||||
default_db_user = env("POSTGRES_USER")
|
||||
default_db_password = env("POSTGRES_PASSWORD")
|
||||
default_db_host = env("POSTGRES_HOST")
|
||||
default_db_port = env("POSTGRES_PORT")
|
||||
|
||||
DATABASES = {
|
||||
"prowler_user": {
|
||||
"ENGINE": "django.db.backends.postgresql",
|
||||
"NAME": env("POSTGRES_DB"),
|
||||
"USER": env("POSTGRES_USER"),
|
||||
"PASSWORD": env("POSTGRES_PASSWORD"),
|
||||
"HOST": env("POSTGRES_HOST"),
|
||||
"PORT": env("POSTGRES_PORT"),
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"NAME": default_db_name,
|
||||
"USER": default_db_user,
|
||||
"PASSWORD": default_db_password,
|
||||
"HOST": default_db_host,
|
||||
"PORT": default_db_port,
|
||||
},
|
||||
"admin": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"NAME": env("POSTGRES_DB"),
|
||||
"NAME": default_db_name,
|
||||
"USER": env("POSTGRES_ADMIN_USER"),
|
||||
"PASSWORD": env("POSTGRES_ADMIN_PASSWORD"),
|
||||
"HOST": env("POSTGRES_HOST"),
|
||||
"PORT": env("POSTGRES_PORT"),
|
||||
"HOST": default_db_host,
|
||||
"PORT": default_db_port,
|
||||
},
|
||||
"replica": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"NAME": env("POSTGRES_REPLICA_DB", default=default_db_name),
|
||||
"USER": env("POSTGRES_REPLICA_USER", default=default_db_user),
|
||||
"PASSWORD": env("POSTGRES_REPLICA_PASSWORD", default=default_db_password),
|
||||
"HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host),
|
||||
"PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port),
|
||||
},
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
@@ -20,6 +20,13 @@ DATABASE_ROUTERS = []
|
||||
TESTING = True
|
||||
SECRETS_ENCRYPTION_KEY = "ZMiYVo7m4Fbe2eXXPyrwxdJss2WSalXSv3xHBcJkPl0="
|
||||
|
||||
# DRF Simple API Key settings
|
||||
DRF_API_KEY = {
|
||||
"FERNET_SECRET": SECRETS_ENCRYPTION_KEY,
|
||||
"API_KEY_LIFETIME": 365,
|
||||
"AUTHENTICATION_KEYWORD_HEADER": "Api-Key",
|
||||
}
|
||||
|
||||
# JWT
|
||||
|
||||
SIMPLE_JWT["ALGORITHM"] = "HS256" # noqa: F405
|
||||
|
||||
@@ -38,6 +38,7 @@ from api.models import (
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
Task,
|
||||
TenantAPIKey,
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
@@ -1368,6 +1369,56 @@ def saml_sociallogin(users_fixture):
|
||||
return sociallogin
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_keys_fixture(tenants_fixture, create_test_user):
|
||||
"""Create test API keys for testing."""
|
||||
tenant = tenants_fixture[0]
|
||||
user = create_test_user
|
||||
|
||||
# Create and assign role to user for API key authentication
|
||||
role = Role.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
name="Test API Key Role",
|
||||
unlimited_visibility=True,
|
||||
manage_account=True,
|
||||
)
|
||||
UserRoleRelationship.objects.create(
|
||||
user=user,
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
# Create API keys with different states
|
||||
api_key1, raw_key1 = TenantAPIKey.objects.create_api_key(
|
||||
name="Test API Key 1",
|
||||
tenant_id=tenant.id,
|
||||
entity=user,
|
||||
)
|
||||
|
||||
api_key2, raw_key2 = TenantAPIKey.objects.create_api_key(
|
||||
name="Test API Key 2",
|
||||
tenant_id=tenant.id,
|
||||
entity=user,
|
||||
expiry_date=datetime.now(timezone.utc) + timedelta(days=60),
|
||||
)
|
||||
|
||||
# Revoked API key
|
||||
api_key3, raw_key3 = TenantAPIKey.objects.create_api_key(
|
||||
name="Revoked API Key",
|
||||
tenant_id=tenant.id,
|
||||
entity=user,
|
||||
)
|
||||
api_key3.revoked = True
|
||||
api_key3.save()
|
||||
|
||||
# Store raw keys on instances for testing
|
||||
api_key1._raw_key = raw_key1
|
||||
api_key2._raw_key = raw_key2
|
||||
api_key3._raw_key = raw_key3
|
||||
|
||||
return [api_key1, api_key2, api_key3]
|
||||
|
||||
|
||||
def get_authorization_header(access_token: str) -> dict:
|
||||
return {"Authorization": f"Bearer {access_token}"}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from celery.utils.log import get_task_logger
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
|
||||
from tasks.utils import batched
|
||||
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Finding, Integration, Provider
|
||||
from api.utils import initialize_prowler_integration, initialize_prowler_provider
|
||||
@@ -289,7 +290,7 @@ def upload_security_hub_integration(
|
||||
has_findings = False
|
||||
batch_number = 0
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
qs = (
|
||||
Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id)
|
||||
.order_by("uid")
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS
|
||||
@@ -14,8 +18,11 @@ from api.compliance import (
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
|
||||
generate_scan_compliance,
|
||||
)
|
||||
from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.db_utils import (
|
||||
create_objects_in_batches,
|
||||
POSTGRES_TENANT_VAR,
|
||||
SET_CONFIG_QUERY,
|
||||
psycopg_connection,
|
||||
rls_transaction,
|
||||
update_objects_in_batches,
|
||||
)
|
||||
@@ -40,6 +47,28 @@ from prowler.lib.scan.scan import Scan as ProwlerScan
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# Column order must match `ComplianceRequirementOverview` schema in
|
||||
# `api/models.py`. Keep this list minimal but sufficient to populate all
|
||||
# non-nullable fields plus the counters we care about.
|
||||
COMPLIANCE_REQUIREMENT_COPY_COLUMNS = (
|
||||
"id",
|
||||
"tenant_id",
|
||||
"inserted_at",
|
||||
"compliance_id",
|
||||
"framework",
|
||||
"version",
|
||||
"description",
|
||||
"region",
|
||||
"requirement_id",
|
||||
"requirement_status",
|
||||
"passed_checks",
|
||||
"failed_checks",
|
||||
"total_checks",
|
||||
"passed_findings",
|
||||
"total_findings",
|
||||
"scan_id",
|
||||
)
|
||||
|
||||
|
||||
def _create_finding_delta(
|
||||
last_status: FindingStatus | None | str, new_status: FindingStatus | None
|
||||
@@ -107,6 +136,124 @@ def _store_resources(
|
||||
return resource_instance, (resource_instance.uid, resource_instance.region)
|
||||
|
||||
|
||||
def _copy_compliance_requirement_rows(
|
||||
tenant_id: str, rows: list[dict[str, Any]]
|
||||
) -> None:
|
||||
"""Stream compliance requirement rows into Postgres using COPY.
|
||||
|
||||
We leverage the admin connection (when available) to bypass the COPY + RLS
|
||||
restriction, writing only the fields required by
|
||||
``ComplianceRequirementOverview``.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID.
|
||||
rows: List of row dictionaries prepared by
|
||||
:func:`create_compliance_requirements`.
|
||||
"""
|
||||
|
||||
csv_buffer = io.StringIO()
|
||||
writer = csv.writer(csv_buffer)
|
||||
|
||||
datetime_now = datetime.now(tz=timezone.utc)
|
||||
for row in rows:
|
||||
writer.writerow(
|
||||
[
|
||||
str(row.get("id")),
|
||||
str(row.get("tenant_id")),
|
||||
(row.get("inserted_at") or datetime_now).isoformat(),
|
||||
row.get("compliance_id") or "",
|
||||
row.get("framework") or "",
|
||||
row.get("version") or "",
|
||||
row.get("description") or "",
|
||||
row.get("region") or "",
|
||||
row.get("requirement_id") or "",
|
||||
row.get("requirement_status") or "",
|
||||
row.get("passed_checks", 0),
|
||||
row.get("failed_checks", 0),
|
||||
row.get("total_checks", 0),
|
||||
row.get("passed_findings", 0),
|
||||
row.get("total_findings", 0),
|
||||
str(row.get("scan_id")),
|
||||
]
|
||||
)
|
||||
|
||||
csv_buffer.seek(0)
|
||||
copy_sql = (
|
||||
"COPY compliance_requirements_overviews ("
|
||||
+ ", ".join(COMPLIANCE_REQUIREMENT_COPY_COLUMNS)
|
||||
+ ") FROM STDIN WITH (FORMAT CSV, DELIMITER ',', QUOTE '\"', ESCAPE '\"', NULL '\\N')"
|
||||
)
|
||||
|
||||
try:
|
||||
with psycopg_connection(MainRouter.admin_db) as connection:
|
||||
connection.autocommit = False
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id])
|
||||
cursor.copy_expert(copy_sql, csv_buffer)
|
||||
connection.commit()
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
finally:
|
||||
csv_buffer.close()
|
||||
|
||||
|
||||
def _persist_compliance_requirement_rows(
|
||||
tenant_id: str, rows: list[dict[str, Any]]
|
||||
) -> None:
|
||||
"""Persist compliance requirement rows using COPY with ORM fallback.
|
||||
|
||||
Args:
|
||||
tenant_id: Target tenant UUID.
|
||||
rows: Precomputed row dictionaries that reflect the compliance
|
||||
overview state for a scan.
|
||||
"""
|
||||
if not rows:
|
||||
return
|
||||
|
||||
try:
|
||||
_copy_compliance_requirement_rows(tenant_id, rows)
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
"COPY bulk insert for compliance requirements failed; falling back to ORM bulk_create",
|
||||
exc_info=error,
|
||||
)
|
||||
fallback_objects = [
|
||||
ComplianceRequirementOverview(
|
||||
id=row["id"],
|
||||
tenant_id=row["tenant_id"],
|
||||
inserted_at=row["inserted_at"],
|
||||
compliance_id=row["compliance_id"],
|
||||
framework=row["framework"],
|
||||
version=row["version"],
|
||||
description=row["description"],
|
||||
region=row["region"],
|
||||
requirement_id=row["requirement_id"],
|
||||
requirement_status=row["requirement_status"],
|
||||
passed_checks=row["passed_checks"],
|
||||
failed_checks=row["failed_checks"],
|
||||
total_checks=row["total_checks"],
|
||||
passed_findings=row.get("passed_findings", 0),
|
||||
total_findings=row.get("total_findings", 0),
|
||||
scan_id=row["scan_id"],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceRequirementOverview.objects.bulk_create(
|
||||
fallback_objects, batch_size=500
|
||||
)
|
||||
|
||||
|
||||
def _normalized_compliance_key(framework: str | None, version: str | None) -> str:
|
||||
"""Return normalized identifier used to group compliance totals."""
|
||||
|
||||
normalized_framework = (framework or "").lower().replace("-", "").replace("_", "")
|
||||
normalized_version = (version or "").lower().replace("-", "").replace("_", "")
|
||||
return f"{normalized_framework}{normalized_version}"
|
||||
|
||||
|
||||
def perform_prowler_scan(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
@@ -143,7 +290,7 @@ def perform_prowler_scan(
|
||||
scan_instance.save()
|
||||
|
||||
# Find the mutelist processor if it exists
|
||||
with rls_transaction(tenant_id):
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
try:
|
||||
mutelist_processor = Processor.objects.get(
|
||||
tenant_id=tenant_id, processor_type=Processor.ProcessorChoices.MUTELIST
|
||||
@@ -272,7 +419,7 @@ def perform_prowler_scan(
|
||||
unique_resources.add((resource_instance.uid, resource_instance.region))
|
||||
|
||||
# Process finding
|
||||
with rls_transaction(tenant_id):
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
finding_uid = finding.uid
|
||||
last_first_seen_at = None
|
||||
if finding_uid not in last_status_cache:
|
||||
@@ -305,6 +452,12 @@ def perform_prowler_scan(
|
||||
# If the finding is muted at this time the reason must be the configured Mutelist
|
||||
muted_reason = "Muted by mutelist" if finding.muted else None
|
||||
|
||||
# Increment failed_findings_count cache if the finding status is FAIL and not muted
|
||||
if status == FindingStatus.FAIL and not finding.muted:
|
||||
resource_uid = finding.resource_uid
|
||||
resource_failed_findings_cache[resource_uid] += 1
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
# Create the finding
|
||||
finding_instance = Finding.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
@@ -325,11 +478,6 @@ def perform_prowler_scan(
|
||||
)
|
||||
finding_instance.add_resources([resource_instance])
|
||||
|
||||
# Increment failed_findings_count cache if the finding status is FAIL and not muted
|
||||
if status == FindingStatus.FAIL and not finding.muted:
|
||||
resource_uid = finding.resource_uid
|
||||
resource_failed_findings_cache[resource_uid] += 1
|
||||
|
||||
# Update scan resource summaries
|
||||
scan_resource_cache.add(
|
||||
(
|
||||
@@ -439,7 +587,7 @@ def aggregate_findings(tenant_id: str, scan_id: str):
|
||||
- muted_new: Muted findings with a delta of 'new'.
|
||||
- muted_changed: Muted findings with a delta of 'changed'.
|
||||
"""
|
||||
with rls_transaction(tenant_id):
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
findings = Finding.objects.filter(tenant_id=tenant_id, scan_id=scan_id)
|
||||
|
||||
aggregation = findings.values(
|
||||
@@ -582,15 +730,32 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
ValidationError: If tenant_id is not a valid UUID.
|
||||
"""
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
scan_instance = Scan.objects.get(pk=scan_id)
|
||||
provider_instance = scan_instance.provider
|
||||
prowler_provider = return_prowler_provider(provider_instance)
|
||||
|
||||
compliance_template = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE[
|
||||
provider_instance.provider
|
||||
]
|
||||
modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0"
|
||||
threatscore_requirements_by_check: dict[str, set[str]] = {}
|
||||
threatscore_framework = compliance_template.get(
|
||||
modeled_threatscore_compliance_id
|
||||
)
|
||||
if threatscore_framework:
|
||||
for requirement_id, requirement in threatscore_framework[
|
||||
"requirements"
|
||||
].items():
|
||||
for check_id in requirement["checks"]:
|
||||
threatscore_requirements_by_check.setdefault(check_id, set()).add(
|
||||
requirement_id
|
||||
)
|
||||
|
||||
# Get check status data by region from findings
|
||||
findings = (
|
||||
Finding.all_objects.filter(scan_id=scan_id, muted=False)
|
||||
.only("id", "check_id", "status")
|
||||
.only("id", "check_id", "status", "compliance")
|
||||
.prefetch_related(
|
||||
Prefetch(
|
||||
"resources",
|
||||
@@ -601,14 +766,36 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
.iterator(chunk_size=1000)
|
||||
)
|
||||
|
||||
findings_count_by_compliance = {}
|
||||
check_status_by_region = {}
|
||||
with rls_transaction(tenant_id):
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
for finding in findings:
|
||||
for resource in finding.small_resources:
|
||||
region = resource.region
|
||||
current_status = check_status_by_region.setdefault(region, {})
|
||||
if current_status.get(finding.check_id) != "FAIL":
|
||||
current_status[finding.check_id] = finding.status
|
||||
if modeled_threatscore_compliance_id in finding.compliance:
|
||||
for requirement_id in finding.compliance[
|
||||
modeled_threatscore_compliance_id
|
||||
]:
|
||||
compliance_key = findings_count_by_compliance.setdefault(
|
||||
region, {}
|
||||
).setdefault(
|
||||
modeled_threatscore_compliance_id.lower().replace(
|
||||
"-", ""
|
||||
),
|
||||
{},
|
||||
)
|
||||
if requirement_id not in compliance_key:
|
||||
compliance_key[requirement_id] = {
|
||||
"total": 0,
|
||||
"pass": 0,
|
||||
}
|
||||
|
||||
compliance_key[requirement_id]["total"] += 1
|
||||
if finding.status == "PASS":
|
||||
compliance_key[requirement_id]["pass"] += 1
|
||||
|
||||
try:
|
||||
# Try to get regions from provider
|
||||
@@ -617,11 +804,6 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
# If not available, use regions from findings
|
||||
regions = set(check_status_by_region.keys())
|
||||
|
||||
# Get compliance template for the provider
|
||||
compliance_template = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE[
|
||||
provider_instance.provider
|
||||
]
|
||||
|
||||
# Create compliance data by region
|
||||
compliance_overview_by_region = {
|
||||
region: deepcopy(compliance_template) for region in regions
|
||||
@@ -640,36 +822,53 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
status,
|
||||
)
|
||||
|
||||
# Prepare compliance requirement objects
|
||||
compliance_requirement_objects = []
|
||||
# Prepare compliance requirement rows
|
||||
compliance_requirement_rows: list[dict[str, Any]] = []
|
||||
utc_datetime_now = datetime.now(tz=timezone.utc)
|
||||
for region, compliance_data in compliance_overview_by_region.items():
|
||||
for compliance_id, compliance in compliance_data.items():
|
||||
modeled_compliance_id = _normalized_compliance_key(
|
||||
compliance["framework"], compliance["version"]
|
||||
)
|
||||
# Create an overview record for each requirement within each compliance framework
|
||||
for requirement_id, requirement in compliance["requirements"].items():
|
||||
compliance_requirement_objects.append(
|
||||
ComplianceRequirementOverview(
|
||||
tenant_id=tenant_id,
|
||||
scan=scan_instance,
|
||||
region=region,
|
||||
compliance_id=compliance_id,
|
||||
framework=compliance["framework"],
|
||||
version=compliance["version"],
|
||||
requirement_id=requirement_id,
|
||||
description=requirement["description"],
|
||||
passed_checks=requirement["checks_status"]["pass"],
|
||||
failed_checks=requirement["checks_status"]["fail"],
|
||||
total_checks=requirement["checks_status"]["total"],
|
||||
requirement_status=requirement["status"],
|
||||
)
|
||||
checks_status = requirement["checks_status"]
|
||||
compliance_requirement_rows.append(
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": tenant_id,
|
||||
"inserted_at": utc_datetime_now,
|
||||
"compliance_id": compliance_id,
|
||||
"framework": compliance["framework"],
|
||||
"version": compliance["version"] or "",
|
||||
"description": requirement.get("description") or "",
|
||||
"region": region,
|
||||
"requirement_id": requirement_id,
|
||||
"requirement_status": requirement["status"],
|
||||
"passed_checks": checks_status["pass"],
|
||||
"failed_checks": checks_status["fail"],
|
||||
"total_checks": checks_status["total"],
|
||||
"scan_id": scan_instance.id,
|
||||
"passed_findings": findings_count_by_compliance.get(
|
||||
region, {}
|
||||
)
|
||||
.get(modeled_compliance_id, {})
|
||||
.get(requirement_id, {})
|
||||
.get("pass", 0),
|
||||
"total_findings": findings_count_by_compliance.get(
|
||||
region, {}
|
||||
)
|
||||
.get(modeled_compliance_id, {})
|
||||
.get(requirement_id, {})
|
||||
.get("total", 0),
|
||||
}
|
||||
)
|
||||
|
||||
# Bulk create requirement records
|
||||
create_objects_in_batches(
|
||||
tenant_id, ComplianceRequirementOverview, compliance_requirement_objects
|
||||
)
|
||||
# Bulk create requirement records using PostgreSQL COPY
|
||||
_persist_compliance_requirement_rows(tenant_id, compliance_requirement_rows)
|
||||
|
||||
return {
|
||||
"requirements_created": len(compliance_requirement_objects),
|
||||
"requirements_created": len(compliance_requirement_rows),
|
||||
"regions_processed": list(regions),
|
||||
"compliance_frameworks": (
|
||||
list(compliance_overview_by_region.get(list(regions)[0], {}).keys())
|
||||
|
||||
@@ -34,6 +34,7 @@ from tasks.jobs.scan import (
|
||||
from tasks.utils import batched, get_next_execution_datetime
|
||||
|
||||
from api.compliance import get_compliance_frameworks
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.decorators import set_tenant
|
||||
from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices
|
||||
@@ -343,70 +344,73 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str):
|
||||
.order_by("uid")
|
||||
.iterator()
|
||||
)
|
||||
for batch, is_last in batched(qs, DJANGO_FINDINGS_BATCH_SIZE):
|
||||
fos = [FindingOutput.transform_api_finding(f, prowler_provider) for f in batch]
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
for batch, is_last in batched(qs, DJANGO_FINDINGS_BATCH_SIZE):
|
||||
fos = [
|
||||
FindingOutput.transform_api_finding(f, prowler_provider) for f in batch
|
||||
]
|
||||
|
||||
# Outputs
|
||||
for mode, cfg in OUTPUT_FORMATS_MAPPING.items():
|
||||
# Skip ASFF generation if not needed
|
||||
if mode == "json-asff" and not generate_asff:
|
||||
continue
|
||||
# Outputs
|
||||
for mode, cfg in OUTPUT_FORMATS_MAPPING.items():
|
||||
# Skip ASFF generation if not needed
|
||||
if mode == "json-asff" and not generate_asff:
|
||||
continue
|
||||
|
||||
cls = cfg["class"]
|
||||
suffix = cfg["suffix"]
|
||||
extra = cfg.get("kwargs", {}).copy()
|
||||
if mode == "html":
|
||||
extra.update(provider=prowler_provider, stats=scan_summary)
|
||||
cls = cfg["class"]
|
||||
suffix = cfg["suffix"]
|
||||
extra = cfg.get("kwargs", {}).copy()
|
||||
if mode == "html":
|
||||
extra.update(provider=prowler_provider, stats=scan_summary)
|
||||
|
||||
writer, initialization = get_writer(
|
||||
output_writers,
|
||||
cls,
|
||||
lambda cls=cls, fos=fos, suffix=suffix: cls(
|
||||
findings=fos,
|
||||
file_path=out_dir,
|
||||
file_extension=suffix,
|
||||
from_cli=False,
|
||||
),
|
||||
is_last,
|
||||
)
|
||||
if not initialization:
|
||||
writer.transform(fos)
|
||||
writer.batch_write_data_to_file(**extra)
|
||||
writer._data.clear()
|
||||
writer, initialization = get_writer(
|
||||
output_writers,
|
||||
cls,
|
||||
lambda cls=cls, fos=fos, suffix=suffix: cls(
|
||||
findings=fos,
|
||||
file_path=out_dir,
|
||||
file_extension=suffix,
|
||||
from_cli=False,
|
||||
),
|
||||
is_last,
|
||||
)
|
||||
if not initialization:
|
||||
writer.transform(fos)
|
||||
writer.batch_write_data_to_file(**extra)
|
||||
writer._data.clear()
|
||||
|
||||
# Compliance CSVs
|
||||
for name in frameworks_avail:
|
||||
compliance_obj = frameworks_bulk[name]
|
||||
# Compliance CSVs
|
||||
for name in frameworks_avail:
|
||||
compliance_obj = frameworks_bulk[name]
|
||||
|
||||
klass = GenericCompliance
|
||||
for condition, cls in COMPLIANCE_CLASS_MAP.get(provider_type, []):
|
||||
if condition(name):
|
||||
klass = cls
|
||||
break
|
||||
klass = GenericCompliance
|
||||
for condition, cls in COMPLIANCE_CLASS_MAP.get(provider_type, []):
|
||||
if condition(name):
|
||||
klass = cls
|
||||
break
|
||||
|
||||
filename = f"{comp_dir}_{name}.csv"
|
||||
filename = f"{comp_dir}_{name}.csv"
|
||||
|
||||
writer, initialization = get_writer(
|
||||
compliance_writers,
|
||||
name,
|
||||
lambda klass=klass, fos=fos: klass(
|
||||
findings=fos,
|
||||
compliance=compliance_obj,
|
||||
file_path=filename,
|
||||
from_cli=False,
|
||||
),
|
||||
is_last,
|
||||
)
|
||||
if not initialization:
|
||||
writer.transform(fos, compliance_obj, name)
|
||||
writer.batch_write_data_to_file()
|
||||
writer._data.clear()
|
||||
writer, initialization = get_writer(
|
||||
compliance_writers,
|
||||
name,
|
||||
lambda klass=klass, fos=fos: klass(
|
||||
findings=fos,
|
||||
compliance=compliance_obj,
|
||||
file_path=filename,
|
||||
from_cli=False,
|
||||
),
|
||||
is_last,
|
||||
)
|
||||
if not initialization:
|
||||
writer.transform(fos, compliance_obj, name)
|
||||
writer.batch_write_data_to_file()
|
||||
writer._data.clear()
|
||||
|
||||
compressed = _compress_output_files(out_dir)
|
||||
upload_uri = _upload_to_s3(tenant_id, compressed, scan_id)
|
||||
|
||||
# S3 integrations (need output_directory)
|
||||
with rls_transaction(tenant_id):
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
s3_integrations = Integration.objects.filter(
|
||||
integrationproviderrelationship__provider_id=provider_id,
|
||||
integration_type=Integration.IntegrationChoices.AMAZON_S3,
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
import csv
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from io import StringIO
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from tasks.jobs.scan import (
|
||||
_copy_compliance_requirement_rows,
|
||||
_create_finding_delta,
|
||||
_persist_compliance_requirement_rows,
|
||||
_store_resources,
|
||||
create_compliance_requirements,
|
||||
perform_prowler_scan,
|
||||
)
|
||||
from tasks.utils import CustomEncoder
|
||||
|
||||
from api.db_router import MainRouter
|
||||
from api.exceptions import ProviderConnectionError
|
||||
from api.models import Finding, Provider, Resource, Scan, StateChoices, StatusChoices
|
||||
from prowler.lib.check.models import Severity
|
||||
@@ -1045,3 +1050,773 @@ class TestCreateComplianceRequirements:
|
||||
|
||||
assert "requirements_created" in result
|
||||
assert result["requirements_created"] >= 0
|
||||
|
||||
|
||||
class TestComplianceRequirementCopy:
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_streams_csv(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
def copy_side_effect(sql, file_obj):
|
||||
captured["sql"] = sql
|
||||
captured["data"] = file_obj.read()
|
||||
|
||||
cursor.copy_expert.side_effect = copy_side_effect
|
||||
|
||||
row = {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": str(uuid.uuid4()),
|
||||
"compliance_id": "cisa_aws",
|
||||
"framework": "CISA",
|
||||
"version": None,
|
||||
"description": "desc",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 1,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 1,
|
||||
"scan_id": uuid.uuid4(),
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
|
||||
mock_psycopg_connection.assert_called_once_with("admin")
|
||||
connection.cursor.assert_called_once()
|
||||
cursor.execute.assert_called_once()
|
||||
cursor.copy_expert.assert_called_once()
|
||||
|
||||
csv_rows = list(csv.reader(StringIO(captured["data"])))
|
||||
assert csv_rows[0][0] == str(row["id"])
|
||||
assert csv_rows[0][5] == ""
|
||||
assert csv_rows[0][-1] == str(row["scan_id"])
|
||||
|
||||
@patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
@patch(
|
||||
"tasks.jobs.scan._copy_compliance_requirement_rows",
|
||||
side_effect=Exception("copy failed"),
|
||||
)
|
||||
def test_persist_compliance_requirement_rows_fallback(
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create
|
||||
):
|
||||
inserted_at = datetime.now(timezone.utc)
|
||||
row = {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": str(uuid.uuid4()),
|
||||
"inserted_at": inserted_at,
|
||||
"compliance_id": "cisa_aws",
|
||||
"framework": "CISA",
|
||||
"version": "1.0",
|
||||
"description": "desc",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 1,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 1,
|
||||
"scan_id": uuid.uuid4(),
|
||||
}
|
||||
|
||||
tenant_id = row["tenant_id"]
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
|
||||
_persist_compliance_requirement_rows(tenant_id, [row])
|
||||
|
||||
mock_copy.assert_called_once_with(tenant_id, [row])
|
||||
mock_rls_transaction.assert_called_once_with(tenant_id)
|
||||
mock_bulk_create.assert_called_once()
|
||||
|
||||
args, kwargs = mock_bulk_create.call_args
|
||||
objects = args[0]
|
||||
assert len(objects) == 1
|
||||
fallback = objects[0]
|
||||
assert fallback.version == row["version"]
|
||||
assert fallback.compliance_id == row["compliance_id"]
|
||||
|
||||
@patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
@patch("tasks.jobs.scan._copy_compliance_requirement_rows")
|
||||
def test_persist_compliance_requirement_rows_no_rows(
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create
|
||||
):
|
||||
_persist_compliance_requirement_rows(str(uuid.uuid4()), [])
|
||||
|
||||
mock_copy.assert_not_called()
|
||||
mock_rls_transaction.assert_not_called()
|
||||
mock_bulk_create.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_multiple_rows(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
"""Test COPY with multiple rows to ensure batch processing works correctly."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
def copy_side_effect(sql, file_obj):
|
||||
captured["sql"] = sql
|
||||
captured["data"] = file_obj.read()
|
||||
|
||||
cursor.copy_expert.side_effect = copy_side_effect
|
||||
|
||||
tenant_id = str(uuid.uuid4())
|
||||
scan_id = uuid.uuid4()
|
||||
inserted_at = datetime.now(timezone.utc)
|
||||
|
||||
rows = [
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": tenant_id,
|
||||
"inserted_at": inserted_at,
|
||||
"compliance_id": "cisa_aws",
|
||||
"framework": "CISA",
|
||||
"version": "1.0",
|
||||
"description": "First requirement",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 5,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 5,
|
||||
"scan_id": scan_id,
|
||||
},
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": tenant_id,
|
||||
"inserted_at": inserted_at,
|
||||
"compliance_id": "cisa_aws",
|
||||
"framework": "CISA",
|
||||
"version": "1.0",
|
||||
"description": "Second requirement",
|
||||
"region": "us-west-2",
|
||||
"requirement_id": "req-2",
|
||||
"requirement_status": "FAIL",
|
||||
"passed_checks": 3,
|
||||
"failed_checks": 2,
|
||||
"total_checks": 5,
|
||||
"scan_id": scan_id,
|
||||
},
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": tenant_id,
|
||||
"inserted_at": inserted_at,
|
||||
"compliance_id": "aws_foundational_security_aws",
|
||||
"framework": "AWS-Foundational-Security-Best-Practices",
|
||||
"version": "2.0",
|
||||
"description": "Third requirement",
|
||||
"region": "eu-west-1",
|
||||
"requirement_id": "req-3",
|
||||
"requirement_status": "MANUAL",
|
||||
"passed_checks": 0,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 3,
|
||||
"scan_id": scan_id,
|
||||
},
|
||||
]
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(tenant_id, rows)
|
||||
|
||||
mock_psycopg_connection.assert_called_once_with("admin")
|
||||
connection.cursor.assert_called_once()
|
||||
cursor.execute.assert_called_once()
|
||||
cursor.copy_expert.assert_called_once()
|
||||
|
||||
csv_rows = list(csv.reader(StringIO(captured["data"])))
|
||||
assert len(csv_rows) == 3
|
||||
|
||||
# Validate first row
|
||||
assert csv_rows[0][0] == str(rows[0]["id"])
|
||||
assert csv_rows[0][1] == tenant_id
|
||||
assert csv_rows[0][3] == "cisa_aws"
|
||||
assert csv_rows[0][4] == "CISA"
|
||||
assert csv_rows[0][6] == "First requirement"
|
||||
assert csv_rows[0][7] == "us-east-1"
|
||||
assert csv_rows[0][10] == "5"
|
||||
assert csv_rows[0][11] == "0"
|
||||
assert csv_rows[0][12] == "5"
|
||||
|
||||
# Validate second row
|
||||
assert csv_rows[1][0] == str(rows[1]["id"])
|
||||
assert csv_rows[1][7] == "us-west-2"
|
||||
assert csv_rows[1][9] == "FAIL"
|
||||
assert csv_rows[1][10] == "3"
|
||||
assert csv_rows[1][11] == "2"
|
||||
|
||||
# Validate third row
|
||||
assert csv_rows[2][0] == str(rows[2]["id"])
|
||||
assert csv_rows[2][3] == "aws_foundational_security_aws"
|
||||
assert csv_rows[2][5] == "2.0"
|
||||
assert csv_rows[2][9] == "MANUAL"
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_null_values(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
"""Test COPY handles NULL/None values correctly in nullable fields."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
def copy_side_effect(sql, file_obj):
|
||||
captured["sql"] = sql
|
||||
captured["data"] = file_obj.read()
|
||||
|
||||
cursor.copy_expert.side_effect = copy_side_effect
|
||||
|
||||
# Row with all nullable fields set to None/empty
|
||||
row = {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": str(uuid.uuid4()),
|
||||
"compliance_id": "test_framework",
|
||||
"framework": "Test",
|
||||
"version": None, # nullable
|
||||
"description": None, # nullable
|
||||
"region": "",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 0,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 0,
|
||||
"scan_id": uuid.uuid4(),
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
|
||||
csv_rows = list(csv.reader(StringIO(captured["data"])))
|
||||
assert len(csv_rows) == 1
|
||||
|
||||
# Validate that None values are converted to empty strings in CSV
|
||||
assert csv_rows[0][5] == "" # version
|
||||
assert csv_rows[0][6] == "" # description
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_special_characters(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
"""Test COPY correctly escapes special characters in CSV."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
def copy_side_effect(sql, file_obj):
|
||||
captured["sql"] = sql
|
||||
captured["data"] = file_obj.read()
|
||||
|
||||
cursor.copy_expert.side_effect = copy_side_effect
|
||||
|
||||
# Row with special characters that need escaping
|
||||
row = {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": str(uuid.uuid4()),
|
||||
"compliance_id": 'framework"with"quotes',
|
||||
"framework": "Framework,with,commas",
|
||||
"version": "1.0",
|
||||
"description": 'Description with "quotes", commas, and\nnewlines',
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 1,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 1,
|
||||
"scan_id": uuid.uuid4(),
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
|
||||
# Verify CSV was generated (csv module handles escaping automatically)
|
||||
csv_rows = list(csv.reader(StringIO(captured["data"])))
|
||||
assert len(csv_rows) == 1
|
||||
|
||||
# Verify special characters are preserved after CSV parsing
|
||||
assert csv_rows[0][3] == 'framework"with"quotes'
|
||||
assert csv_rows[0][4] == "Framework,with,commas"
|
||||
assert "quotes" in csv_rows[0][6]
|
||||
assert "commas" in csv_rows[0][6]
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_missing_inserted_at(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
"""Test COPY uses current datetime when inserted_at is missing."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
captured = {}
|
||||
|
||||
def copy_side_effect(sql, file_obj):
|
||||
captured["sql"] = sql
|
||||
captured["data"] = file_obj.read()
|
||||
|
||||
cursor.copy_expert.side_effect = copy_side_effect
|
||||
|
||||
# Row without inserted_at field
|
||||
row = {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": str(uuid.uuid4()),
|
||||
"compliance_id": "test_framework",
|
||||
"framework": "Test",
|
||||
"version": "1.0",
|
||||
"description": "desc",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 1,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 1,
|
||||
"scan_id": uuid.uuid4(),
|
||||
# Note: inserted_at is intentionally missing
|
||||
}
|
||||
|
||||
before_call = datetime.now(timezone.utc)
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
after_call = datetime.now(timezone.utc)
|
||||
|
||||
csv_rows = list(csv.reader(StringIO(captured["data"])))
|
||||
assert len(csv_rows) == 1
|
||||
|
||||
# Verify inserted_at was auto-generated and is a valid ISO datetime
|
||||
inserted_at_str = csv_rows[0][2]
|
||||
inserted_at = datetime.fromisoformat(inserted_at_str)
|
||||
assert before_call <= inserted_at <= after_call
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_transaction_rollback_on_copy_error(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
"""Test transaction is rolled back when copy_expert fails."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
# Simulate copy_expert failure
|
||||
cursor.copy_expert.side_effect = Exception("COPY command failed")
|
||||
|
||||
row = {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": str(uuid.uuid4()),
|
||||
"compliance_id": "test",
|
||||
"framework": "Test",
|
||||
"version": "1.0",
|
||||
"description": "desc",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 1,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 1,
|
||||
"scan_id": uuid.uuid4(),
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
with pytest.raises(Exception, match="COPY command failed"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
|
||||
# Verify rollback was called
|
||||
connection.rollback.assert_called_once()
|
||||
connection.commit.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_transaction_rollback_on_set_config_error(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
"""Test transaction is rolled back when SET_CONFIG fails."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
# Simulate cursor.execute failure
|
||||
cursor.execute.side_effect = Exception("SET prowler.tenant_id failed")
|
||||
|
||||
row = {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": str(uuid.uuid4()),
|
||||
"compliance_id": "test",
|
||||
"framework": "Test",
|
||||
"version": "1.0",
|
||||
"description": "desc",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 1,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 1,
|
||||
"scan_id": uuid.uuid4(),
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
with pytest.raises(Exception, match="SET prowler.tenant_id failed"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
|
||||
# Verify rollback was called
|
||||
connection.rollback.assert_called_once()
|
||||
connection.commit.assert_not_called()
|
||||
|
||||
@patch("tasks.jobs.scan.psycopg_connection")
|
||||
def test_copy_compliance_requirement_rows_commit_on_success(
|
||||
self, mock_psycopg_connection, settings
|
||||
):
|
||||
"""Test transaction is committed on successful COPY."""
|
||||
settings.DATABASES.setdefault("admin", settings.DATABASES["default"])
|
||||
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor_context = MagicMock()
|
||||
cursor_context.__enter__.return_value = cursor
|
||||
cursor_context.__exit__.return_value = False
|
||||
connection.cursor.return_value = cursor_context
|
||||
connection.__enter__.return_value = connection
|
||||
connection.__exit__.return_value = False
|
||||
|
||||
context_manager = MagicMock()
|
||||
context_manager.__enter__.return_value = connection
|
||||
context_manager.__exit__.return_value = False
|
||||
mock_psycopg_connection.return_value = context_manager
|
||||
|
||||
cursor.copy_expert.return_value = None # Success
|
||||
|
||||
row = {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": str(uuid.uuid4()),
|
||||
"compliance_id": "test",
|
||||
"framework": "Test",
|
||||
"version": "1.0",
|
||||
"description": "desc",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 1,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 1,
|
||||
"scan_id": uuid.uuid4(),
|
||||
}
|
||||
|
||||
with patch.object(MainRouter, "admin_db", "admin"):
|
||||
_copy_compliance_requirement_rows(str(row["tenant_id"]), [row])
|
||||
|
||||
# Verify commit was called and rollback was not
|
||||
connection.commit.assert_called_once()
|
||||
connection.rollback.assert_not_called()
|
||||
# Verify autocommit was disabled
|
||||
assert connection.autocommit is False
|
||||
|
||||
@patch("tasks.jobs.scan._copy_compliance_requirement_rows")
|
||||
def test_persist_compliance_requirement_rows_success(self, mock_copy):
|
||||
"""Test successful COPY path without fallback to ORM."""
|
||||
mock_copy.return_value = None # Success, no exception
|
||||
|
||||
tenant_id = str(uuid.uuid4())
|
||||
rows = [
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": tenant_id,
|
||||
"inserted_at": datetime.now(timezone.utc),
|
||||
"compliance_id": "test",
|
||||
"framework": "Test",
|
||||
"version": "1.0",
|
||||
"description": "desc",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 1,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 1,
|
||||
"scan_id": uuid.uuid4(),
|
||||
}
|
||||
]
|
||||
|
||||
_persist_compliance_requirement_rows(tenant_id, rows)
|
||||
|
||||
# Verify COPY was called
|
||||
mock_copy.assert_called_once_with(tenant_id, rows)
|
||||
|
||||
@patch("tasks.jobs.scan.logger")
|
||||
@patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
@patch(
|
||||
"tasks.jobs.scan._copy_compliance_requirement_rows",
|
||||
side_effect=Exception("COPY failed"),
|
||||
)
|
||||
def test_persist_compliance_requirement_rows_fallback_logging(
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_logger
|
||||
):
|
||||
"""Test logger.exception is called when COPY fails and fallback occurs."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
row = {
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": tenant_id,
|
||||
"inserted_at": datetime.now(timezone.utc),
|
||||
"compliance_id": "test",
|
||||
"framework": "Test",
|
||||
"version": "1.0",
|
||||
"description": "desc",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 1,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 1,
|
||||
"scan_id": uuid.uuid4(),
|
||||
}
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
|
||||
_persist_compliance_requirement_rows(tenant_id, [row])
|
||||
|
||||
# Verify logger.exception was called
|
||||
mock_logger.exception.assert_called_once()
|
||||
args, kwargs = mock_logger.exception.call_args
|
||||
assert "COPY bulk insert" in args[0]
|
||||
assert "falling back to ORM" in args[0]
|
||||
assert kwargs.get("exc_info") is not None
|
||||
|
||||
@patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
@patch(
|
||||
"tasks.jobs.scan._copy_compliance_requirement_rows",
|
||||
side_effect=Exception("copy failed"),
|
||||
)
|
||||
def test_persist_compliance_requirement_rows_fallback_multiple_rows(
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create
|
||||
):
|
||||
"""Test ORM fallback with multiple rows."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
scan_id = uuid.uuid4()
|
||||
inserted_at = datetime.now(timezone.utc)
|
||||
|
||||
rows = [
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": tenant_id,
|
||||
"inserted_at": inserted_at,
|
||||
"compliance_id": "cisa_aws",
|
||||
"framework": "CISA",
|
||||
"version": "1.0",
|
||||
"description": "First requirement",
|
||||
"region": "us-east-1",
|
||||
"requirement_id": "req-1",
|
||||
"requirement_status": "PASS",
|
||||
"passed_checks": 5,
|
||||
"failed_checks": 0,
|
||||
"total_checks": 5,
|
||||
"scan_id": scan_id,
|
||||
},
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
"tenant_id": tenant_id,
|
||||
"inserted_at": inserted_at,
|
||||
"compliance_id": "cisa_aws",
|
||||
"framework": "CISA",
|
||||
"version": "1.0",
|
||||
"description": "Second requirement",
|
||||
"region": "us-west-2",
|
||||
"requirement_id": "req-2",
|
||||
"requirement_status": "FAIL",
|
||||
"passed_checks": 2,
|
||||
"failed_checks": 3,
|
||||
"total_checks": 5,
|
||||
"scan_id": scan_id,
|
||||
},
|
||||
]
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
|
||||
_persist_compliance_requirement_rows(tenant_id, rows)
|
||||
|
||||
mock_copy.assert_called_once_with(tenant_id, rows)
|
||||
mock_rls_transaction.assert_called_once_with(tenant_id)
|
||||
mock_bulk_create.assert_called_once()
|
||||
|
||||
args, kwargs = mock_bulk_create.call_args
|
||||
objects = args[0]
|
||||
assert len(objects) == 2
|
||||
assert kwargs["batch_size"] == 500
|
||||
|
||||
# Validate first object
|
||||
assert objects[0].id == rows[0]["id"]
|
||||
assert objects[0].tenant_id == rows[0]["tenant_id"]
|
||||
assert objects[0].compliance_id == rows[0]["compliance_id"]
|
||||
assert objects[0].framework == rows[0]["framework"]
|
||||
assert objects[0].region == rows[0]["region"]
|
||||
assert objects[0].passed_checks == 5
|
||||
assert objects[0].failed_checks == 0
|
||||
|
||||
# Validate second object
|
||||
assert objects[1].id == rows[1]["id"]
|
||||
assert objects[1].requirement_id == rows[1]["requirement_id"]
|
||||
assert objects[1].requirement_status == rows[1]["requirement_status"]
|
||||
assert objects[1].passed_checks == 2
|
||||
assert objects[1].failed_checks == 3
|
||||
|
||||
@patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
@patch(
|
||||
"tasks.jobs.scan._copy_compliance_requirement_rows",
|
||||
side_effect=Exception("copy failed"),
|
||||
)
|
||||
def test_persist_compliance_requirement_rows_fallback_all_fields(
|
||||
self, mock_copy, mock_rls_transaction, mock_bulk_create
|
||||
):
|
||||
"""Test ORM fallback correctly maps all fields from row dict to model."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
row_id = uuid.uuid4()
|
||||
scan_id = uuid.uuid4()
|
||||
inserted_at = datetime.now(timezone.utc)
|
||||
|
||||
row = {
|
||||
"id": row_id,
|
||||
"tenant_id": tenant_id,
|
||||
"inserted_at": inserted_at,
|
||||
"compliance_id": "aws_foundational_security_aws",
|
||||
"framework": "AWS-Foundational-Security-Best-Practices",
|
||||
"version": "2.0",
|
||||
"description": "Ensure MFA is enabled",
|
||||
"region": "eu-west-1",
|
||||
"requirement_id": "iam.1",
|
||||
"requirement_status": "FAIL",
|
||||
"passed_checks": 10,
|
||||
"failed_checks": 5,
|
||||
"total_checks": 15,
|
||||
"scan_id": scan_id,
|
||||
}
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
|
||||
_persist_compliance_requirement_rows(tenant_id, [row])
|
||||
|
||||
args, kwargs = mock_bulk_create.call_args
|
||||
objects = args[0]
|
||||
assert len(objects) == 1
|
||||
|
||||
obj = objects[0]
|
||||
# Validate ALL fields are correctly mapped
|
||||
assert obj.id == row_id
|
||||
assert obj.tenant_id == tenant_id
|
||||
assert obj.inserted_at == inserted_at
|
||||
assert obj.compliance_id == "aws_foundational_security_aws"
|
||||
assert obj.framework == "AWS-Foundational-Security-Best-Practices"
|
||||
assert obj.version == "2.0"
|
||||
assert obj.description == "Ensure MFA is enabled"
|
||||
assert obj.region == "eu-west-1"
|
||||
assert obj.requirement_id == "iam.1"
|
||||
assert obj.requirement_status == "FAIL"
|
||||
assert obj.passed_checks == 10
|
||||
assert obj.failed_checks == 5
|
||||
assert obj.total_checks == 15
|
||||
assert obj.scan_id == scan_id
|
||||
|
||||
+533
@@ -0,0 +1,533 @@
|
||||
Always that you are writting documentation try to follow this text/communication style guide:
|
||||
|
||||
# Prowler's Brand Voice
|
||||
|
||||
Prowler is the open cloud security platform trusted by thousands of organizations automating security monitoring and compliance with hundreds of built-in security checks, remediation solutions, and compliance frameworks. With over 10 million downloads, thousands of contributors, and a vibrant global community, Prowler is driving the open-source cloud security movement by providing transparent, customizable, and user-friendly solutions that help teams secure AWS, Azure, GCP, Kubernetes, and Microsoft 365 environments. Leveraging open-source innovation and cost savings, the Prowler platform makes cloud security 10 times more cost-effective and accessible than alternatives.
|
||||
|
||||
These values must be demonstrated in all our conversations and communications.
|
||||
|
||||
---
|
||||
|
||||
## Unbiased Communication
|
||||
|
||||
Prowler aims to reach every person in the globe. Our communications must be as inclusive and diverse as possible every time. We are guided by the following principles:
|
||||
|
||||
### Avoid Gendered Pronouns
|
||||
|
||||
Reference to gendered pronouns (she/her/hers, he/his/his, they/them/theirs) must be avoided whenever possible.
|
||||
|
||||
* Use second person for communications (you/your/yours).
|
||||
* Use a third-person reference instead of a gendered pronoun (the customer, the user).
|
||||
* In case a gendered pronoun must be forcibly used, use they/them/theirs.
|
||||
* Avoid double references like she/he, s/he, etc.
|
||||
|
||||
### Use Alternatives for Gendered Nouns
|
||||
|
||||
Avoid nouns that include gendered components. Examples:
|
||||
|
||||
* Businessman 🡪 Entrepreneur, businessperson, executive
|
||||
* Salesman 🡪 Sales executive, sales representative
|
||||
* Mankind 🡪 Humanity, people
|
||||
* Penmanship 🡪 Calligraphy, handwriting
|
||||
* Middleman 🡪 Intermediary, negotiator
|
||||
|
||||
### Diversity, Equity, and Inclusion
|
||||
|
||||
All communications must prioritize diversity and inclusivity. When incorporating examples, ensure representation across sex, gender, age, identity, race, culture, background, ability, and socioeconomic status. Strive for balanced and respectful depictions.
|
||||
|
||||
### Cultural and Geographical Awareness
|
||||
|
||||
Before referencing a region, country, culture, national status, political status, or socioeconomic realities, conduct thorough research. Maintain a respectful, informed approach and avoid unnecessary conflicts.
|
||||
|
||||
### Avoiding Generalizations
|
||||
|
||||
Avoid broad assumptions about gender, sex, race, sexual orientation, nationality, or culture. Generalizations can introduce bias and misrepresentation. Example to avoid: "Cybersecurity is of the utmost importance in the country, where corruption runs amok."
|
||||
|
||||
### Respectful Language
|
||||
|
||||
Derogatory terms must not be used. If uncertain about terminology, consult individuals from the relevant region or community to ensure accuracy and appropriateness.
|
||||
|
||||
### Clear and Accessible Language
|
||||
|
||||
* **Jargon:** Use technical terminology only when the audience is expected to understand it. If uncertain, opt for clear and universally accessible language.
|
||||
* **Slang:** Minimize slang usage. Even when confident about the audience, prefer formal and neutral language to enhance clarity.
|
||||
|
||||
### Militaristic Language
|
||||
|
||||
Current tendencies in a topic as sensitive as cybersecurity avoid violent and militaristic references save for explicit reference to combat. These are some alternatives:
|
||||
|
||||
* Combat, fight, eliminate 🡪 Address, protect, safeguard, ward
|
||||
* Kill chain 🡪 Cyberattack chain
|
||||
* Attacker 🡪 Cyberattacker, bad actor, threat actor
|
||||
* Defense-in-depth approach 🡪 Multilayered approach
|
||||
* First line of defense, frontline 🡪 Security, protection, defense
|
||||
* External attack surface 🡪 Vulnerabilities, point of access, external exposure
|
||||
|
||||
### Note on Safety and Security
|
||||
|
||||
“Safety” and “security” are terms often misunderstood. “Safety” is the microscopic, personal and individual term, while “security” is the macroscopic, broader, national term. Examples:
|
||||
|
||||
a. Seat belts are great for personal safety.
|
||||
b. National security is of the utmost concern nowadays.
|
||||
|
||||
---
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
### Prowler Features
|
||||
|
||||
Prowler Features are considered proper nouns. They are to be referenced without articles in all pieces of writing.
|
||||
|
||||
This is a list of Prowler Features:
|
||||
|
||||
* **Prowler App**
|
||||
* **Prowler CLI**
|
||||
* **Prowler SDK**
|
||||
* **Built-in Compliance Checks**
|
||||
* **Multi-cloud Security Scanning**
|
||||
* **Autonomous Cloud Security Analyst (AI)**
|
||||
* **Threat & Misconfiguration Detection**
|
||||
* **Role-Based Access Control (RBAC)**
|
||||
* **Identity & Access Risk Detection**
|
||||
* **Tag-Based Scanning & Filtering**
|
||||
* **Audit Logs & Security Reports**
|
||||
* **Agentless & Works Anywhere**
|
||||
* **Automated Scans & Continuous Monitoring**
|
||||
* **Chat-based Security Querying (AI)**
|
||||
* **AI-Generated Detections & Remediations**
|
||||
* **Prowler Studio**
|
||||
* **Custom Security Policies**
|
||||
* **Prowler Cloud**
|
||||
* **Prowler Registry**
|
||||
* **Open Source & Full APIs**
|
||||
|
||||
---
|
||||
|
||||
## Verbal Constructions in Technical Writing
|
||||
|
||||
Choosing verbal constructions (using verbs) over nominal (using nouns) constructions can significantly impact the clarity, conciseness and especially readability of the content.
|
||||
|
||||
Nominal constructions often introduce unnecessary complexity or vagueness. For example:
|
||||
|
||||
* Nominal: "The creation of the report was successful."
|
||||
* Verbal: "The report was successfully created."
|
||||
|
||||
Verbal constructions also tend to use fewer words, resulting in a more polished and concise style:
|
||||
|
||||
* Nominal: "The implementation of the solution reduced system downtime."
|
||||
* Verbal: "The solution reduced system downtime."
|
||||
|
||||
Verbal constructions are to be chosen over nominal constructions whenever possible.
|
||||
|
||||
---
|
||||
|
||||
### Addendum: Verbal Structures Actually State your Purpose
|
||||
|
||||
* **Example 1:** Recommendation for multiple subscriptions
|
||||
* **Example 2:** Recommendation for Managing Multiple Subscriptions
|
||||
|
||||
Example 1 is vague and even potentially ambiguous. Verbs state your purpose and they must be used whenever possible.
|
||||
|
||||
---
|
||||
|
||||
## Avoiding The Second Person Except for Imperative Instructions
|
||||
|
||||
Explicit use of second-person pronouns (you) and possessives (your) should be minimized whenever possible. Those constructions are best reserved for cases when instructions are directly given in an imperative form:
|
||||
|
||||
**Example of Improvement Through Avoiding Second Person Pronouns**
|
||||
|
||||
**Original:**
|
||||
Prowler App can be installed in different ways, depending on your environment:
|
||||
|
||||
**Improved Version:**
|
||||
Prowler App offers flexible installation methods tailored to various environments:
|
||||
|
||||
---
|
||||
|
||||
## Title-Case Capitalization
|
||||
|
||||
We use title case.
|
||||
|
||||
**Example:** This Is an Example on Title Case
|
||||
|
||||
Title case tends to be better for SEO because it improves readability and makes headlines more visually distinct, which can lead to higher click-through rates (CTR).
|
||||
|
||||
---
|
||||
|
||||
### Other Considerations on Capitalization
|
||||
|
||||
Follow these additional guidelines for capitalization:
|
||||
|
||||
### Inner Capitalization
|
||||
|
||||
Avoid internal capitalization of words in body text unless it is part of a proper name or brand denomination. Example: instead of E-mail and e-Book use email/e-mail and e-book.
|
||||
|
||||
### Acronym Capitalization
|
||||
|
||||
Do not capitalize the individual words of the spelled-out form of acronyms. Example: instead of CTI (Cyber Threat Intelligence) use CTI (cyber threat intelligence), but AWS (Amazon Web Services) is to be kept as is.
|
||||
|
||||
### Avoid Capitalization for Emphasis
|
||||
|
||||
Do not capitalize words in order to emphasize them.
|
||||
|
||||
### Capitalization of Languages and Standards
|
||||
|
||||
Check for the proper capitalization of language and standard names.
|
||||
|
||||
Language examples: HTML, JSON, YAML, XML, etc., must be capitalized.
|
||||
|
||||
Standard examples: standards follow title-case capitalization: Industrial Automation and Control Systems (IACS).
|
||||
|
||||
### Capitalization of Laws and Regulations
|
||||
|
||||
Laws and Regulations follow title-case capitalization. If referring to a non-domestic law or regulation, add the nationality and the original name.
|
||||
|
||||
**Example:** Code for the Cybersecurity Law published in the Spanish Official State Bulletin (BOE, Boletín Oficial del Estado).
|
||||
|
||||
Most UE Regulations have an official translation for all UE languages; please check it on EUR-Lex portal and choose the proper language: https://eur-lex.europa.eu/.
|
||||
|
||||
The different languages can be chosen on the portal under Languages, formats and link to OJ.
|
||||
|
||||
---
|
||||
|
||||
## Hyphenation
|
||||
|
||||
Hyphenation is to be used for noun modifiers in prenominal position, i.e., placed before nouns.
|
||||
|
||||
**Example:** Prowler is a world-leading company in open-source software.
|
||||
|
||||
It is not to be used for predicate adjectives in postnominal position.
|
||||
|
||||
**Example:** Prowler has many features built in.
|
||||
|
||||
### Note on Hyphenation and SEO
|
||||
|
||||
Google treats hyphens as word separators, as if they were blank spaces, i.e., the term `high quality checks` is treated as if it was the same term as `high-quality checks`. Hyphenation does not affect SEO on body text, thus the grammatically correct approach is recommended as sign of good writing. However, underscores (`_`) are treated as different words. This has implications particularly for URLs.
|
||||
|
||||
Hyphens are preferred for URLs as they improve readability and indexing.
|
||||
|
||||
**Example:**
|
||||
* Better approach: `example.com/this-is-an-URL`
|
||||
* Less ideal approach: `example.com/this_is_an_URL`
|
||||
|
||||
---
|
||||
|
||||
## Bullet Points
|
||||
|
||||
Bullet points offer several advantages:
|
||||
|
||||
* **Improved readability:** Bullet points make information scannable and enable vertical reading, allowing users to easily spot relevant details and breaking content into more digestible pieces.
|
||||
* **Highlighting of relevant information:** They emphasize the most salient points, improving focus and enabling quick summarization at a glance.
|
||||
* **Improved retention:** Bullet points enhance memory retention and contribute to a clearer, more polished final product.
|
||||
* **Structured presentation:** They improve user experience through the logical organization of content.
|
||||
* **SEO relevance (Search Engine Optimization):** Bullet points make content easier to consume and offer the following SEO benefits:
|
||||
* Reduced bounce rates
|
||||
* Increased time spent on page
|
||||
* Strategic keyword optimization
|
||||
* Improved chances of being featured in search engine snippets
|
||||
* Enhanced crawlability for search engines.
|
||||
|
||||
---
|
||||
|
||||
### When to Use Bullet Points
|
||||
|
||||
The use of bullet points is highly recommended when:
|
||||
|
||||
* Information can be logically divided into multiple categories, each sharing characteristics, features, or other relevant classifications.
|
||||
* Items are significant enough as standalone concepts to deserve their own bullet point.
|
||||
|
||||
**Example of Improvement Through Bullet Points**
|
||||
|
||||
**Original:**
|
||||
It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMS, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme), and your custom security frameworks.
|
||||
|
||||
**Improved with Bullet Points:**
|
||||
|
||||
**Prowler CLI Features:**
|
||||
Prowler CLI includes hundreds of built-in controls to ensure compliance with standards and frameworks, including:
|
||||
|
||||
* **Industry standards:** CIS, NIST 800, NIST CSF, and CISA
|
||||
* **Regulatory compliance and governance:** RBI, FedRAMP, and PCI-DSS
|
||||
* **Frameworks for sensitive data and privacy:** GDPR, HIPAA, and FFIEC
|
||||
* **Frameworks for organizational governance and quality control:** SOC2 and GXP
|
||||
* **AWS-specific guidance:** AWS Foundational Technical Review (FTR) and AWS Well-Architected Framework (Security Pillar)
|
||||
* **Regional compliance:** ENS (Spanish National Security Scheme)
|
||||
* **Custom security frameworks:** Tailored to meet your organization’s specific needs
|
||||
|
||||
---
|
||||
|
||||
### Punctuation of Bullet Points
|
||||
|
||||
There are several options for punctuating bullet points. Regardless of the style chosen, it is imperative to maintain consistency throughout the text.
|
||||
|
||||
* **No punctuation (minimalistic):** This strategy is suitable when no verbs are involved and is best used to highlight products or features in isolation. For example:
|
||||
|
||||
Prowler App is composed of three key components:
|
||||
* Prowler UI
|
||||
* Prowler API
|
||||
* Prowler SDK
|
||||
|
||||
This example highlights each element individually and fosters retention with a noise-free approach.
|
||||
|
||||
* **Periods for full sentences:** This approach works best when each bullet point forms a full sentence or includes verbs. For example:
|
||||
|
||||
Prowler App is composed of three key components:
|
||||
* Prowler UI, a web-based interface, built with Next.js, providing a user-friendly experience for executing Prowler scans and visualizing results.
|
||||
* Prowler API, a backend service, developed with Django REST Framework, responsible for running Prowler scans and storing the generated results.
|
||||
* Prowler SDK, a Python SDK designed to extend the functionality of the Prowler CLI for advanced capabilities.
|
||||
|
||||
This example demonstrates a polished list using proper punctuation.
|
||||
|
||||
* **Semi-colons with final period:** This approach was traditionally used for those cases when bullet points were part of a continuous sentence or logical succession. However, it is being deprecated, consistent with the declining use of semi-colons in modern writing. It is to be avoided whenever possible. For example:
|
||||
|
||||
Prowler UI:
|
||||
* is a web-based interface;
|
||||
* is built with Next.js;
|
||||
* provides a user-friendly experience for executing Prowler scans and visualizing results.
|
||||
|
||||
---
|
||||
|
||||
### Advantages of Adding Headers to Bullet Points
|
||||
|
||||
Adding headers to bullet points in technical writing is a powerful technique that enhances both the clarity and usability of the content. It has also advantages for SEO:
|
||||
|
||||
* Increased crawlability of search engines
|
||||
* Enhanced keyword integration
|
||||
* Improved user engagement
|
||||
* Enhanced snippetting by search engines
|
||||
* Reduced bounce rates
|
||||
|
||||
It is recommended to add headers to bullet points whenever possible.
|
||||
|
||||
---
|
||||
|
||||
## Quotation Marks
|
||||
|
||||
### Quotation Marks Usage in Technical Documentation
|
||||
|
||||
Proper use of quotation marks enhances clarity and consistency in technical writing. Below are key guidelines for using double and single quotation marks, following American English conventions.
|
||||
|
||||
### Double Quotation Marks
|
||||
|
||||
* Use for titles of books, movies, songs, and articles.
|
||||
* Enclose direct quotes:
|
||||
* **Example:** The developer said, “We will try to fix this issue.”
|
||||
* Capitalize the first word if quoting a full sentence.
|
||||
* When quoting a phrase within a sentence, do not capitalize:
|
||||
* **Example:** The news portal called our product “one of the most efficient online help authoring tools.”
|
||||
* Use scare quotes when words acquire a different or ironic meaning:
|
||||
* **Example:** The update is “scheduled” to release next week.
|
||||
* To refer to a term without applying its meaning, use double quotes (or italics):
|
||||
* **Example:** Avoid terms like “don’t worry” in pop-ups to prevent user anxiety.
|
||||
|
||||
### Single Quotation Marks
|
||||
|
||||
* Used inside double quotes:
|
||||
* **Example:** He said, “I am not sure what ‘single sourcing’ means.”
|
||||
* In British English, the order is often reversed (single quotation marks on the outside).
|
||||
|
||||
---
|
||||
|
||||
### Double Quoting in Software Documentation
|
||||
|
||||
Double quoting is to be used through software documentation where their use does not interfere with formatting restrictions.
|
||||
|
||||
1. **Menu Items & UI Options**
|
||||
* Use double quotation marks when referring to selectable items in software interfaces.
|
||||
* **Example:** Click “File” and select “Save As” to export your document.
|
||||
2. **Buttons & Commands**
|
||||
* Use double quotes for labeled interface elements that users interact with.
|
||||
* **Example:** Select “Submit” to finalize the form.
|
||||
3. **Exact Input & User Actions**
|
||||
* If users need to enter exact text, enclose it in double quotes.
|
||||
* **Example:** Type “admin” in the username field.
|
||||
4. **Avoid Quoting Software Names**
|
||||
* Do not use quotation marks for software product names unless required for clarity.
|
||||
* **Correct:** Open Microsoft Excel.
|
||||
* **Incorrect:** Open “Microsoft Excel.”
|
||||
|
||||
---
|
||||
|
||||
## Interaction Verbs
|
||||
|
||||
The following are the correct verbs that must be used when referring to user interactions with the software.
|
||||
|
||||
### Mouse & Trackpad Actions (Desktop/Laptop)
|
||||
|
||||
* **Click:** Press and release the left mouse button or trackpad without moving the pointer.
|
||||
* **Example:** Click the “OK” button to confirm. 🡪 Transitive
|
||||
* **Click on:** Often interchangeable with "Click," but less commonly used in technical writing for UI interactions.
|
||||
* **Example:** Click on the "Settings" icon to open preferences. (Less recommended—“Click” is preferred.)
|
||||
* **Double-click:** Press and release twice in quick succession, usually to open files or applications.
|
||||
* **Example:** Double-click the document to open it. 🡪 Transitive
|
||||
* **Right-click:** Press and release the right mouse button to open a context menu.
|
||||
* **Example:** Right-click the folder and select “Properties.” 🡪 Transitive
|
||||
|
||||
### Touchscreen Actions (Mobile & Touch)
|
||||
|
||||
* **Tap:** Touch the screen lightly with a finger or stylus, equivalent to "Click" on a mouse.
|
||||
* **Example:** Tap the “Sign in” button.
|
||||
* **Double-tap:** Quickly touch the screen twice, often used for zooming or selecting text.
|
||||
* **Example:** Double-tap an image to zoom in.
|
||||
* **Press and hold:** Touch and hold the screen for a moment to access additional options.
|
||||
* **Example:** Press and hold an app icon to see more actions. (Similar to “Right-click” in desktop environments.)
|
||||
|
||||
### Additional Actions
|
||||
|
||||
* **Drag:** Click or tap an item and move it while holding down the button or finger.
|
||||
* **Example:** Drag the file into the folder.
|
||||
* **Swipe:** Move a finger across the touchscreen horizontally or vertically.
|
||||
* **Example:** Swipe left to dismiss the notification.
|
||||
* **Pinch to zoom:** Use two fingers to zoom in or out.
|
||||
* **Example:** Pinch the screen to zoom in on the image.
|
||||
* **Scroll:** Move the mouse wheel, swipe, or use the arrow keys to navigate up/down.
|
||||
* **Example:** Scroll down to see more results.
|
||||
|
||||
The widely-accepted terminology for gestures is Windows’: https://support.microsoft.com/en-us/windows/touch-gestures-for-windows-a9d28305-4818-a5df-4e2b-e5590f850741
|
||||
|
||||
---
|
||||
|
||||
## Sentence Structure for Technical Writing and SEO
|
||||
|
||||
When writing technical documentation, clarity, conciseness, and searchability (SEO) are key factors. Let’s compare the following two sentence structures, extracted from Prowler’s documentation:
|
||||
|
||||
**Option 1:**
|
||||
"Open a terminal and execute the following command to create a new custom role."
|
||||
|
||||
**Option 2:**
|
||||
"To create a new custom role, open a terminal and execute the following command."
|
||||
|
||||
### SEO Optimization
|
||||
|
||||
* Search engines prioritize clear intent at the beginning of a sentence.
|
||||
* Option 2 starts with the action users are likely to search for (e.g., "Create a custom role"), which improves SEO rankings and makes the content more likely to match search queries.
|
||||
* Option 1 places the primary search term toward the end, making it less effective for keyword optimization.
|
||||
|
||||
### Technical Writing Best Practices
|
||||
|
||||
* Technical writing emphasizes clear objectives first, followed by actions.
|
||||
* Option 2 follows this best practice by stating the goal first ("To create a new custom role") and then providing instructions.
|
||||
* Option 1 is still acceptable for step-by-step guides, but Option 2 is more effective for tutorials, manuals, and documentation.
|
||||
|
||||
### Key Takeaways
|
||||
|
||||
* Draft trying to mimic the most likely way users are to find the information (“Ctrl + F approach”).
|
||||
* Place keywords and key terms at the beginning of sentences so that they rank better SEO-wise.
|
||||
* Rule of thumb: “In order to what” precedes the “what”. “What” must mirror the user’s most likely way of drafting or searching.
|
||||
|
||||
---
|
||||
|
||||
## Section Titles and Headers in Technical Writing
|
||||
|
||||
Effective headers and section titles enhance document readability and structure, making content more accessible to the reader. This chapter outlines best practices for crafting clear, consistent, and meaningful headings.
|
||||
|
||||
1. **Purpose of Headers**
|
||||
Headers serve several key functions:
|
||||
* **Improve Navigation:** Allow users to quickly locate relevant information.
|
||||
* **Enhance Readability:** Break down complex topics into manageable sections.
|
||||
* **Establish Hierarchy:** Define the logical flow of content.
|
||||
* **SEO:** Headers impact SEO both directly and indirectly:
|
||||
* Search engines use headings to determine the hierarchy and relevance of content.
|
||||
* **H1:** The primary heading (should be unique and descriptive).
|
||||
* **H2-H6:** Subheadings that break down content logically.
|
||||
* Best practices for SEO-friendly headers:
|
||||
* Include keywords naturally in headings.
|
||||
* Avoid keyword stuffing—keep it clear and readable.
|
||||
* Use structured hierarchy (H1 → H2 → H3, etc.).
|
||||
2. **Header Levels and Formatting**
|
||||
Use a structured approach for organizing section titles. Common conventions include:
|
||||
* **Title:** The primary heading of the document (e.g., H1).
|
||||
* **Main Sections:** First-level headers (H2), introducing key content areas.
|
||||
* **Subsections:** Second-level headers (H3) to detail specific topics within sections.
|
||||
* **Subtopics:** Third-level headers (H4+) used sparingly for finer details.
|
||||
|
||||
**Example:**
|
||||
|
||||
```markdown
|
||||
# Document Title (H1)
|
||||
## Main Section (H2)
|
||||
### Subsection (H3)
|
||||
#### Subtopic (H4)
|
||||
```
|
||||
|
||||
3. **Writing Effective Headers**
|
||||
When crafting headers and section titles, follow these guidelines:
|
||||
* **Be Descriptive:** Clearly indicate what the section covers.
|
||||
* **Poor:** Introduction (too vague)
|
||||
* **Good:** Introduction to AWS CloudShell Installation (informative)
|
||||
* **Keep It Concise:** Use precise language without unnecessary words.
|
||||
* **Maintain Consistency:** Apply uniform formatting and style conventions throughout.
|
||||
* **Avoid Special Characters:** Limit punctuation for clarity—avoid excessive symbols, dashes, or underscores.
|
||||
4. **Capitalization Rules**
|
||||
Use Title Case for headers to ensure a professional look:
|
||||
* **Good:** How to Clone and Install Prowler from GitHub
|
||||
* **Poor:** How to clone and install Prowler from GitHub
|
||||
|
||||
For technical documentation, sentence case may be used for readability in subheadings. Please note this differs from headers and it is only a recommendation, but consistency is to be kept throughout the documentation:
|
||||
|
||||
* **Example:**
|
||||
* How to Clone and Install Prowler from GitHub (header: Title case)
|
||||
* How to install poetry dependencies (subheading: Sentence case)
|
||||
5. **Using Keywords in Headers**
|
||||
Headers should include relevant keywords to improve document searchability:
|
||||
* **Good:** Scanning AWS Accounts in Parallel
|
||||
* **Poor:** Ways to scan on AWS (vague and imprecise)
|
||||
6. **Consistency Across Documents**
|
||||
Ensure uniformity in section titles across related documentation:
|
||||
* **Standardized Header Naming:** Use consistent wording for common sections (e.g., "Installation," "Setup," "Configuration").
|
||||
* **Numbering Sections (If Necessary):** For structured guides, include numbering where appropriate (e.g., "Step 1: Install Prowler").
|
||||
|
||||
---
|
||||
|
||||
## Avoid Assumptions Regarding Audience’s Expertise
|
||||
|
||||
### Understand Your Audience’s Expertise
|
||||
|
||||
Despite knowing your target audience, assumptions on target audience’s expertise or knowledge are to be avoided.
|
||||
|
||||
Adjust the level of detail based on expected reader proficiency, but make sure to be as explanatory as humanly possible.
|
||||
|
||||
### Define Key Terms and Acronyms on First Use
|
||||
|
||||
Even if your audience is technical, some domain-specific terms may vary.
|
||||
* Introduce jargon only after defining it clearly.
|
||||
* If using acronyms (e.g., IAM, MFA), spell them out on first mention:
|
||||
* AWS Identity and Access Management (IAM)
|
||||
* Multifactor Authentication (MFA)
|
||||
|
||||
### Don’t Assume Unwritten Knowledge
|
||||
|
||||
Even experienced readers may not know every prerequisite. If a process relies on prior steps, briefly reference them:
|
||||
|
||||
* Before configuring security groups, ensure VPC networking is set up.
|
||||
|
||||
### Use Consistent Formatting
|
||||
|
||||
### Provide as Many Examples as Deemed Right… and Then Some
|
||||
|
||||
### Anticipate Common Knowledge Gaps
|
||||
|
||||
### Avoid Excessive Notes
|
||||
|
||||
Notes are often omitted by readers and they clutter text, so use them sparingly and only for additional information that is not essential or prompts any error or mistake.
|
||||
|
||||
---
|
||||
|
||||
## Using Warnings and Danger Calls for High-Severity Information
|
||||
|
||||
In technical documentation, warnings and danger calls highlight critical risks, guiding users in preventing security breaches or system failures. Proper usage ensures clarity and actionable guidance.
|
||||
|
||||
1. **Define Severity Levels**
|
||||
Before applying Note, Warning, or Danger, clearly define their significance:
|
||||
* **Note:** Provides general information or best practices (low severity).
|
||||
* **Warning:** Indicates potential issues if instructions are not followed (moderate severity).
|
||||
* **Danger:** Highlights actions that could result in severe consequences, such as system corruption or data loss (high severity).
|
||||
2. **Explain Consequences**
|
||||
Each warning or danger call should explicitly describe the impact of ignoring the caution:
|
||||
* **Good:** Disabling encryption may expose sensitive data to unauthorized access.
|
||||
* **Poor:** Avoid disabling encryption.
|
||||
3. **Provide Remediation and Troubleshooting**
|
||||
Whenever possible, direct users to troubleshooting guides or mitigation steps to resolve the issue.
|
||||
|
||||
**Example:**
|
||||
**Danger:** Running this command will **permanently delete all data**. Refer to @Data Recovery Guide for restoration steps.
|
||||
@@ -182,9 +182,6 @@ Microsoft 365 requires specifying the auth method:
|
||||
# To use service principal authentication for MSGraph and PowerShell modules
|
||||
prowler m365 --sp-env-auth
|
||||
|
||||
# To use both service principal (for MSGraph) and user credentials (for PowerShell modules)
|
||||
prowler m365 --env-auth
|
||||
|
||||
# To use az cli authentication
|
||||
prowler m365 --az-cli-auth
|
||||
|
||||
|
||||
@@ -273,7 +273,7 @@ The `CheckTitle` field must be plain text, clearly and succinctly define **the b
|
||||
|
||||
**Always write the `CheckTitle` to describe the *PASS* case**, the desired secure or compliant state of the resource(s). This helps ensure that findings are easy to interpret and that the title always reflects the best practice being met.
|
||||
|
||||
For detailed guidelines on writing effective check titles, including how to determine singular vs. plural scope and common mistakes to avoid, see [CheckTitle Guidelines](./check-metadata-guidelines.md#check-title-guidelines).
|
||||
For detailed guidelines on writing effective check titles, including how to determine singular vs. plural scope and common mistakes to avoid, see [Check Title Guidelines](./check-metadata-guidelines.md#check-title-guidelines).
|
||||
|
||||
#### CheckType
|
||||
|
||||
@@ -282,7 +282,7 @@ For detailed guidelines on writing effective check titles, including how to dete
|
||||
|
||||
It follows the [AWS Security Hub Types](https://docs.aws.amazon.com/securityhub/latest/userguide/asff-required-attributes.html#Types) format using the pattern `namespace/category/classifier`.
|
||||
|
||||
For the complete AWS Security Hub selection guidelines, see [CheckType Guidelines](./check-metadata-guidelines.md#check-type-guidelines-aws-only).
|
||||
For the complete AWS Security Hub selection guidelines, see [Check Type Guidelines](./check-metadata-guidelines.md#check-type-guidelines-aws-only).
|
||||
|
||||
#### ServiceName
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# LLM Provider
|
||||
|
||||
This page details the [Large Language Model (LLM)](https://en.wikipedia.org/wiki/Large_language_model) provider implementation in Prowler.
|
||||
|
||||
The LLM provider enables security testing of language models using red team techniques. By default, Prowler uses the built-in LLM configuration that targets OpenAI models with comprehensive security test suites. To configure it, follow the [LLM getting started guide](../tutorials/llm/getting-started-llm.md).
|
||||
|
||||
## LLM Provider Classes Architecture
|
||||
|
||||
The LLM provider implementation follows the general [Provider structure](./provider.md). This section focuses on the LLM-specific implementation, highlighting how the generic provider concepts are realized for LLM security testing in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](./provider.md).
|
||||
|
||||
### Main Class
|
||||
|
||||
- **Location:** [`prowler/providers/llm/llm_provider.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/llm/llm_provider.py)
|
||||
- **Base Class:** Inherits from `Provider` (see [base class details](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/common/provider.py)).
|
||||
- **Purpose:** Central orchestrator for LLM-specific logic, configuration management, and integration with promptfoo for red team testing.
|
||||
- **Key LLM Responsibilities:**
|
||||
- Initializes and manages LLM configuration using promptfoo.
|
||||
- Validates configuration and sets up the LLM testing context.
|
||||
- Loads and manages red team test configuration, plugins, and target models.
|
||||
- Provides properties and methods for downstream LLM security testing.
|
||||
- Integrates with promptfoo for comprehensive LLM security evaluation.
|
||||
|
||||
### Data Models
|
||||
|
||||
- **Location:** [`prowler/providers/llm/models.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/llm/models.py)
|
||||
- **Purpose:** Define structured data for LLM output options and configuration.
|
||||
- **Key LLM Models:**
|
||||
- `LLMOutputOptions`: Customizes output filename logic for LLM-specific reporting.
|
||||
|
||||
### LLM Security Testing Integration
|
||||
|
||||
- **Location:** [`prowler/providers/llm/llm_provider.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/llm/llm_provider.py)
|
||||
- **Purpose:** Integrates with promptfoo for comprehensive LLM security testing.
|
||||
- **Key LLM Responsibilities:**
|
||||
- Executes promptfoo red team evaluations against target LLMs.
|
||||
- Processes security test results and converts them to Prowler reports.
|
||||
- Manages test concurrency and progress tracking.
|
||||
- Handles real-time streaming of test results.
|
||||
|
||||
### Configuration Management
|
||||
|
||||
The LLM provider uses promptfoo configuration files to define:
|
||||
|
||||
- **Target Models**: The LLM models to test (e.g., OpenAI GPT, Anthropic Claude)
|
||||
- **Red Team Plugins**: Security test suites (OWASP, MITRE, NIST, EU AI Act)
|
||||
- **Test Parameters**: Concurrency, test counts, and evaluation criteria
|
||||
|
||||
### Default Configuration
|
||||
|
||||
Prowler includes a comprehensive default LLM configuration that:
|
||||
|
||||
- Targets OpenAI models by default
|
||||
- Includes multiple security test frameworks (OWASP, MITRE, NIST, EU AI Act)
|
||||
- Provides extensive test coverage for LLM security vulnerabilities
|
||||
- Supports custom configuration for specific testing needs
|
||||
|
||||
## Specific Patterns in LLM Security Testing
|
||||
|
||||
The LLM provider implements security testing through integration with promptfoo, following these patterns:
|
||||
|
||||
### Red Team Testing Framework
|
||||
|
||||
- **Plugin-based Architecture**: Uses promptfoo plugins for different security test categories
|
||||
- **Comprehensive Coverage**: Includes OWASP LLM Top 10, MITRE ATLAS, NIST AI Risk Management, and EU AI Act compliance
|
||||
- **Real-Time Evaluation**: Streams test results as they are generated
|
||||
- **Progress Tracking**: Provides detailed progress information during test execution
|
||||
|
||||
### Test Execution Flow
|
||||
|
||||
1. **Configuration Loading**: Loads promptfoo configuration with target models and test plugins
|
||||
2. **Test Generation**: Generates security test cases based on configured plugins
|
||||
3. **Concurrent Execution**: Runs tests with configurable concurrency limits
|
||||
4. **Result Processing**: Converts promptfoo results to Prowler security reports
|
||||
5. **Progress Monitoring**: Tracks and displays test execution progress
|
||||
|
||||
### Security Test Categories
|
||||
|
||||
The LLM provider supports comprehensive security testing across multiple frameworks:
|
||||
|
||||
- **OWASP LLM Top 10**: Covers prompt injection, data leakage, and model security
|
||||
- **MITRE ATLAS**: Adversarial threat landscape for AI systems
|
||||
- **NIST AI Risk Management**: AI system risk assessment and mitigation
|
||||
- **EU AI Act**: European Union AI regulation compliance
|
||||
- **Custom Tests**: Support for organization-specific security requirements
|
||||
|
||||
## Error Handling and Validation
|
||||
|
||||
The LLM provider includes comprehensive error handling for:
|
||||
|
||||
- **Configuration Validation**: Ensures valid promptfoo configuration files
|
||||
- **Model Access**: Handles authentication and access issues with target LLMs
|
||||
- **Test Execution**: Manages test failures and timeout scenarios
|
||||
- **Result Processing**: Handles malformed or incomplete test results
|
||||
|
||||
## Integration with Prowler Ecosystem
|
||||
|
||||
The LLM provider seamlessly integrates with Prowler's existing infrastructure:
|
||||
|
||||
- **Output Formats**: Supports all Prowler output formats (JSON, CSV, HTML, etc.)
|
||||
- **Compliance Frameworks**: Integrates with Prowler's compliance reporting
|
||||
- **Fixer Integration**: Supports automated remediation recommendations
|
||||
- **Dashboard Integration**: Compatible with Prowler App for centralized management
|
||||
@@ -14,6 +14,7 @@ The official supported providers right now are:
|
||||
| **Github** | Official | Stable | UI, API, CLI |
|
||||
| **IaC** | Official | Beta | CLI |
|
||||
| **MongoDB Atlas** | Official | Beta | CLI |
|
||||
| **LLM** | Official | Beta | CLI |
|
||||
| **NHN** | Unofficial | Beta | CLI |
|
||||
|
||||
Prowler supports **auditing, incident response, continuous monitoring, hardening, forensic readiness, and remediation**.
|
||||
|
||||
@@ -4,6 +4,9 @@ Prowler App supports multiple installation methods based on your environment.
|
||||
|
||||
Refer to the [Prowler App Tutorial](../tutorials/prowler-app.md) for detailed usage instructions.
|
||||
|
||||
???+ warning
|
||||
Prowler configuration is based in `.env` files. Every version of Prowler can have differences on that file, so, please, use the file that corresponds with that version or repository branch or tag.
|
||||
|
||||
=== "Docker Compose"
|
||||
|
||||
_Requirements_:
|
||||
@@ -25,6 +28,9 @@ Refer to the [Prowler App Tutorial](../tutorials/prowler-app.md) for detailed us
|
||||
???+ note
|
||||
You can change the environment variables in the `.env` file. Note that it is not recommended to use the default values in production environments.
|
||||
|
||||
???+ note
|
||||
For a secure setup, leave empty or remove `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY` in `.env` before first start. When absent, the API auto‑generates a unique key pair and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate, delete the stored key files and restart the API.
|
||||
|
||||
???+ note
|
||||
There is a development mode available, you can use the file https://github.com/prowler-cloud/prowler/blob/master/docker-compose-dev.yml to run the app in development mode.
|
||||
|
||||
|
||||
@@ -1,70 +1,229 @@
|
||||
# Azure Authentication in Prowler
|
||||
|
||||
Prowler for Azure supports multiple authentication types. To use a specific method, pass the appropriate flag during execution:
|
||||
Prowler for Azure supports multiple authentication types. Authentication methods vary between Prowler App and Prowler CLI:
|
||||
|
||||
- [**Service Principal Application**](https://learn.microsoft.com/en-us/entra/identity-platform/app-objects-and-service-principals?tabs=browser#service-principal-object) (**Recommended**)
|
||||
- Existing **AZ CLI credentials**
|
||||
- **Interactive browser authentication**
|
||||
- [**Managed Identity**](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) authentication
|
||||
**Prowler App:**
|
||||
|
||||
> ⚠️ **Important:** For Prowler App, only Service Principal authentication is supported.
|
||||
- [**Service Principal Application**](#service-principal-application-authentication-recommended)
|
||||
|
||||
### Service Principal Application Authentication
|
||||
**Prowler CLI:**
|
||||
|
||||
Enable Prowler authentication using a Service Principal Application by setting up the following environment variables:
|
||||
- [**Service Principal Application**](#service-principal-application-authentication-recommended) (**Recommended**)
|
||||
- [**AZ CLI credentials**](#az-cli-authentication)
|
||||
- [**Interactive browser authentication**](#browser-authentication)
|
||||
- [**Managed Identity Authentication**](#managed-identity-authentication)
|
||||
|
||||
```console
|
||||
export AZURE_CLIENT_ID="XXXXXXXXX"
|
||||
export AZURE_TENANT_ID="XXXXXXXXX"
|
||||
export AZURE_CLIENT_SECRET="XXXXXXX"
|
||||
```
|
||||
|
||||
Execution with the `--sp-env-auth` flag fails if these variables are not set or exported.
|
||||
|
||||
Refer to the [Create Prowler Service Principal](create-prowler-service-principal.md) guide for detailed setup instructions.
|
||||
|
||||
### Azure Authentication Methods
|
||||
|
||||
Prowler for Azure supports the following authentication methods:
|
||||
|
||||
- **AZ CLI Authentication (`--az-cli-auth`)** – Automated authentication using stored AZ CLI credentials.
|
||||
- **Managed Identity Authentication (`--managed-identity-auth`)** – Automated authentication via Azure Managed Identity.
|
||||
- **Browser Authentication (`--browser-auth`)** – Requires the user to authenticate using the default browser. The `tenant-id` parameter is mandatory for this method.
|
||||
|
||||
### Required Permissions
|
||||
## Required Permissions
|
||||
|
||||
Prowler for Azure requires two types of permission scopes:
|
||||
|
||||
#### Microsoft Entra ID Permissions
|
||||
### Microsoft Entra ID Permissions
|
||||
|
||||
These permissions allow Prowler to retrieve metadata from the assumed identity and perform specific Entra checks. While not mandatory for execution, they enhance functionality.
|
||||
|
||||
Required permissions:
|
||||
#### Assigning Required API Permissions
|
||||
|
||||
Assign the following Microsoft Graph permissions:
|
||||
|
||||
- `Directory.Read.All`
|
||||
- `Policy.Read.All`
|
||||
- `UserAuthenticationMethod.Read.All` (used for Entra multifactor authentication checks)
|
||||
- `UserAuthenticationMethod.Read.All` (optional, for multifactor authentication (MFA) checks)
|
||||
|
||||
???+ note
|
||||
Replace `Directory.Read.All` with `Domain.Read.All` for more restrictive permissions. Note that Entra checks related to DirectoryRoles and GetUsers will not run with this permission.
|
||||
???+ note
|
||||
Replace `Directory.Read.All` with `Domain.Read.All` for more restrictive permissions. Note that Entra checks related to DirectoryRoles and GetUsers will not run with this permission.
|
||||
|
||||
=== "Azure Portal"
|
||||
|
||||
1. Go to your App Registration > "API permissions"
|
||||
|
||||

|
||||
|
||||
2. Click "+ Add a permission" > "Microsoft Graph" > "Application permissions"
|
||||
|
||||

|
||||

|
||||
|
||||
3. Search and select:
|
||||
|
||||
- `Directory.Read.All`
|
||||
- `Policy.Read.All`
|
||||
- `UserAuthenticationMethod.Read.All`
|
||||
|
||||

|
||||
|
||||
4. Click "Add permissions", then grant admin consent
|
||||
|
||||

|
||||
|
||||
=== "Azure CLI"
|
||||
|
||||
1. To grant permissions to a Service Principal, execute the following command in a terminal:
|
||||
|
||||
```console
|
||||
az ad app permission add --id {appId} --api 00000003-0000-0000-c000-000000000000 --api-permissions 7ab1d382-f21e-4acd-a863-ba3e13f7da61=Role 246dd0d5-5bd0-4def-940b-0421030a5b68=Role 38d9df27-64da-44fd-b7c5-a6fbac20248f=Role
|
||||
```
|
||||
|
||||
2. Once the permissions are assigned, admin consent is required to finalize the changes. An administrator should run:
|
||||
|
||||
```console
|
||||
az ad app permission admin-consent --id {appId}
|
||||
```
|
||||
|
||||
|
||||
#### Subscription Scope Permissions
|
||||
### Subscription Scope Permissions
|
||||
|
||||
These permissions are required to perform security checks against Azure resources. The following **RBAC roles** must be assigned per subscription to the entity used by Prowler:
|
||||
|
||||
- `Reader` – Grants read-only access to Azure resources.
|
||||
- `ProwlerRole` – A custom role with minimal permissions, defined in the [prowler-azure-custom-role](https://github.com/prowler-cloud/prowler/blob/master/permissions/prowler-azure-custom-role.json).
|
||||
- `ProwlerRole` – A custom role with minimal permissions needed for some specific checks, defined in the [prowler-azure-custom-role](https://github.com/prowler-cloud/prowler/blob/master/permissions/prowler-azure-custom-role.json).
|
||||
|
||||
???+ note
|
||||
The `assignableScopes` field in the JSON custom role file must be updated to reflect the correct subscription or management group. Use one of the following formats: `/subscriptions/<subscription-id>` or `/providers/Microsoft.Management/managementGroups/<management-group-id>`.
|
||||
|
||||
### Assigning Permissions
|
||||
#### Assigning "Reader" Role at the Subscription Level
|
||||
By default, Prowler scans all accessible subscriptions. If you need to audit specific subscriptions, you must assign the necessary role `Reader` for each one. For streamlined and less repetitive role assignments in multi-subscription environments, refer to the [following section](subscriptions.md#recommendation-for-managing-multiple-subscriptions).
|
||||
|
||||
To properly configure permissions, follow these guides:
|
||||
=== "Azure Portal"
|
||||
|
||||
1. To grant Prowler access to scan a specific Azure subscription, follow these steps in Azure Portal:
|
||||
Navigate to the subscription you want to audit with Prowler.
|
||||
|
||||
1. In the left menu, select “Access control (IAM)”.
|
||||
|
||||
2. Click “+ Add” and select “Add role assignment”.
|
||||
|
||||
3. In the search bar, enter `Reader`, select it and click “Next”.
|
||||
|
||||
4. In the “Members” tab, click “+ Select members”, then add the accounts to assign this role.
|
||||
|
||||
5. Click “Review + assign” to finalize and apply the role assignment.
|
||||
|
||||

|
||||
|
||||
=== "Azure CLI"
|
||||
|
||||
1. Open a terminal and execute the following command to assign the `Reader` role to the identity that is going to be assumed by Prowler:
|
||||
|
||||
```console
|
||||
az role assignment create --role "Reader" --assignee <user, group, or service principal> --scope /subscriptions/<subscription-id>
|
||||
```
|
||||
|
||||
2. If the command is executed successfully, the output is going to be similar to the following:
|
||||
|
||||
```json
|
||||
{
|
||||
"condition": null,
|
||||
"conditionVersion": null,
|
||||
"createdBy": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"createdOn": "YYYY-MM-DDTHH:MM:SS.SSSSSS+00:00",
|
||||
"delegatedManagedIdentityResourceId": null,
|
||||
"description": null,
|
||||
"id": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/providers/Microsoft.Authorization/roleAssignments/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"name": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"principalId": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"principalName": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"principalType": "ServicePrincipal",
|
||||
"roleDefinitionId": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/providers/Microsoft.Authorization/roleDefinitions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"roleDefinitionName": "Reader",
|
||||
"scope": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"type": "Microsoft.Authorization/roleAssignments",
|
||||
"updatedBy": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"updatedOn": "YYYY-MM-DDTHH:MM:SS.SSSSSS+00:00"
|
||||
}
|
||||
```
|
||||
|
||||
#### Assigning "ProwlerRole" Permissions at the Subscription Level
|
||||
|
||||
Some read-only permissions required for specific security checks are not included in the built-in Reader role. To support these checks, Prowler utilizes a custom role, defined in [prowler-azure-custom-role](https://github.com/prowler-cloud/prowler/blob/master/permissions/prowler-azure-custom-role.json). Once created, this role can be assigned following the same process as the `Reader` role.
|
||||
|
||||
The checks requiring this `ProwlerRole` can be found in this [section](../../tutorials/azure/authentication.md#checks-requiring-prowlerrole).
|
||||
|
||||
=== "Azure Portal"
|
||||
|
||||
1. Download the [Prowler Azure Custom Role](https://github.com/prowler-cloud/prowler/blob/master/permissions/prowler-azure-custom-role.json)
|
||||
|
||||

|
||||
|
||||
2. Modify `assignableScopes` to match your Subscription ID (e.g. `/subscriptions/xxxx-xxxx-xxxx-xxxx`)
|
||||
|
||||
3. Go to your Azure Subscription > "Access control (IAM)"
|
||||
|
||||

|
||||
|
||||
4. Click "+ Add" > "Add custom role", choose "Start from JSON" and upload the modified file
|
||||
|
||||

|
||||
|
||||
5. Click "Review + Create" to finish
|
||||
|
||||

|
||||
|
||||
6. Return to "Access control (IAM)" > "+ Add" > "Add role assignment"
|
||||
|
||||
- Assign the `Reader` role to the Application created in the previous step
|
||||
- Then repeat the same process assigning the custom `ProwlerRole`
|
||||
|
||||

|
||||
|
||||
???+ note
|
||||
The `assignableScopes` field in the JSON custom role file must be updated to reflect the correct subscription or management group. Use one of the following formats: `/subscriptions/<subscription-id>` or `/providers/Microsoft.Management/managementGroups/<management-group-id>`.
|
||||
|
||||
=== "Azure CLI"
|
||||
|
||||
1. To create a new custom role, open a terminal and execute the following command:
|
||||
|
||||
```console
|
||||
az role definition create --role-definition '{ 640ms lun 16 dic 17:04:17 2024
|
||||
"Name": "ProwlerRole",
|
||||
"IsCustom": true,
|
||||
"Description": "Role used for checks that require read-only access to Azure resources and are not covered by the Reader role.",
|
||||
"AssignableScopes": [
|
||||
"/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" // USE YOUR SUBSCRIPTION ID
|
||||
],
|
||||
"Actions": [
|
||||
"Microsoft.Web/sites/host/listkeys/action",
|
||||
"Microsoft.Web/sites/config/list/Action"
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
2. If the command is executed successfully, the output is going to be similar to the following:
|
||||
|
||||
```json
|
||||
{
|
||||
"assignableScopes": [
|
||||
"/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"
|
||||
],
|
||||
"createdBy": null,
|
||||
"createdOn": "YYYY-MM-DDTHH:MM:SS.SSSSSS+00:00",
|
||||
"description": "Role used for checks that require read-only access to Azure resources and are not covered by the Reader role.",
|
||||
"id": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/providers/Microsoft.Authorization/roleDefinitions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"name": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"permissions": [
|
||||
{
|
||||
"actions": [
|
||||
"Microsoft.Web/sites/host/listkeys/action",
|
||||
"Microsoft.Web/sites/config/list/Action"
|
||||
],
|
||||
"condition": null,
|
||||
"conditionVersion": null,
|
||||
"dataActions": [],
|
||||
"notActions": [],
|
||||
"notDataActions": []
|
||||
}
|
||||
],
|
||||
"roleName": "ProwlerRole",
|
||||
"roleType": "CustomRole",
|
||||
"type": "Microsoft.Authorization/roleDefinitions",
|
||||
"updatedBy": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"updatedOn": "YYYY-MM-DDTHH:MM:SS.SSSSSS+00:00"
|
||||
}
|
||||
```
|
||||
|
||||
### Additional Resources
|
||||
|
||||
For more detailed guidance on subscription management and permissions:
|
||||
|
||||
- [Microsoft Entra ID permissions](create-prowler-service-principal.md#assigning-proper-permissions)
|
||||
- [Azure subscription permissions](subscriptions.md)
|
||||
- [Create Prowler Service Principal](create-prowler-service-principal.md)
|
||||
|
||||
???+ warning
|
||||
Some permissions in `ProwlerRole` involve **write access**. If a `ReadOnly` lock is attached to certain resources, you may encounter errors, and findings for those checks will not be available.
|
||||
@@ -75,3 +234,56 @@ The following security checks require the `ProwlerRole` permissions for executio
|
||||
|
||||
- `app_function_access_keys_configured`
|
||||
- `app_function_ftps_deployment_disabled`
|
||||
|
||||
---
|
||||
|
||||
## Service Principal Application Authentication (Recommended)
|
||||
|
||||
This method is required for Prowler App and recommended for Prowler CLI.
|
||||
|
||||
### Creating the Service Principal
|
||||
For more information, see [Creating Prowler Service Principal](create-prowler-service-principal.md).
|
||||
|
||||
### Environment Variables (CLI)
|
||||
|
||||
For Prowler CLI, set up the following environment variables:
|
||||
|
||||
```console
|
||||
export AZURE_CLIENT_ID="XXXXXXXXX"
|
||||
export AZURE_TENANT_ID="XXXXXXXXX"
|
||||
export AZURE_CLIENT_SECRET="XXXXXXX"
|
||||
```
|
||||
|
||||
Execution with the `--sp-env-auth` flag fails if these variables are not set or exported.
|
||||
|
||||
## AZ CLI Authentication
|
||||
|
||||
*Available only for Prowler CLI*
|
||||
|
||||
Use stored Azure CLI credentials:
|
||||
|
||||
```console
|
||||
prowler azure --az-cli-auth
|
||||
```
|
||||
|
||||
## Managed Identity Authentication
|
||||
|
||||
*Available only for Prowler CLI*
|
||||
|
||||
Authenticate via Azure Managed Identity (when running on Azure resources):
|
||||
|
||||
```console
|
||||
prowler azure --managed-identity-auth
|
||||
```
|
||||
|
||||
## Browser Authentication
|
||||
|
||||
*Available only for Prowler CLI*
|
||||
|
||||
Authenticate using the default browser:
|
||||
|
||||
```console
|
||||
prowler azure --browser-auth --tenant-id <tenant-id>
|
||||
```
|
||||
|
||||
> **Note:** The `tenant-id` parameter is mandatory for browser authentication.
|
||||
|
||||
@@ -2,26 +2,39 @@
|
||||
|
||||
To enable Prowler to assume an identity for scanning with the required privileges, a Service Principal must be created. This Service Principal authenticates against Azure and retrieves necessary metadata for checks.
|
||||
|
||||
### Methods for Creating a Service Principal
|
||||
|
||||
Service Principal Applications can be created using either the Azure Portal or the Azure CLI.
|
||||
|
||||
## Creating a Service Principal via Azure Portal / Entra Admin Center
|
||||
|
||||
1. Access Microsoft Entra ID.
|
||||
2. In the left menu bar, navigate to **"App registrations"**.
|
||||
3. Click **"+ New registration"** in the menu bar to register a new application
|
||||
4. Fill the **"Name"**, select the **"Supported account types"** and click **"Register"**. You will be redirected to the applications page.
|
||||
5. In the left menu bar, select **"Certificates & secrets"**.
|
||||
6. Under the **"Certificates & secrets"** view, click **"+ New client secret"**.
|
||||
7. Fill the **"Description"** and **"Expires"** fields, then click **"Add"**.
|
||||
8. Copy the secret value, as it will be used as `AZURE_CLIENT_SECRET` environment variable.
|
||||
|
||||

|
||||
|
||||
## From Azure CLI
|
||||
## Creating a Service Principal via Azure Portal / Entra Admin Center
|
||||
|
||||
### Creating a Service Principal
|
||||
1. Access **Microsoft Entra ID** in the [Azure Portal](https://portal.azure.com)
|
||||
|
||||

|
||||
|
||||
2. Navigate to "Manage" > "App registrations"
|
||||
|
||||

|
||||
|
||||
3. Click "+ New registration", complete the form, and click "Register"
|
||||
|
||||

|
||||
|
||||
4. Go to "Certificates & secrets" > "+ New client secret"
|
||||
|
||||

|
||||

|
||||
|
||||
5. Fill in the required fields and click "Add", then copy the generated value
|
||||
|
||||
| Value | Description |
|
||||
|-------|-----------|
|
||||
| Client ID | Application ID |
|
||||
| Client Secret | Secret to Connect to the App |
|
||||
| Tenant ID | Microsoft Entra Tenant ID |
|
||||
|
||||
|
||||
## Creating a Service Principal from Azure CLI
|
||||
|
||||
To create a Service Principal using the Azure CLI, follow these steps:
|
||||
|
||||
@@ -46,55 +59,4 @@ To create a Service Principal using the Azure CLI, follow these steps:
|
||||
|
||||
## Assigning Proper Permissions
|
||||
|
||||
To allow Prowler to retrieve metadata from the assumed identity and run Entra checks, assign the following permissions:
|
||||
|
||||
- `Directory.Read.All`
|
||||
- `Policy.Read.All`
|
||||
- `UserAuthenticationMethod.Read.All` (used only for the Entra checks related with multifactor authentication)
|
||||
|
||||
Permissions can be assigned via the Azure Portal or the Azure CLI.
|
||||
|
||||
???+ note
|
||||
After creating and assigning the necessary Entra permissions, follow this [tutorial](../azure/subscriptions.md) to add subscription permissions to the application and start scanning your resources.
|
||||
|
||||
### Assigning the Reader Role in Azure Portal
|
||||
|
||||
1. Access Microsoft Entra ID.
|
||||
|
||||
2. In the left menu bar, navigate to “App registrations”.
|
||||
|
||||
3. Select the created application.
|
||||
|
||||
4. In the left menu bar, select “API permissions”.
|
||||
|
||||
5. Click “+ Add a permission” and select “Microsoft Graph”.
|
||||
|
||||
6. In the “Microsoft Graph” view, select “Application permissions”.
|
||||
|
||||
7. Finally, search for "Directory", "Policy" and "UserAuthenticationMethod" select the following permissions:
|
||||
|
||||
- `Directory.Read.All`
|
||||
|
||||
- `Policy.Read.All`
|
||||
|
||||
- `UserAuthenticationMethod.Read.All`
|
||||
|
||||
8. Click “Add permissions” to apply the new permissions.
|
||||
|
||||
9. Finally, an admin must click “Grant admin consent for \[your tenant]” to apply the permissions.
|
||||
|
||||

|
||||
|
||||
### From Azure CLI
|
||||
|
||||
1. To grant permissions to a Service Principal, execute the following command in a terminal:
|
||||
|
||||
```console
|
||||
az ad app permission add --id {appId} --api 00000003-0000-0000-c000-000000000000 --api-permissions 7ab1d382-f21e-4acd-a863-ba3e13f7da61=Role 246dd0d5-5bd0-4def-940b-0421030a5b68=Role 38d9df27-64da-44fd-b7c5-a6fbac20248f=Role
|
||||
```
|
||||
|
||||
2. Once the permissions are assigned, admin consent is required to finalize the changes. An administrator should run:
|
||||
|
||||
```console
|
||||
az ad app permission admin-consent --id {appId}
|
||||
```
|
||||
Go to [Assigning Proper Permissions](./authentication.md#required-permissions) to learn how to assign the necessary permissions to the Service Principal.
|
||||
@@ -1,31 +1,23 @@
|
||||
# Getting Started with Azure on Prowler Cloud/App
|
||||
# Getting Started With Azure on Prowler
|
||||
|
||||
## Prowler App
|
||||
|
||||
<iframe width="560" height="380" src="https://www.youtube-nocookie.com/embed/v1as8vTFlMg" title="Prowler Cloud Onboarding Azure" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen="1"></iframe>
|
||||
> Walkthrough video onboarding an Azure Subscription using Service Principal.
|
||||
|
||||
Set up your Azure subscription to enable security scanning using Prowler Cloud/App.
|
||||
|
||||
???+ note "Government Cloud Support"
|
||||
Government cloud subscriptions (Azure Government) are not currently supported, but we expect to add support for them in the near future.
|
||||
|
||||
## Requirements
|
||||
### Prerequisites
|
||||
|
||||
To configure your Azure subscription, you’ll need:
|
||||
Before setting up Azure in Prowler App, you need to create a Service Principal with proper permissions.
|
||||
|
||||
1. Get the `Subscription ID`
|
||||
2. Access to Prowler Cloud/App
|
||||
3. Configure authentication in Azure:
|
||||
|
||||
3.1 Create a Service Principal
|
||||
|
||||
3.2 Assign required permissions
|
||||
|
||||
3.3 Assign permissions at the subscription level
|
||||
|
||||
4. Add the credentials to Prowler Cloud/App
|
||||
For detailed instructions on how to create the Service Principal and configure permissions, see [Authentication > Service Principal](./authentication.md#service-principal-application-authentication-recommended).
|
||||
|
||||
---
|
||||
|
||||
## Step 1: Get the Subscription ID
|
||||
### Step 1: Get the Subscription ID
|
||||
|
||||
1. Go to the [Azure Portal](https://portal.azure.com/#home) and search for `Subscriptions`
|
||||
2. Locate and copy your Subscription ID
|
||||
@@ -35,9 +27,9 @@ To configure your Azure subscription, you’ll need:
|
||||
|
||||
---
|
||||
|
||||
## Step 2: Access Prowler Cloud/App
|
||||
### Step 2: Access Prowler App
|
||||
|
||||
1. Go to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](../prowler-app.md)
|
||||
1. Navigate to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](../prowler-app.md)
|
||||
2. Navigate to `Configuration` > `Cloud Providers`
|
||||
|
||||

|
||||
@@ -54,117 +46,19 @@ To configure your Azure subscription, you’ll need:
|
||||
|
||||

|
||||
|
||||
---
|
||||
### Step 3: Add Credentials to Prowler App
|
||||
|
||||
## Step 3: Configure the Azure Subscription
|
||||
|
||||
### Create the Service Principal
|
||||
|
||||
A Service Principal is required to grant Prowler the necessary privileges.
|
||||
|
||||
1. Access **Microsoft Entra ID**
|
||||
|
||||

|
||||
|
||||
2. Navigate to `Manage` > `App registrations`
|
||||
|
||||

|
||||
|
||||
3. Click `+ New registration`, complete the form, and click `Register`
|
||||
|
||||

|
||||
|
||||
4. Go to `Certificates & secrets` > `+ New client secret`
|
||||
|
||||

|
||||

|
||||
|
||||
5. Fill in the required fields and click `Add`, then copy the generated value
|
||||
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| Client ID | Application ID |
|
||||
| Client Secret | AZURE_CLIENT_SECRET |
|
||||
| Tenant ID | Azure Active Directory tenant ID |
|
||||
|
||||
---
|
||||
|
||||
### Assign Required API Permissions
|
||||
|
||||
Assign the following Microsoft Graph permissions:
|
||||
|
||||
- Directory.Read.All
|
||||
|
||||
- Policy.Read.All
|
||||
|
||||
- UserAuthenticationMethod.Read.All (optional, for MFA checks)
|
||||
|
||||
???+ note
|
||||
You can replace `Directory.Read.All` with `Domain.Read.All` that is a more restrictive permission but you won't be able to run the Entra checks related with DirectoryRoles and GetUsers.
|
||||
|
||||
1. Go to your App Registration > `API permissions`
|
||||
|
||||

|
||||
|
||||
2. Click `+ Add a permission` > `Microsoft Graph` > `Application permissions`
|
||||
|
||||

|
||||

|
||||
|
||||
3. Search and select:
|
||||
|
||||
- `Directory.Read.All`
|
||||
- `Policy.Read.All`
|
||||
- `UserAuthenticationMethod.Read.All`
|
||||
|
||||

|
||||
|
||||
4. Click `Add permissions`, then grant admin consent
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
### Assign Permissions at the Subscription Level
|
||||
|
||||
1. Download the [Prowler Azure Custom Role](https://github.com/prowler-cloud/prowler/blob/master/permissions/prowler-azure-custom-role.json)
|
||||
|
||||

|
||||
|
||||
2. Modify `assignableScopes` to match your Subscription ID (e.g. `/subscriptions/xxxx-xxxx-xxxx-xxxx`)
|
||||
|
||||
3. Go to your Azure Subscription > `Access control (IAM)`
|
||||
|
||||

|
||||
|
||||
4. Click `+ Add` > `Add custom role`, choose "Start from JSON" and upload the modified file
|
||||
|
||||

|
||||
|
||||
5. Click `Review + Create` to finish
|
||||
|
||||

|
||||
|
||||
6. Return to `Access control (IAM)` > `+ Add` > `Add role assignment`
|
||||
|
||||
- Assign the `Reader` role to the Application created in the previous step
|
||||
- Then repeat the same process assigning the custom `ProwlerRole`
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Step 4: Add Credentials to Prowler Cloud/App
|
||||
Having completed the [Service Principal setup from the Authentication guide](./authentication.md#service-principal-application-authentication-recommended):
|
||||
|
||||
1. Go to your App Registration overview and copy the `Client ID` and `Tenant ID`
|
||||
|
||||

|
||||
|
||||
2. Go to Prowler Cloud/App and paste:
|
||||
2. Go to Prowler App and paste:
|
||||
|
||||
- `Client ID`
|
||||
- `Tenant ID`
|
||||
- `AZURE_CLIENT_SECRET` from earlier
|
||||
- `Client Secret` from [earlier](./authentication.md#service-principal-application-authentication-recommended)
|
||||
|
||||

|
||||
|
||||
@@ -172,6 +66,70 @@ Assign the following Microsoft Graph permissions:
|
||||
|
||||

|
||||
|
||||
4. Click `Launch Scan`
|
||||
4. Click "Launch Scan"
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
### Configure Azure Credentials
|
||||
|
||||
To authenticate with Azure, Prowler CLI supports multiple authentication methods. Choose the method that best suits your environment.
|
||||
|
||||
For detailed authentication setup instructions, see [Authentication](./authentication.md).
|
||||
|
||||
**Service Principal (Recommended)**
|
||||
|
||||
Set up environment variables:
|
||||
|
||||
```console
|
||||
export AZURE_CLIENT_ID="XXXXXXXXX"
|
||||
export AZURE_TENANT_ID="XXXXXXXXX"
|
||||
export AZURE_CLIENT_SECRET="XXXXXXX"
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```console
|
||||
prowler azure --sp-env-auth
|
||||
```
|
||||
|
||||
**Azure CLI Credentials**
|
||||
|
||||
Use stored Azure CLI credentials:
|
||||
|
||||
```console
|
||||
prowler azure --az-cli-auth
|
||||
```
|
||||
|
||||
**Browser Authentication**
|
||||
|
||||
Authenticate using your default browser:
|
||||
|
||||
```console
|
||||
prowler azure --browser-auth --tenant-id <tenant-id>
|
||||
```
|
||||
|
||||
**Managed Identity**
|
||||
|
||||
When running on Azure resources:
|
||||
|
||||
```console
|
||||
prowler azure --managed-identity-auth
|
||||
```
|
||||
|
||||
### Subscription Selection
|
||||
|
||||
To scan a specific Azure subscription:
|
||||
|
||||
```console
|
||||
prowler azure --subscription-ids <subscription-id>
|
||||
```
|
||||
|
||||
To scan multiple Azure subscriptions:
|
||||
|
||||
```console
|
||||
prowler azure --subscription-ids <subscription-id1> <subscription-id2> <subscription-id3>
|
||||
```
|
||||
|
||||
@@ -18,131 +18,7 @@ Prowler allows you to specify one or more subscriptions for scanning (up to N),
|
||||
The multi-subscription feature is available only in the CLI. In Prowler App, each scan is limited to a single subscription.
|
||||
|
||||
## Assigning Permissions for Subscription Scans
|
||||
|
||||
To perform scans, ensure that the identity assumed by Prowler has the appropriate permissions.
|
||||
|
||||
By default, Prowler scans all accessible subscriptions. If you need to audit specific subscriptions, you must assign the necessary role `Reader` for each one. For streamlined and less repetitive role assignments in multi-subscription environments, refer to the [following section](#recommendation-for-managing-multiple-subscriptions).
|
||||
|
||||
### Assigning the Reader Role in Azure Portal
|
||||
|
||||
1. To grant Prowler access to scan a specific Azure subscription, follow these steps in Azure Portal:
|
||||
Navigate to the subscription you want to audit with Prowler.
|
||||
|
||||
2. In the left menu, select “Access control (IAM)”.
|
||||
|
||||
3. Click “+ Add” and select “Add role assignment”.
|
||||
|
||||
4. In the search bar, enter `Reader`, select it and click “Next”.
|
||||
|
||||
5. In the “Members” tab, click “+ Select members”, then add the accounts to assign this role.
|
||||
|
||||
6. Click “Review + assign” to finalize and apply the role assignment.
|
||||
|
||||

|
||||
|
||||
### From Azure CLI
|
||||
|
||||
1. Open a terminal and execute the following command to assign the `Reader` role to the identity that is going to be assumed by Prowler:
|
||||
|
||||
```console
|
||||
az role assignment create --role "Reader" --assignee <user, group, or service principal> --scope /subscriptions/<subscription-id>
|
||||
```
|
||||
|
||||
2. If the command is executed successfully, the output is going to be similar to the following:
|
||||
|
||||
```json
|
||||
{
|
||||
"condition": null,
|
||||
"conditionVersion": null,
|
||||
"createdBy": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"createdOn": "YYYY-MM-DDTHH:MM:SS.SSSSSS+00:00",
|
||||
"delegatedManagedIdentityResourceId": null,
|
||||
"description": null,
|
||||
"id": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/providers/Microsoft.Authorization/roleAssignments/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"name": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"principalId": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"principalName": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"principalType": "ServicePrincipal",
|
||||
"roleDefinitionId": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/providers/Microsoft.Authorization/roleDefinitions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"roleDefinitionName": "Reader",
|
||||
"scope": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"type": "Microsoft.Authorization/roleAssignments",
|
||||
"updatedBy": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"updatedOn": "YYYY-MM-DDTHH:MM:SS.SSSSSS+00:00"
|
||||
}
|
||||
```
|
||||
|
||||
### Prowler Custom Role
|
||||
|
||||
Some read-only permissions required for specific security checks are not included in the built-in Reader role. To support these checks, Prowler utilizes a custom role, defined in [prowler-azure-custom-role](https://github.com/prowler-cloud/prowler/blob/master/permissions/prowler-azure-custom-role.json). Once created, this role can be assigned following the same process as the `Reader` role.
|
||||
|
||||
The checks requiring this `ProwlerRole` can be found in this [section](../../tutorials/azure/authentication.md#checks-requiring-prowlerrole).
|
||||
|
||||
#### Create ProwlerRole via Azure Portal
|
||||
|
||||
1. Download the [prowler-azure-custom-role](https://github.com/prowler-cloud/prowler/blob/master/permissions/prowler-azure-custom-role.json) file and modify the `assignableScopes` field to match the target subscription. Example format: `/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`.
|
||||
|
||||
2. Access your Azure subscription.
|
||||
|
||||
3. Select “Access control (IAM)”.
|
||||
|
||||
4. Click “+ Add” and select “Add custom role”.
|
||||
|
||||
5. Under “Baseline permissions”, select “Start from JSON” and upload the modified role file.
|
||||
|
||||
6. Click “Review + create” to finalize the role creation.
|
||||
|
||||
#### Create ProwlerRole via Azure CLI
|
||||
|
||||
1. To create a new custom role, open a terminal and execute the following command:
|
||||
|
||||
```console
|
||||
az role definition create --role-definition '{ 640ms lun 16 dic 17:04:17 2024
|
||||
"Name": "ProwlerRole",
|
||||
"IsCustom": true,
|
||||
"Description": "Role used for checks that require read-only access to Azure resources and are not covered by the Reader role.",
|
||||
"AssignableScopes": [
|
||||
"/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" // USE YOUR SUBSCRIPTION ID
|
||||
],
|
||||
"Actions": [
|
||||
"Microsoft.Web/sites/host/listkeys/action",
|
||||
"Microsoft.Web/sites/config/list/Action"
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
2. If the command is executed successfully, the output is going to be similar to the following:
|
||||
|
||||
```json
|
||||
{
|
||||
"assignableScopes": [
|
||||
"/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"
|
||||
],
|
||||
"createdBy": null,
|
||||
"createdOn": "YYYY-MM-DDTHH:MM:SS.SSSSSS+00:00",
|
||||
"description": "Role used for checks that require read-only access to Azure resources and are not covered by the Reader role.",
|
||||
"id": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/providers/Microsoft.Authorization/roleDefinitions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"name": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"permissions": [
|
||||
{
|
||||
"actions": [
|
||||
"Microsoft.Web/sites/host/listkeys/action",
|
||||
"Microsoft.Web/sites/config/list/Action"
|
||||
],
|
||||
"condition": null,
|
||||
"conditionVersion": null,
|
||||
"dataActions": [],
|
||||
"notActions": [],
|
||||
"notDataActions": []
|
||||
}
|
||||
],
|
||||
"roleName": "ProwlerRole",
|
||||
"roleType": "CustomRole",
|
||||
"type": "Microsoft.Authorization/roleDefinitions",
|
||||
"updatedBy": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
|
||||
"updatedOn": "YYYY-MM-DDTHH:MM:SS.SSSSSS+00:00"
|
||||
}
|
||||
```
|
||||
Check the [Authentication > Subscription Scope Permissions](authentication.md#subscription-scope-permissions) guide for more information on how to assign permissions for subscription scans.
|
||||
|
||||
## Recommendation for Managing Multiple Subscriptions
|
||||
|
||||
|
||||
@@ -0,0 +1,483 @@
|
||||
# Prowler ThreatScore Documentation
|
||||
|
||||
## Introduction
|
||||
|
||||
The **Prowler ThreatScore** is a comprehensive compliance scoring system that provides a unified metric for assessing your organization's security posture across compliance frameworks. It aggregates findings from individual security checks into a single, normalized score ranging from 0 to 100.
|
||||
|
||||
### Purpose
|
||||
- **Unified View**: Get a single metric representing overall compliance health
|
||||
- **Risk Prioritization**: Understand which areas pose the highest security risks
|
||||
- **Progress Tracking**: Monitor improvements in compliance posture over time
|
||||
- **Executive Reporting**: Provide clear, quantifiable security metrics to stakeholders
|
||||
|
||||
## How ThreatScore Works
|
||||
|
||||
The ThreatScore calculation considers four critical factors for each compliance requirement:
|
||||
|
||||
### 1. Pass Rate (`rate_i`)
|
||||
The percentage of security checks that passed for a specific requirement:
|
||||
```
|
||||
Pass Rate = (Number of PASS findings) / (Total findings)
|
||||
```
|
||||
|
||||
### 2. Total Findings (`total_i`)
|
||||
The total number of checks performed (both PASS and FAIL) for a requirement. This represents the amount of evidence available - more findings provide greater confidence in the assessment.
|
||||
|
||||
### 3. Weight (`weight_i`)
|
||||
A numerical value (1-1000) representing the business importance or criticality of the requirement within your organization's context.
|
||||
|
||||
### 4. Risk Level (`risk_i`)
|
||||
A severity rating (1-5) indicating the potential impact of non-compliance with this requirement.
|
||||
|
||||
## Score Interpretation Guidelines
|
||||
|
||||
| ThreatScore | Interpretation | Recommended Actions |
|
||||
|------------------|----------------|-------------------|
|
||||
| 90-100% | Excellent | Maintain current controls, focus on continuous improvement |
|
||||
| 80-89% | Good | Address remaining gaps, prepare for compliance audits |
|
||||
| 70-79% | Acceptable | Prioritize high-risk failures, develop improvement plan |
|
||||
| 60-69% | Needs Improvement | Immediate attention required, may not pass compliance audit |
|
||||
| Below 60% | Critical | Emergency response needed, potential regulatory issues |
|
||||
|
||||
## Mathematical Formula
|
||||
|
||||
The ThreatScore uses a weighted average formula that accounts for all four factors:
|
||||
|
||||
```
|
||||
ThreatScore = (Σ(rate_i × total_i × weight_i × risk_i) / Σ(total_i × weight_i × risk_i)) × 100
|
||||
```
|
||||
|
||||
### Formula Properties
|
||||
- **Normalization**: Always produces a score between 0 and 100
|
||||
- **Evidence-weighted**: Requirements with more findings have proportionally greater influence
|
||||
- **Risk-sensitive**: Higher risk requirements impact the score more significantly
|
||||
- **Business-aligned**: Weight values allow customization based on organizational priorities
|
||||
|
||||
## Parameters Explained
|
||||
|
||||
### Weight Values (1-1000)
|
||||
|
||||
The weight parameter allows customization of ThreatScore calculation based on organizational priorities and regulatory requirements.
|
||||
|
||||
#### Weight Assignment Guidelines
|
||||
|
||||
| Weight Range | Priority Level | Use Cases |
|
||||
|--------------|----------------|-----------|
|
||||
| 1-100 | Low | Optional or nice-to-have controls |
|
||||
| 101-300 | Medium | Standard security practices |
|
||||
| 301-600 | High | Important security controls |
|
||||
| 601-850 | Critical | Regulatory compliance requirements |
|
||||
| 851-1000 | Maximum | Mission-critical security controls |
|
||||
|
||||
#### Weight Selection Strategy
|
||||
1. **Regulatory Mapping**: Assign higher weights to controls required by industry regulations
|
||||
2. **Business Impact**: Consider the potential business impact of control failures
|
||||
3. **Risk Tolerance**: Align weights with organizational risk appetite
|
||||
4. **Stakeholder Input**: Involve compliance and business teams in weight decisions
|
||||
|
||||
### Risk Levels (1-5)
|
||||
|
||||
Risk levels represent the potential security impact of non-compliance with a requirement.
|
||||
|
||||
| Risk Level | Severity | Impact Description |
|
||||
|------------|----------|-------------------|
|
||||
| 1 | Very Low | Minimal security impact, informational |
|
||||
| 2 | Low | Limited exposure, low probability of exploitation |
|
||||
| 3 | Medium | Moderate security risk, potential for limited damage |
|
||||
| 4 | High | Significant security risk, high probability of impact |
|
||||
| 5 | Critical | Severe security risk, immediate threat to organization |
|
||||
|
||||
#### Risk Level Assessment Criteria
|
||||
- **Confidentiality Impact**: Data exposure potential
|
||||
- **Integrity Impact**: Risk of unauthorized data modification
|
||||
- **Availability Impact**: Service disruption potential
|
||||
- **Compliance Impact**: Regulatory violation consequences
|
||||
- **Exploitability**: Ease of exploitation by threat actors
|
||||
|
||||
## Security Pillars and Subpillars
|
||||
|
||||
Prowler organizes security requirements into a hierarchical structure of pillars and subpillars, providing a comprehensive framework for security assessment and compliance evaluation.
|
||||
|
||||
### Security Pillars Overview
|
||||
|
||||
The ThreatScore calculation considers requirements organized within the following security pillars:
|
||||
|
||||
#### 1. IAM (Identity and Access Management)
|
||||
|
||||
**Purpose**: Controls who can access what resources and under what conditions
|
||||
|
||||
**Subpillars**:
|
||||
|
||||
- **1.1 Authentication**: Verifying user and system identities
|
||||
- **1.2 Authorization**: Controlling access to resources based on authenticated identity
|
||||
- **1.3 Privilege Escalation**: Preventing unauthorized elevation of permissions
|
||||
|
||||
#### 2. Attack Surface
|
||||
|
||||
**Purpose**: Minimizing exposure points that could be exploited by threat actors across network, storage, and application layers
|
||||
|
||||
**Subpillars**:
|
||||
|
||||
- **2.1 Network**: Network infrastructure security, segmentation, firewall rules, VPC configurations, and traffic controls
|
||||
- **2.2 Storage**: Data storage systems security, database security, file system permissions, backup security, and storage encryption
|
||||
- **2.3 Application**: Application-level controls and configurations, application security settings, code security, runtime protections
|
||||
|
||||
#### 3. Logging and Monitoring
|
||||
|
||||
**Purpose**: Ensuring comprehensive visibility and audit capabilities
|
||||
|
||||
**Subpillars**:
|
||||
|
||||
- **3.1 Logging**: Capturing security-relevant events and activities
|
||||
- **3.2 Retention**: Maintaining logs for appropriate time periods
|
||||
- **3.3 Monitoring**: Active surveillance and alerting on security events
|
||||
|
||||
#### 4. Encryption
|
||||
|
||||
**Purpose**: Protecting data confidentiality through cryptographic controls
|
||||
|
||||
**Subpillars**:
|
||||
|
||||
- **4.1 In-Transit**: Encrypting data during transmission
|
||||
- **4.2 At-Rest**: Encrypting stored data
|
||||
|
||||
### Pillar Hierarchy and ThreatScore Impact
|
||||
|
||||
#### Hierarchy Structure
|
||||
```
|
||||
Security Framework
|
||||
├── 1. IAM
|
||||
│ ├── 1.1 Authentication
|
||||
│ ├── 1.2 Authorization
|
||||
│ └── 1.3 Privilege Escalation
|
||||
├── 2. Attack Surface
|
||||
│ ├── 2.1 Network
|
||||
│ ├── 2.2 Storage
|
||||
│ └── 2.3 Application
|
||||
├── 3. Logging and Monitoring
|
||||
│ ├── 3.1 Logging
|
||||
│ ├── 3.2 Retention
|
||||
│ └── 3.3 Monitoring
|
||||
└── 4. Encryption
|
||||
├── 4.1 In-Transit
|
||||
└── 4.2 At-Rest
|
||||
|
||||
Example Requirement Structure:
|
||||
├── Pillar: 1. IAM
|
||||
│ ├── Subpillar: 1.1 Authentication
|
||||
│ │ ├── Requirement: MFA Implementation
|
||||
│ │ │ ├── Check 1: Admin accounts use MFA
|
||||
│ │ │ ├── Check 2: Regular users use MFA
|
||||
│ │ │ └── Check 3: Service accounts use MFA
|
||||
│ │ └── [Additional Requirements]
|
||||
│ └── [Additional Subpillars: Authorization, Privilege Escalation]
|
||||
```
|
||||
|
||||
#### Weight and Risk Assignment by Pillar
|
||||
|
||||
Different pillars typically receive different weight and risk assignments based on their security impact:
|
||||
|
||||
| Pillar | Typical Weight Range | Typical Risk Range | Rationale |
|
||||
|--------|---------------------|-------------------|-----------|
|
||||
| 1. IAM | 800-1000 | 4-5 | Critical for access control, high impact if compromised |
|
||||
| 2. Attack Surface | 500-900 | 3-5 | Highly dependent on exposure and criticality across network, storage, and application layers |
|
||||
| 3. Logging and Monitoring | 600-800 | 3-4 | Important for detection and compliance, moderate direct impact |
|
||||
| 4. Encryption | 700-950 | 4-5 | Essential for data protection, regulatory compliance |
|
||||
|
||||
**Subpillar Weight Considerations**:
|
||||
|
||||
- **2.1 Network (Attack Surface)**: 500-800, Risk 3-4 - Network perimeter defense
|
||||
- **2.2 Storage (Attack Surface)**: 600-900, Risk 4-5 - Data exposure impact
|
||||
- **2.3 Application (Attack Surface)**: 400-700, Risk 2-4 - Varies by application criticality
|
||||
|
||||
### Pillar-Specific Scoring Considerations
|
||||
|
||||
#### High-Impact Pillars (1. IAM, 4. Encryption)
|
||||
|
||||
- **Characteristics**: Direct impact on data protection and access control
|
||||
- **ThreatScore Impact**: Failures in these pillars significantly lower overall score
|
||||
- **Weight Strategy**: Assign maximum weights (800-1000) to critical requirements
|
||||
- **Risk Strategy**: Most requirements rated 4-5 due to severe consequences
|
||||
|
||||
#### Variable-Impact Pillar (2. Attack Surface)
|
||||
|
||||
- **Characteristics**: Impact varies significantly across subpillars (Network, Storage, Application)
|
||||
- **ThreatScore Impact**: Depends on specific subpillar and business context
|
||||
- **Weight Strategy**:
|
||||
- 2.1 Network subpillar: 500-800 (perimeter defense importance)
|
||||
- 2.2 Storage subpillar: 600-900 (data exposure risk)
|
||||
- 2.3 Application subpillar: 400-700 (application-specific criticality)
|
||||
- **Risk Strategy**: Wide range (2-5) based on exposure, data sensitivity, and business criticality
|
||||
|
||||
#### Monitoring Pillar (3. Logging and Monitoring)
|
||||
|
||||
- **Characteristics**: Essential for compliance and incident response
|
||||
- **ThreatScore Impact**: Moderate influence, critical for audit requirements
|
||||
- **Weight Strategy**: Consistent weights (600-800) across logging, retention, and monitoring subpillars
|
||||
- **Risk Strategy**: Moderate risk levels (3-4) with emphasis on compliance impact
|
||||
|
||||
### Cross-Pillar Dependencies
|
||||
|
||||
#### Authentication ↔ Authorization (IAM)
|
||||
|
||||
- Strong authentication enables effective authorization controls
|
||||
- Weight both subpillars highly as they're interdependent
|
||||
|
||||
#### Logging ↔ Monitoring (Logging and Monitoring)
|
||||
|
||||
- Logging provides the data that monitoring systems analyze
|
||||
- Balance weights to ensure both data collection and analysis are prioritized
|
||||
|
||||
#### In-Transit ↔ At-Rest (Encryption)
|
||||
|
||||
- Comprehensive data protection requires both encryption types
|
||||
- Consider data flow patterns when assigning relative weights
|
||||
|
||||
### Pillar Coverage in ThreatScore
|
||||
|
||||
#### Complete Coverage Benefits
|
||||
|
||||
- **Comprehensive Assessment**: All security domains represented in score
|
||||
- **Balanced View**: Prevents over-emphasis on single security aspect
|
||||
- **Regulatory Alignment**: Covers requirements across major compliance frameworks
|
||||
|
||||
#### Partial Coverage Considerations
|
||||
|
||||
- **Focused Assessment**: Target specific security domains
|
||||
- **Resource Optimization**: Concentrate efforts on high-priority areas
|
||||
- **Gradual Implementation**: Phase in additional pillars over time
|
||||
|
||||
## Scoring Examples
|
||||
|
||||
### Example 1: Basic Two-Requirement Scenario
|
||||
|
||||
Consider a compliance framework with two requirements:
|
||||
|
||||
**Requirement 1: Encryption at Rest**
|
||||
|
||||
- Findings: 200 PASS, 500 FAIL (total = 700)
|
||||
- Pass Rate: 200/700 = 0.286 (28.6%)
|
||||
- Weight: 500 (High priority - data protection)
|
||||
- Risk Level: 4 (High risk - data exposure)
|
||||
|
||||
**Requirement 2: Access Logging**
|
||||
|
||||
- Findings: 300 PASS, 100 FAIL (total = 400)
|
||||
- Pass Rate: 300/400 = 0.75 (75%)
|
||||
- Weight: 800 (Critical for audit compliance)
|
||||
- Risk Level: 3 (Medium risk - audit trail)
|
||||
|
||||
**Calculation:**
|
||||
```
|
||||
Numerator = (0.286 × 700 × 500 × 4) + (0.75 × 400 × 800 × 3)
|
||||
= (400,400) + (720,000)
|
||||
= 1,120,400
|
||||
|
||||
Denominator = (700 × 500 × 4) + (400 × 800 × 3)
|
||||
= 1,400,000 + 960,000
|
||||
= 2,360,000
|
||||
|
||||
ThreatScore = (1,120,400 / 2,360,000) × 100 = 47.5%
|
||||
```
|
||||
|
||||
### Example 2: Enterprise Scenario with Pillar Structure
|
||||
|
||||
This example demonstrates how pillar organization affects ThreatScore calculation:
|
||||
|
||||
| Pillar | Subpillar | Requirement | Pass | Fail | Total | Weight | Risk | Pass Rate |
|
||||
|--------|-----------|-------------|------|------|-------|--------|------|-----------|
|
||||
| 1. IAM | 1.2 Authorization | Access Controls | 280 | 120 | 400 | 800 | 4 | 70% |
|
||||
| 2. Attack Surface | 2.1 Network | Network Segmentation | 150 | 50 | 200 | 750 | 4 | 75% |
|
||||
| 2. Attack Surface | 2.2 Storage | Backup Security | 200 | 100 | 300 | 600 | 3 | 66.7% |
|
||||
| 3. Logging and Monitoring | 3.1 Logging | Audit Logging | 350 | 50 | 400 | 700 | 3 | 87.5% |
|
||||
| 4. Encryption | 4.2 At-Rest | Encryption | 450 | 50 | 500 | 950 | 5 | 90% |
|
||||
|
||||
**Step-by-step Calculation:**
|
||||
|
||||
1. **Calculate weighted contributions for each requirement:**
|
||||
|
||||
```
|
||||
Numerator = Σ(rate_i × total_i × weight_i × risk_i)
|
||||
```
|
||||
|
||||
- **Access Controls (1.2 Authorization)**: 0.70 × 400 × 800 × 4 = 896,000
|
||||
- **Network Segmentation (2.1 Network)**: 0.75 × 200 × 750 × 4 = 450,000
|
||||
- **Backup Security (2.2 Storage)**: 0.667 × 300 × 600 × 3 = 360,060
|
||||
- **Audit Logging (3.1 Logging)**: 0.875 × 400 × 700 × 3 = 735,000
|
||||
- **Encryption (4.2 At-Rest)**: 0.90 × 500 × 950 × 5 = 2,137,500
|
||||
|
||||
2. **Sum numerator:** 2,137,500 + 896,000 + 735,000 + 360,060 + 450,000 = **4,578,560**
|
||||
|
||||
3. **Calculate total weights for each requirement:**
|
||||
|
||||
```
|
||||
Denominator = Σ(total_i × weight_i × risk_i)
|
||||
```
|
||||
|
||||
- **Access Controls (1.2 Authorization)**: 400 × 800 × 4 = 1,280,000
|
||||
- **Network Segmentation (2.1 Network)**: 200 × 750 × 4 = 600,000
|
||||
- **Backup Security (2.2 Storage)**: 300 × 600 × 3 = 540,000
|
||||
- **Audit Logging (3.1 Logging)**: 400 × 700 × 3 = 840,000
|
||||
- **Encryption (4.2 At-Rest)**: 500 × 950 × 5 = 2,375,000
|
||||
|
||||
4. **Sum denominator:** 2,375,000 + 1,280,000 + 840,000 + 540,000 + 600,000 = **5,635,000**
|
||||
|
||||
5. **Final ThreatScore calculation:**
|
||||
|
||||
```
|
||||
ThreatScore = (Numerator / Denominator) × 100
|
||||
ThreatScore = (4,578,560 / 5,635,000) × 100 = 81.2%
|
||||
```
|
||||
|
||||
**Pillar-Level Analysis:**
|
||||
|
||||
- **1. IAM pillar (1.2 Authorization)**: Significant impact despite lower pass rate (70%) due to high weight (800)
|
||||
- **2. Attack Surface pillar (2.1 Network)**: Strong performance (75%) with high weight (750) balances the score
|
||||
- **2. Attack Surface pillar (2.2 Storage)**: Lowest performance (66.7%) but limited impact due to moderate weight (600)
|
||||
- **3. Logging and Monitoring pillar (3.1 Logging)**: Moderate contribution with good performance (87.5%)
|
||||
- **4. Encryption pillar (4.2 At-Rest)**: Highest contribution due to maximum weight (950) and risk (5)
|
||||
|
||||
### Example 3: Multi-Pillar Comprehensive Scenario
|
||||
|
||||
|
||||
| Pillar | Subpillar | Requirement | Pass | Fail | Weight | Risk | Pass Rate |
|
||||
|--------|-----------|-------------|------|------|--------|------|-----------|
|
||||
| 1. IAM | 1.1 Authentication | MFA Implementation | 180 | 20 | 900 | 5 | 90% |
|
||||
| 1. IAM | 1.2 Authorization | Least Privilege Access | 150 | 50 | 850 | 4 | 75% |
|
||||
| 1. IAM | 1.3 Privilege Escalation | Admin Account Controls | 95 | 5 | 950 | 5 | 95% |
|
||||
| 2. Attack Surface | 2.1 Network | Firewall Configuration | 400 | 100 | 600 | 3 | 80% |
|
||||
| 2. Attack Surface | 2.1 Network | Public Endpoint Security | 80 | 20 | 700 | 4 | 80% |
|
||||
| 2. Attack Surface | 2.2 Storage | Data Classification | 300 | 100 | 650 | 3 | 75% |
|
||||
| 2. Attack Surface | 2.3 Application | Input Validation | 150 | 50 | 500 | 3 | 75% |
|
||||
| 3. Logging and Monitoring | 3.1 Logging | Transaction Logging | 500 | 50 | 750 | 3 | 90.9% |
|
||||
| 3. Logging and Monitoring | 3.3 Monitoring | Real-time Alerts | 200 | 50 | 700 | 4 | 80% |
|
||||
| 4. Encryption | 4.2 At-Rest | Database Encryption | 300 | 20 | 900 | 5 | 93.8% |
|
||||
| 4. Encryption | 4.1 In-Transit | API/Web Encryption | 250 | 10 | 800 | 4 | 96.2% |
|
||||
|
||||
**Pillar Performance Summary**:
|
||||
|
||||
- **1. IAM Pillar Average**: ~87% (weighted by findings across Authentication, Authorization, and Privilege Escalation subpillars)
|
||||
- **2. Attack Surface Pillar Average**: ~77% (weighted across Network, Storage, and Application subpillars)
|
||||
- 2.1 Network subpillar: ~80% average
|
||||
- 2.2 Storage subpillar: 75%
|
||||
- 2.3 Application subpillar: 75%
|
||||
- **3. Logging and Monitoring Average**: ~87% (weighted by findings across Logging and Monitoring subpillars)
|
||||
- **4. Encryption Pillar Average**: ~94% (weighted by findings across In-Transit and At-Rest subpillars)
|
||||
|
||||
**Overall ThreatScore**: ~85.3%
|
||||
|
||||
This comprehensive example demonstrates how:
|
||||
|
||||
- High-performing, high-weight pillars (4. Encryption, 1. IAM) significantly boost the score
|
||||
- The 2. Attack Surface pillar shows how diverse subpillars (Network, Storage, Application) are aggregated
|
||||
- Multiple requirements within pillars provide detailed granular assessment
|
||||
- Cross-pillar balance prevents single points of failure in security posture
|
||||
|
||||
### Example 4: Impact of Parameter Changes
|
||||
|
||||
Using the scenario, let's see how parameter changes affect the score:
|
||||
|
||||
#### Scenario A: Increase Encryption Risk Level
|
||||
|
||||
Change Encryption risk from 5 to 3:
|
||||
|
||||
- **New ThreatScore: 77.8%** (decrease of 3.4 points)
|
||||
- **Impact**: Lower risk weighting reduces the influence of high-performing critical controls
|
||||
|
||||
#### Scenario B: Improve Access Controls Pass Rate
|
||||
|
||||
Change Access Controls from 70% to 90% pass rate:
|
||||
|
||||
- **New ThreatScore: 85.1%** (increase of 3.9 points)
|
||||
- **Impact**: Improving performance on high-weight requirements has significant score impact
|
||||
|
||||
#### Scenario C: Add New Low-Weight Requirement
|
||||
|
||||
Add "Documentation Completeness" (50 PASS, 10 FAIL, weight=100, risk=1):
|
||||
|
||||
- **New ThreatScore: 81.3%** (minimal change of 0.1 points)
|
||||
- **Impact**: Low-weight requirements have minimal impact on overall score
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Edge Cases and Special Conditions
|
||||
|
||||
#### Zero Findings Scenario
|
||||
When a requirement has `total_i = 0` (no findings):
|
||||
|
||||
- **Behavior**: Requirement is completely excluded from calculation
|
||||
- **Rationale**: No evidence means no contribution to confidence in the score
|
||||
- **Impact**: Other requirements receive proportionally more influence
|
||||
|
||||
#### Perfect Score Scenario
|
||||
When all requirements have 100% pass rate:
|
||||
|
||||
- **Result**: ThreatScore = 100%
|
||||
- **Interpretation**: All implemented security checks are passing
|
||||
|
||||
#### Zero Pass Rate Scenario
|
||||
When all requirements have 0% pass rate:
|
||||
|
||||
- **Result**: ThreatScore = 0%
|
||||
- **Interpretation**: Critical security failures across all requirements
|
||||
|
||||
#### Single Requirement Framework
|
||||
For frameworks with only one requirement:
|
||||
|
||||
- **Formula simplification**: ThreatScore = pass_rate × 100
|
||||
- **Impact**: Weight and risk values become irrelevant for score calculation
|
||||
|
||||
### Performance Considerations
|
||||
|
||||
#### Computational Complexity
|
||||
- **Time Complexity**: O(n) where n = number of requirements
|
||||
- **Space Complexity**: O(1) - constant space for accumulation
|
||||
- **Scalability**: Efficiently handles frameworks with thousands of requirements
|
||||
|
||||
#### Calculation Precision
|
||||
- **Floating Point**: Use double precision for intermediate calculations
|
||||
- **Rounding**: Final score rounded to 1 decimal place for display
|
||||
- **Overflow Protection**: Validate that weight × risk × total values don't exceed system limits
|
||||
|
||||
### Data Requirements
|
||||
|
||||
#### Minimum Data Set
|
||||
For each requirement, the following data must be available:
|
||||
|
||||
- **pass_count**: Number of PASS findings (integer ≥ 0)
|
||||
- **fail_count**: Number of FAIL findings (integer ≥ 0)
|
||||
- **weight**: Business importance (integer 1-1000)
|
||||
- **risk**: Risk level (integer 1-5)
|
||||
|
||||
#### Data Validation Rules
|
||||
```
|
||||
total_i = pass_i + fail_i
|
||||
rate_i = pass_i / total_i (when total_i > 0)
|
||||
1 ≤ weight_i ≤ 1000
|
||||
1 ≤ risk_i ≤ 5
|
||||
```
|
||||
|
||||
#### Handling Invalid Data
|
||||
- **Negative values**: Treat as 0 and log warning
|
||||
- **Out-of-range weights/risk**: Clamp to valid range and log warning
|
||||
- **Missing data**: Exclude requirement from calculation and log warning
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Monitoring and Trending
|
||||
|
||||
1. **Establish Baseline**
|
||||
- Record initial ThreatScore after implementing measurement
|
||||
- Set realistic improvement targets based on organizational capacity
|
||||
- Track score changes over time to identify trends
|
||||
|
||||
2. **Regular Reporting**
|
||||
- Generate monthly ThreatScore reports for stakeholders
|
||||
- Highlight significant score changes and their causes
|
||||
- Include requirement-level breakdowns for detailed analysis
|
||||
|
||||
3. **Continuous Improvement**
|
||||
- Use score trends to identify systematic issues
|
||||
- Correlate score changes with security incidents or changes
|
||||
- Adjust weights and risk levels based on lessons learned
|
||||
|
||||
@@ -1,47 +1,211 @@
|
||||
# Github Authentication in Prowler
|
||||
# GitHub Authentication in Prowler
|
||||
|
||||
Prowler supports multiple methods to [authenticate with GitHub](https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api). These include:
|
||||
|
||||
- **Personal Access Token (PAT)**
|
||||
- **OAuth App Token**
|
||||
- **GitHub App Credentials**
|
||||
- [Personal Access Token (PAT)](./authentication.md#personal-access-token-pat)
|
||||
- [OAuth App Token](./authentication.md#oauth-app-token)
|
||||
- [GitHub App Credentials](./authentication.md#github-app-credentials)
|
||||
|
||||
This flexibility enables scanning and analysis of GitHub accounts, including repositories, organizations, and applications, using the method that best suits the use case.
|
||||
|
||||
## Supported Login Methods
|
||||
## Personal Access Token (PAT)
|
||||
|
||||
Here are the available login methods and their respective flags:
|
||||
Personal Access Tokens provide the simplest GitHub authentication method, but it can only access resources owned by a single user or organization.
|
||||
|
||||
### Personal Access Token (PAT)
|
||||
???+ warning "Classic Tokens Deprecated"
|
||||
GitHub has deprecated Personal Access Tokens (classic) in favor of fine-grained Personal Access Tokens. We recommend using fine-grained tokens as they provide better security through more granular permissions and resource-specific access control.
|
||||
|
||||
Use this method by providing your personal access token directly.
|
||||
#### **Option 1: Create a Fine-Grained Personal Access Token (Recommended)**
|
||||
|
||||
```console
|
||||
prowler github --personal-access-token pat
|
||||
```
|
||||
1. **Navigate to GitHub Settings**
|
||||
- Open [GitHub](https://github.com) and sign in
|
||||
- Click the profile picture in the top right corner
|
||||
- Select "Settings" from the dropdown menu
|
||||
|
||||
### OAuth App Token
|
||||
2. **Access Developer Settings**
|
||||
- Scroll down the left sidebar
|
||||
- Click "Developer settings"
|
||||
|
||||
Authenticate using an OAuth app token.
|
||||
3. **Generate Fine-Grained Token**
|
||||
- Click "Personal access tokens"
|
||||
- Select "Fine-grained tokens"
|
||||
- Click "Generate new token"
|
||||
|
||||
```console
|
||||
prowler github --oauth-app-token oauth_token
|
||||
```
|
||||
4. **Configure Token Settings**
|
||||
- **Token name**: Give your token a descriptive name (e.g., "Prowler Security Scanner")
|
||||
- **Expiration**: Set an appropriate expiration date (recommended: 90 days or less)
|
||||
- **Repository access**: Choose "All repositories" or "Only select repositories" based on your needs
|
||||
|
||||
### GitHub App Credentials
|
||||
Use GitHub App credentials by specifying the App ID and the private key path.
|
||||
???+ note "Public repositories"
|
||||
Even if you select 'Only select repositories', the token will have access to the public repositories that you own or are a member of.
|
||||
|
||||
```console
|
||||
prowler github --github-app-id app_id --github-app-key-path app_key_path
|
||||
```
|
||||
5. **Configure Token Permissions**
|
||||
To enable Prowler functionality, configure the following permissions:
|
||||
|
||||
### Automatic Login Method Detection
|
||||
- **Repository permissions:**
|
||||
- **Administration**: Read-only access
|
||||
- **Contents**: Read-only access
|
||||
- **Metadata**: Read-only access
|
||||
- **Pull requests**: Read-only access
|
||||
|
||||
If no login method is explicitly provided, Prowler will automatically attempt to authenticate using environment variables in the following order of precedence:
|
||||
- **Organization permissions:**
|
||||
- **Administration**: Read-only access
|
||||
- **Members**: Read-only access
|
||||
|
||||
1. `GITHUB_PERSONAL_ACCESS_TOKEN`
|
||||
2. `GITHUB_OAUTH_APP_TOKEN`
|
||||
3. `GITHUB_APP_ID` and `GITHUB_APP_KEY` (where the key is the content of the private key file)
|
||||
- **Account permissions:**
|
||||
- **Email addresses**: Read-only access
|
||||
|
||||
6. **Copy and Store the Token**
|
||||
- Copy the generated token immediately (GitHub displays tokens only once)
|
||||
- Store tokens securely using environment variables
|
||||
|
||||

|
||||
|
||||
#### **Option 2: Create a Classic Personal Access Token (Not Recommended)**
|
||||
|
||||
???+ warning "Security Risk"
|
||||
Classic tokens provide broad permissions that may exceed what Prowler actually needs. Use fine-grained tokens instead for better security.
|
||||
|
||||
1. **Navigate to GitHub Settings**
|
||||
- Open [GitHub](https://github.com) and sign in
|
||||
- Click the profile picture in the top right corner
|
||||
- Select "Settings" from the dropdown menu
|
||||
|
||||
2. **Access Developer Settings**
|
||||
- Scroll down the left sidebar
|
||||
- Click "Developer settings"
|
||||
|
||||
3. **Generate Classic Token**
|
||||
- Click "Personal access tokens"
|
||||
- Select "Tokens (classic)"
|
||||
- Click "Generate new token"
|
||||
|
||||
4. **Configure Token Permissions**
|
||||
To enable Prowler functionality, configure the following scopes:
|
||||
- `repo`: Full control of private repositories (includes `repo:status` and `repo:contents`)
|
||||
- `read:org`: Read organization and team membership
|
||||
- `read:user`: Read user profile data
|
||||
- `security_events`: Access security events (secret scanning and Dependabot alerts)
|
||||
- `read:enterprise`: Read enterprise data (if using GitHub Enterprise)
|
||||
|
||||
5. **Copy and Store the Token**
|
||||
- Copy the generated token immediately (GitHub displays tokens only once)
|
||||
- Store tokens securely using environment variables
|
||||
|
||||
## OAuth App Token
|
||||
|
||||
OAuth Apps enable applications to act on behalf of users with explicit consent.
|
||||
|
||||
### Create an OAuth App Token
|
||||
|
||||
1. **Navigate to Developer Settings**
|
||||
- Open GitHub Settings → Developer settings
|
||||
- Click "OAuth Apps"
|
||||
|
||||
2. **Register New Application**
|
||||
- Click "New OAuth App"
|
||||
- Complete the required fields:
|
||||
- **Application name**: Descriptive application name
|
||||
- **Homepage URL**: Application homepage
|
||||
- **Authorization callback URL**: User redirection URL after authorization
|
||||
|
||||
3. **Obtain Authorization Code**
|
||||
- Request authorization code (replace `{app_id}` with the application ID):
|
||||
```
|
||||
https://github.com/login/oauth/authorize?client_id={app_id}
|
||||
```
|
||||
|
||||
4. **Exchange Code for Token**
|
||||
- Exchange authorization code for access token (replace `{app_id}`, `{secret}`, and `{code}`):
|
||||
```
|
||||
https://github.com/login/oauth/access_token?code={code}&client_id={app_id}&client_secret={secret}
|
||||
```
|
||||
|
||||
## GitHub App Credentials
|
||||
GitHub Apps provide the recommended integration method for accessing multiple repositories or organizations.
|
||||
|
||||
### Create a GitHub App
|
||||
|
||||
1. **Navigate to Developer Settings**
|
||||
- Open GitHub Settings → Developer settings
|
||||
- Click "GitHub Apps"
|
||||
|
||||
2. **Create New GitHub App**
|
||||
- Click "New GitHub App"
|
||||
- Complete the required fields:
|
||||
- **GitHub App name**: Unique application name
|
||||
- **Homepage URL**: Application homepage
|
||||
- **Webhook URL**: Webhook payload URL (optional)
|
||||
- **Permissions**: Application permission requirements
|
||||
|
||||
3. **Configure Permissions**
|
||||
To enable Prowler functionality, configure these permissions:
|
||||
- **Repository permissions**:
|
||||
- Contents (Read)
|
||||
- Metadata (Read)
|
||||
- Pull requests (Read)
|
||||
- **Organization permissions**:
|
||||
- Members (Read)
|
||||
- Administration (Read)
|
||||
- **Account permissions**:
|
||||
- Email addresses (Read)
|
||||
|
||||
4. **Where can this GitHub App be installed?**
|
||||
- Select "Any account" to be able to install the GitHub App in any organization.
|
||||
|
||||
5. **Generate Private Key**
|
||||
- Scroll to the "Private keys" section after app creation
|
||||
- Click "Generate a private key"
|
||||
- Download the `.pem` file and store securely
|
||||
|
||||
5. **Record App ID**
|
||||
- Locate the App ID at the top of the GitHub App settings page
|
||||
|
||||
### Install the GitHub App
|
||||
|
||||
1. **Install Application**
|
||||
- Navigate to GitHub App settings
|
||||
- Click "Install App" in the left sidebar
|
||||
- Select the target account/organization
|
||||
- Choose specific repositories or select "All repositories"
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Security Considerations
|
||||
|
||||
Implement the following security measures:
|
||||
|
||||
- **Secure Credential Storage**: Store credentials using environment variables instead of hardcoding tokens
|
||||
- **Secrets Management**: Use dedicated secrets management systems in production environments
|
||||
- **Regular Token Rotation**: Rotate tokens and keys regularly
|
||||
- **Least Privilege Principle**: Grant only minimum required permissions
|
||||
- **Permission Auditing**: Review and audit permissions regularly
|
||||
- **Token Expiration**: Set appropriate expiration times for tokens
|
||||
- **Usage Monitoring**: Monitor token usage and revoke unused tokens
|
||||
|
||||
### Authentication Method Selection
|
||||
|
||||
Choose the appropriate method based on use case:
|
||||
|
||||
- **Personal Access Token**: Individual use, testing, or simple automation
|
||||
- **OAuth App Token**: Applications requiring user consent and delegation
|
||||
- **GitHub App**: Production integrations, especially for organizations
|
||||
|
||||
## Troubleshooting Common Issues
|
||||
|
||||
### Insufficient Permissions
|
||||
- Verify token/app has necessary scopes/permissions
|
||||
- Check organization restrictions on third-party applications
|
||||
|
||||
### Token Expiration
|
||||
- Confirm token has not expired
|
||||
- Verify fine-grained tokens have correct resource access
|
||||
|
||||
### Rate Limiting
|
||||
- GitHub implements API call rate limits
|
||||
- Consider GitHub Apps for higher rate limits
|
||||
|
||||
### Organization Settings
|
||||
- Some organizations restrict third-party applications
|
||||
- Contact organization administrator if access is denied
|
||||
|
||||
???+ note
|
||||
Ensure the corresponding environment variables are set up before running Prowler for automatic detection when not specifying the login method.
|
||||
|
||||
@@ -1,264 +1,90 @@
|
||||
# Getting Started with GitHub
|
||||
|
||||
This guide explains how to set up authentication with GitHub for Prowler. The documentation covers credential retrieval processes for each supported authentication method.
|
||||
## Prowler App
|
||||
|
||||
## Prerequisites
|
||||
<iframe width="560" height="380" src="https://www.youtube-nocookie.com/embed/9ETI84Xpu2g" title="Prowler Cloud Onboarding Github" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen="1"></iframe>
|
||||
|
||||
- GitHub account
|
||||
- Token creation permissions (organization-level access requires admin permissions)
|
||||
> Walkthrough video onboarding a GitHub Account using GitHub App.
|
||||
|
||||
## Authentication Methods
|
||||
### Step 1: Access Prowler Cloud/App
|
||||
|
||||
### 1. Personal Access Token (PAT)
|
||||
1. Navigate to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](../prowler-app.md)
|
||||
2. Go to "Configuration" > "Cloud Providers"
|
||||
|
||||
Personal Access Tokens provide the simplest GitHub authentication method, but it can only access resources owned by a single user or organization.
|
||||

|
||||
|
||||
???+ warning "Classic Tokens Deprecated"
|
||||
GitHub has deprecated Personal Access Tokens (classic) in favor of fine-grained Personal Access Tokens. We recommend using fine-grained tokens as they provide better security through more granular permissions and resource-specific access control.
|
||||
3. Click "Add Cloud Provider"
|
||||
|
||||
#### **Option 1: Create a Fine-Grained Personal Access Token (Recommended)**
|
||||

|
||||
|
||||
1. **Navigate to GitHub Settings**
|
||||
- Open [GitHub](https://github.com) and sign in
|
||||
- Click the profile picture in the top right corner
|
||||
- Select "Settings" from the dropdown menu
|
||||
4. Select "GitHub"
|
||||
|
||||
2. **Access Developer Settings**
|
||||
- Scroll down the left sidebar
|
||||
- Click "Developer settings"
|
||||

|
||||
|
||||
3. **Generate Fine-Grained Token**
|
||||
- Click "Personal access tokens"
|
||||
- Select "Fine-grained tokens"
|
||||
- Click "Generate new token"
|
||||
5. Add the GitHub Account ID (username or organization name) and an optional alias, then click "Next"
|
||||
|
||||
4. **Configure Token Settings**
|
||||
- **Token name**: Give your token a descriptive name (e.g., "Prowler Security Scanner")
|
||||
- **Expiration**: Set an appropriate expiration date (recommended: 90 days or less)
|
||||
- **Repository access**: Choose "All repositories" or "Only select repositories" based on your needs
|
||||

|
||||
|
||||
???+ note "Public repositories"
|
||||
Even if you select 'Only select repositories', the token will have access to the public repositories that you own or are a member of.
|
||||
### Step 2: Choose the preferred authentication method
|
||||
|
||||
5. **Configure Token Permissions**
|
||||
To enable Prowler functionality, configure the following permissions:
|
||||
6. Choose the preferred authentication method:
|
||||
|
||||
- **Repository permissions:**
|
||||
- **Administration**: Read-only access
|
||||
- **Contents**: Read-only access
|
||||
- **Metadata**: Read-only access
|
||||
- **Pull requests**: Read-only access
|
||||

|
||||
|
||||
- **Organization permissions:**
|
||||
- **Administration**: Read-only access
|
||||
- **Members**: Read-only access
|
||||
7. Configure the authentication method:
|
||||
|
||||
- **Account permissions:**
|
||||
- **Email addresses**: Read-only access
|
||||
=== "Personal Access Token"
|
||||

|
||||
|
||||
6. **Copy and Store the Token**
|
||||
- Copy the generated token immediately (GitHub displays tokens only once)
|
||||
- Store tokens securely using environment variables
|
||||
For more details on how to create a Personal Access Token, see [Authentication > Personal Access Token](./authentication.md#personal-access-token-pat).
|
||||
|
||||

|
||||
=== "OAuth App Token"
|
||||
|
||||
#### **Option 2: Create a Classic Personal Access Token (Not Recommended)**
|
||||

|
||||
|
||||
???+ warning "Security Risk"
|
||||
Classic tokens provide broad permissions that may exceed what Prowler actually needs. Use fine-grained tokens instead for better security.
|
||||
For more details on how to create an OAuth App Token, see [Authentication > OAuth App Token](./authentication.md#oauth-app-token).
|
||||
|
||||
1. **Navigate to GitHub Settings**
|
||||
- Open [GitHub](https://github.com) and sign in
|
||||
- Click the profile picture in the top right corner
|
||||
- Select "Settings" from the dropdown menu
|
||||
=== "GitHub App"
|
||||
|
||||
2. **Access Developer Settings**
|
||||
- Scroll down the left sidebar
|
||||
- Click "Developer settings"
|
||||

|
||||
|
||||
3. **Generate Classic Token**
|
||||
- Click "Personal access tokens"
|
||||
- Select "Tokens (classic)"
|
||||
- Click "Generate new token"
|
||||
For more details on how to create a GitHub App, see [Authentication > GitHub App](./authentication.md#github-app-credentials).
|
||||
|
||||
4. **Configure Token Permissions**
|
||||
To enable Prowler functionality, configure the following scopes:
|
||||
- `repo`: Full control of private repositories (includes `repo:status` and `repo:contents`)
|
||||
- `read:org`: Read organization and team membership
|
||||
- `read:user`: Read user profile data
|
||||
- `security_events`: Access security events (secret scanning and Dependabot alerts)
|
||||
- `read:enterprise`: Read enterprise data (if using GitHub Enterprise)
|
||||
|
||||
5. **Copy and Store the Token**
|
||||
- Copy the generated token immediately (GitHub displays tokens only once)
|
||||
- Store tokens securely using environment variables
|
||||
## Prowler CLI
|
||||
|
||||
#### How to Use Personal Access Tokens
|
||||
### Automatic Login Method Detection
|
||||
|
||||
Choose one of the following methods:
|
||||
If no login method is explicitly provided, Prowler will automatically attempt to authenticate using environment variables in the following order of precedence:
|
||||
|
||||
**Command-line flag:**
|
||||
1. `GITHUB_PERSONAL_ACCESS_TOKEN`
|
||||
2. `GITHUB_OAUTH_APP_TOKEN`
|
||||
3. `GITHUB_APP_ID` and `GITHUB_APP_KEY` (where the key is the content of the private key file)
|
||||
|
||||
???+ note
|
||||
Ensure the corresponding environment variables are set up before running Prowler for automatic detection when not specifying the login method.
|
||||
|
||||
For more details on how to set up authentication with GitHub, see [Authentication > GitHub](./authentication.md).
|
||||
|
||||
### Personal Access Token (PAT)
|
||||
|
||||
Use this method by providing your personal access token directly.
|
||||
|
||||
```console
|
||||
prowler github --personal-access-token your_token_here
|
||||
prowler github --personal-access-token pat
|
||||
```
|
||||
|
||||
**Environment variable:**
|
||||
### OAuth App Token
|
||||
|
||||
Authenticate using an OAuth app token.
|
||||
|
||||
```console
|
||||
export GITHUB_PERSONAL_ACCESS_TOKEN="your_token_here"
|
||||
prowler github
|
||||
prowler github --oauth-app-token oauth_token
|
||||
```
|
||||
|
||||
### 2. OAuth App Token
|
||||
|
||||
OAuth Apps enable applications to act on behalf of users with explicit consent.
|
||||
|
||||
#### How to Create an OAuth App
|
||||
|
||||
1. **Navigate to Developer Settings**
|
||||
- Open GitHub Settings → Developer settings
|
||||
- Click "OAuth Apps"
|
||||
|
||||
2. **Register New Application**
|
||||
- Click "New OAuth App"
|
||||
- Complete the required fields:
|
||||
- **Application name**: Descriptive application name
|
||||
- **Homepage URL**: Application homepage
|
||||
- **Authorization callback URL**: User redirection URL after authorization
|
||||
|
||||
3. **Obtain Authorization Code**
|
||||
- Request authorization code (replace `{app_id}` with the application ID):
|
||||
```
|
||||
https://github.com/login/oauth/authorize?client_id={app_id}
|
||||
```
|
||||
|
||||
4. **Exchange Code for Token**
|
||||
- Exchange authorization code for access token (replace `{app_id}`, `{secret}`, and `{code}`):
|
||||
```
|
||||
https://github.com/login/oauth/access_token?code={code}&client_id={app_id}&client_secret={secret}
|
||||
```
|
||||
|
||||
#### How to Use OAuth Tokens
|
||||
|
||||
Choose one of the following methods:
|
||||
|
||||
**Command-line flag:**
|
||||
### GitHub App Credentials
|
||||
Use GitHub App credentials by specifying the App ID and the private key path.
|
||||
|
||||
```console
|
||||
prowler github --oauth-app-token your_oauth_token
|
||||
prowler github --github-app-id app_id --github-app-key-path app_key_path
|
||||
```
|
||||
|
||||
**Environment variable:**
|
||||
|
||||
```console
|
||||
export GITHUB_OAUTH_APP_TOKEN="your_oauth_token"
|
||||
prowler github
|
||||
```
|
||||
|
||||
### 3. GitHub App Credentials
|
||||
|
||||
GitHub Apps provide the recommended integration method for accessing multiple repositories or organizations.
|
||||
|
||||
#### How to Create a GitHub App
|
||||
|
||||
1. **Navigate to Developer Settings**
|
||||
- Open GitHub Settings → Developer settings
|
||||
- Click "GitHub Apps"
|
||||
|
||||
2. **Create New GitHub App**
|
||||
- Click "New GitHub App"
|
||||
- Complete the required fields:
|
||||
- **GitHub App name**: Unique application name
|
||||
- **Homepage URL**: Application homepage
|
||||
- **Webhook URL**: Webhook payload URL (optional)
|
||||
- **Permissions**: Application permission requirements
|
||||
|
||||
3. **Configure Permissions**
|
||||
To enable Prowler functionality, configure these permissions:
|
||||
- **Repository permissions**:
|
||||
- Contents (Read)
|
||||
- Metadata (Read)
|
||||
- Pull requests (Read)
|
||||
- **Organization permissions**:
|
||||
- Members (Read)
|
||||
- Administration (Read)
|
||||
- **Account permissions**:
|
||||
- Email addresses (Read)
|
||||
|
||||
4. **Where can this GitHub App be installed?**
|
||||
- Select "Any account" to be able to install the GitHub App in any organization.
|
||||
|
||||
5. **Generate Private Key**
|
||||
- Scroll to the "Private keys" section after app creation
|
||||
- Click "Generate a private key"
|
||||
- Download the `.pem` file and store securely
|
||||
|
||||
5. **Record App ID**
|
||||
- Locate the App ID at the top of the GitHub App settings page
|
||||
|
||||
#### How to Install the GitHub App
|
||||
|
||||
1. **Install Application**
|
||||
- Navigate to GitHub App settings
|
||||
- Click "Install App" in the left sidebar
|
||||
- Select the target account/organization
|
||||
- Choose specific repositories or select "All repositories"
|
||||
|
||||
#### How to Use GitHub App Credentials
|
||||
|
||||
Choose one of the following methods:
|
||||
|
||||
**Command-line flags:**
|
||||
|
||||
```console
|
||||
prowler github --github-app-id your_app_id --github-app-key /path/to/private-key.pem
|
||||
```
|
||||
|
||||
**Environment variables:**
|
||||
|
||||
```console
|
||||
export GITHUB_APP_ID="your_app_id"
|
||||
export GITHUB_APP_KEY="private-key-content"
|
||||
prowler github
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Security Considerations
|
||||
|
||||
Implement the following security measures:
|
||||
|
||||
- **Secure Credential Storage**: Store credentials using environment variables instead of hardcoding tokens
|
||||
- **Secrets Management**: Use dedicated secrets management systems in production environments
|
||||
- **Regular Token Rotation**: Rotate tokens and keys regularly
|
||||
- **Least Privilege Principle**: Grant only minimum required permissions
|
||||
- **Permission Auditing**: Review and audit permissions regularly
|
||||
- **Token Expiration**: Set appropriate expiration times for tokens
|
||||
- **Usage Monitoring**: Monitor token usage and revoke unused tokens
|
||||
|
||||
### Authentication Method Selection
|
||||
|
||||
Choose the appropriate method based on use case:
|
||||
|
||||
- **Personal Access Token**: Individual use, testing, or simple automation
|
||||
- **OAuth App Token**: Applications requiring user consent and delegation
|
||||
- **GitHub App**: Production integrations, especially for organizations
|
||||
|
||||
## Troubleshooting Common Issues
|
||||
|
||||
### Insufficient Permissions
|
||||
- Verify token/app has necessary scopes/permissions
|
||||
- Check organization restrictions on third-party applications
|
||||
|
||||
### Token Expiration
|
||||
- Confirm token has not expired
|
||||
- Verify fine-grained tokens have correct resource access
|
||||
|
||||
### Rate Limiting
|
||||
- GitHub implements API call rate limits
|
||||
- Consider GitHub Apps for higher rate limits
|
||||
|
||||
### Organization Settings
|
||||
- Some organizations restrict third-party applications
|
||||
- Contact organization administrator if access is denied
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 96 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 108 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 100 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 102 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 91 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 100 KiB |
@@ -1,10 +1,10 @@
|
||||
# Getting Started with the IaC Provider
|
||||
|
||||
Prowler's Infrastructure as Code (IaC) provider enables you to scan local or remote infrastructure code for security and compliance issues using [Trivy](https://trivy.dev/). This provider supports a wide range of IaC frameworks, allowing you to assess your code before deployment.
|
||||
Prowler's Infrastructure as Code (IaC) provider enables scanning of local or remote infrastructure code for security and compliance issues using [Trivy](https://trivy.dev/). This provider supports a wide range of IaC frameworks, allowing assessment of code before deployment.
|
||||
|
||||
## Supported Scanners
|
||||
|
||||
The IaC provider leverages Trivy to support multiple scanners, including:
|
||||
The IaC provider leverages [Trivy](https://trivy.dev/latest/docs/scanner/vulnerability/) to support multiple scanners, including:
|
||||
|
||||
- Vulnerability
|
||||
- Misconfiguration
|
||||
@@ -13,31 +13,34 @@ The IaC provider leverages Trivy to support multiple scanners, including:
|
||||
|
||||
## How It Works
|
||||
|
||||
- The IaC provider scans your local directory (or a specified path) for supported IaC files, or scan a remote repository.
|
||||
- The IaC provider scans local directories (or specified paths) for supported IaC files, or scans remote repositories.
|
||||
- No cloud credentials or authentication are required for local scans.
|
||||
- For remote repository scans, authentication can be provided via [git URL](https://git-scm.com/docs/git-clone#_git_urls), CLI flags or environment variables.
|
||||
- Check the [IaC Authentication](./authentication.md) page for more details.
|
||||
- Mutelist logic is handled by Trivy, not Prowler.
|
||||
- Results are output in the same formats as other Prowler providers (CSV, JSON, HTML, etc.).
|
||||
|
||||
## Usage
|
||||
## Prowler CLI
|
||||
|
||||
To run Prowler with the IaC provider, use the `iac` argument. You can specify the directory or repository to scan, frameworks to include, and paths to exclude.
|
||||
### Usage
|
||||
|
||||
### Scan a Local Directory (default)
|
||||
Use the `iac` argument to run Prowler with the IaC provider. Specify the directory or repository to scan, frameworks to include, and paths to exclude.
|
||||
|
||||
#### Scan a Local Directory (default)
|
||||
|
||||
```sh
|
||||
prowler iac --scan-path ./my-iac-directory
|
||||
```
|
||||
|
||||
### Scan a Remote GitHub Repository
|
||||
#### Scan a Remote GitHub Repository
|
||||
|
||||
```sh
|
||||
prowler iac --scan-repository-url https://github.com/user/repo.git
|
||||
```
|
||||
|
||||
#### Authentication for Remote Private Repositories
|
||||
##### Authentication for Remote Private Repositories
|
||||
|
||||
You can provide authentication for private repositories using one of the following methods:
|
||||
Authentication for private repositories can be provided using one of the following methods:
|
||||
|
||||
- **GitHub Username and Personal Access Token (PAT):**
|
||||
```sh
|
||||
@@ -52,12 +55,12 @@ You can provide authentication for private repositories using one of the followi
|
||||
- If not provided via CLI, the following environment variables will be used (in order of precedence):
|
||||
- `GITHUB_OAUTH_APP_TOKEN`
|
||||
- `GITHUB_USERNAME` and `GITHUB_PERSONAL_ACCESS_TOKEN`
|
||||
- If neither CLI flags nor environment variables are set, the scan will attempt to clone without authentication or using the provided in the [git URL](https://git-scm.com/docs/git-clone#_git_urls).
|
||||
- If neither CLI flags nor environment variables are set, the scan will attempt to clone without authentication or using the credentials provided in the [git URL](https://git-scm.com/docs/git-clone#_git_urls).
|
||||
|
||||
#### Mutually Exclusive Flags
|
||||
##### Mutually Exclusive Flags
|
||||
- `--scan-path` and `--scan-repository-url` are mutually exclusive. Only one can be specified at a time.
|
||||
|
||||
### Specify Scanners
|
||||
#### Specify Scanners
|
||||
|
||||
Scan only vulnerability and misconfiguration scanners:
|
||||
|
||||
@@ -65,24 +68,16 @@ Scan only vulnerability and misconfiguration scanners:
|
||||
prowler iac --scan-path ./my-iac-directory --scanners vuln misconfig
|
||||
```
|
||||
|
||||
### Exclude Paths
|
||||
#### Exclude Paths
|
||||
|
||||
```sh
|
||||
prowler iac --scan-path ./my-iac-directory --exclude-path ./my-iac-directory/test,./my-iac-directory/examples
|
||||
```
|
||||
|
||||
## Output
|
||||
### Output
|
||||
|
||||
You can use the standard Prowler output options, for example:
|
||||
Use the standard Prowler output options, for example:
|
||||
|
||||
```sh
|
||||
prowler iac --scan-path ./iac --output-formats csv json html
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- The IaC provider does not require cloud authentication for local scans.
|
||||
- For remote repository scans, authentication is optional but required for private repos.
|
||||
- CLI flags override environment variables for authentication.
|
||||
- It is ideal for CI/CD pipelines and local development environments.
|
||||
- For more details on supported scanners, see the [Trivy documentation](https://trivy.dev/latest/docs/scanner/vulnerability/).
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
# Getting Started With LLM on Prowler
|
||||
|
||||
## Overview
|
||||
|
||||
Prowler's LLM provider enables comprehensive security testing of large language models using red team techniques. It integrates with [promptfoo](https://promptfoo.dev/) to provide extensive security evaluation capabilities.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before using the LLM provider, ensure the following requirements are met:
|
||||
|
||||
- **promptfoo installed**: The LLM provider requires promptfoo to be installed on the system
|
||||
- **LLM API access**: Valid API keys for the target LLM models to test
|
||||
- **Email verification**: promptfoo requires email verification for red team evaluations
|
||||
|
||||
## Installation
|
||||
|
||||
### Install promptfoo
|
||||
|
||||
Install promptfoo using one of the following methods:
|
||||
|
||||
**Using npm:**
|
||||
```bash
|
||||
npm install -g promptfoo
|
||||
```
|
||||
|
||||
**Using Homebrew (macOS):**
|
||||
```bash
|
||||
brew install promptfoo
|
||||
```
|
||||
|
||||
**Using other package managers:**
|
||||
See the [promptfoo installation guide](https://promptfoo.dev/docs/installation/) for additional installation methods.
|
||||
|
||||
### Verify Installation
|
||||
|
||||
```bash
|
||||
promptfoo --version
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Step 1: Email Verification
|
||||
|
||||
promptfoo requires email verification for red team evaluations. Set the email address:
|
||||
|
||||
```bash
|
||||
promptfoo config set email your-email@company.com
|
||||
```
|
||||
|
||||
### Step 2: Configure LLM API Keys
|
||||
|
||||
Set up API keys for the target LLM models. For OpenAI (default configuration):
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY="your-openai-api-key"
|
||||
```
|
||||
|
||||
For other providers, see the [promptfoo documentation](https://promptfoo.dev/docs/providers/) for specific configuration requirements.
|
||||
|
||||
### Step 3: Generate Test Cases (Optional)
|
||||
|
||||
Prowler provides a default suite of red team tests but to customize the test cases, generate them first:
|
||||
|
||||
```bash
|
||||
promptfoo redteam generate
|
||||
```
|
||||
|
||||
This creates test cases based on your configuration.
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic Usage
|
||||
|
||||
Run LLM security testing with the default configuration:
|
||||
|
||||
```bash
|
||||
prowler llm
|
||||
```
|
||||
|
||||
### Custom Configuration
|
||||
|
||||
Use a custom promptfoo configuration file:
|
||||
|
||||
```bash
|
||||
prowler llm --config-path /path/to/your/config.yaml
|
||||
```
|
||||
|
||||
### Output Options
|
||||
|
||||
Generate reports in various formats:
|
||||
|
||||
```bash
|
||||
# JSON output
|
||||
prowler llm --output-format json
|
||||
|
||||
# CSV output
|
||||
prowler llm --output-format csv
|
||||
|
||||
# HTML report
|
||||
prowler llm --output-format html
|
||||
```
|
||||
|
||||
### Concurrency Control
|
||||
|
||||
Adjust the number of concurrent tests:
|
||||
|
||||
```bash
|
||||
prowler llm --max-concurrency 5
|
||||
```
|
||||
|
||||
## Default Configuration
|
||||
|
||||
Prowler includes a comprehensive default LLM configuration that provides:
|
||||
|
||||
- **Target Models**: OpenAI GPT models by default
|
||||
- **Security Frameworks**:
|
||||
- OWASP LLM Top 10
|
||||
- OWASP API Top 10
|
||||
- MITRE ATLAS
|
||||
- NIST AI Risk Management Framework
|
||||
- EU AI Act compliance
|
||||
- **Test Coverage**: Over 5,000 security test cases
|
||||
- **Plugin Support**: Multiple security testing plugins
|
||||
|
||||
## Advanced Configuration
|
||||
|
||||
### Custom Test Suites
|
||||
|
||||
Create custom test configurations by modifying the promptfoo config file in `prowler/config/llm_config.yaml` or pass a custom configuration with `--config-file` flag:
|
||||
|
||||
```yaml
|
||||
description: Custom LLM Security Tests
|
||||
targets:
|
||||
- id: openai:gpt-4
|
||||
redteam:
|
||||
plugins:
|
||||
- id: owasp:llm
|
||||
numTests: 10
|
||||
- id: mitre:atlas
|
||||
numTests: 5
|
||||
```
|
||||
|
||||
@@ -1,20 +1,138 @@
|
||||
# Microsoft 365 Authentication for Prowler
|
||||
# Microsoft 365 Authentication in Prowler
|
||||
|
||||
Prowler for Microsoft 365 (M365) supports the following authentication methods:
|
||||
Prowler for Microsoft 365 supports multiple authentication types. Authentication methods vary between Prowler App and Prowler CLI:
|
||||
|
||||
- [**Service Principal Application**](https://learn.microsoft.com/en-us/entra/identity-platform/app-objects-and-service-principals?tabs=browser#service-principal-object) (**Recommended**)
|
||||
- **Service Principal Application with Microsoft User Credentials**
|
||||
- **Stored AZ CLI credentials**
|
||||
- **Interactive browser authentication**
|
||||
**Prowler App:**
|
||||
|
||||
- [**Service Principal Application**](#service-principal-authentication-recommended) (**Recommended**)
|
||||
- [**Service Principal with User Credentials**](#service-principal-and-user-credentials-authentication) (Deprecated)
|
||||
|
||||
**Prowler CLI:**
|
||||
|
||||
- [**Service Principal Application**](#service-principal-authentication-recommended) (**Recommended**)
|
||||
- [**Interactive browser authentication**](#interactive-browser-authentication)
|
||||
|
||||
## Required Permissions
|
||||
|
||||
To run the full Prowler provider, including PowerShell checks, two types of permission scopes must be set in **Microsoft Entra ID**.
|
||||
|
||||
### Service Principal Authentication Permissions (Recommended)
|
||||
|
||||
When using service principal authentication, add these **Application Permissions**:
|
||||
|
||||
**Microsoft Graph API Permissions:**
|
||||
|
||||
- `AuditLog.Read.All`: Required for Entra service.
|
||||
- `Directory.Read.All`: Required for all services.
|
||||
- `Policy.Read.All`: Required for all services.
|
||||
- `SharePointTenantSettings.Read.All`: Required for SharePoint service.
|
||||
|
||||
**External API Permissions:**
|
||||
|
||||
- `Exchange.ManageAsApp` from external API `Office 365 Exchange Online`: Required for Exchange PowerShell module app authentication. The `Global Reader` role must also be assigned to the app.
|
||||
- `application_access` from external API `Skype and Teams Tenant Admin API`: Required for Teams PowerShell module app authentication.
|
||||
|
||||
???+ note
|
||||
`Directory.Read.All` can be replaced with `Domain.Read.All` for more restrictive permissions, but Entra checks related to DirectoryRoles and GetUsers will not run. If using this option, you must also add the `Organization.Read.All` permission to the service principal application for authentication.
|
||||
|
||||
???+ note
|
||||
This is the **recommended authentication method** because it allows running the full M365 provider including PowerShell checks, providing complete coverage of all available security checks.
|
||||
|
||||
### Browser Authentication Permissions
|
||||
|
||||
When using browser authentication, permissions are delegated to the user, so the user must have the appropriate permissions rather than the application.
|
||||
|
||||
???+ warning
|
||||
Prowler App supports the **Service Principal** authentication method and the **Service Principal with User Credentials** authentication method, but this last one will be deprecated in October once Microsoft will enforce MFA in all tenants not allowing User authentication without interactive method.
|
||||
With browser authentication, you will only be able to run checks that work through MS Graph API. PowerShell module checks will not be executed.
|
||||
|
||||
### Service Principal Authentication (Recommended)
|
||||
### Step-by-Step Permission Assignment
|
||||
|
||||
**Authentication flag:** `--sp-env-auth`
|
||||
#### Create Service Principal Application
|
||||
|
||||
Enable Prowler authentication as the **Service Principal Application** by configuring the following environment variables:
|
||||
1. Access **Microsoft Entra ID**
|
||||
|
||||

|
||||
|
||||
2. Navigate to "Applications" > "App registrations"
|
||||
|
||||

|
||||
|
||||
3. Click "+ New registration", complete the form, and click "Register"
|
||||
|
||||

|
||||
|
||||
4. Go to "Certificates & secrets" > "Client secrets" > "+ New client secret"
|
||||
|
||||

|
||||
|
||||
5. Fill in the required fields and click "Add", then copy the generated value (this will be `AZURE_CLIENT_SECRET`)
|
||||
|
||||

|
||||
|
||||
#### Grant Microsoft Graph API Permissions
|
||||
|
||||
1. Go to App Registration > Select your Prowler App > click on "API permissions"
|
||||
|
||||

|
||||
|
||||
2. Click "+ Add a permission" > "Microsoft Graph" > "Application permissions"
|
||||
|
||||

|
||||
|
||||
3. Search and select the required permissions:
|
||||
- `AuditLog.Read.All`: Required for Entra service
|
||||
- `Directory.Read.All`: Required for all services
|
||||
- `Policy.Read.All`: Required for all services
|
||||
- `SharePointTenantSettings.Read.All`: Required for SharePoint service
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
4. Click "Add permissions", then click "Grant admin consent for <your-tenant-name>"
|
||||
|
||||
#### Grant PowerShell Module Permissions (For Service Principal Authentication)
|
||||
|
||||
1. **Add Exchange API:**
|
||||
|
||||
- Search and select "Office 365 Exchange Online" API in **APIs my organization uses**
|
||||
|
||||

|
||||
|
||||
- Select "Exchange.ManageAsApp" permission and click "Add permissions"
|
||||
|
||||

|
||||
|
||||
- Assign `Global Reader` role to the app: Go to `Roles and administrators` > click `here` for directory level assignment
|
||||
|
||||

|
||||
|
||||
- Search for `Global Reader` and assign it to your application
|
||||
|
||||

|
||||
|
||||
2. **Add Teams API:**
|
||||
|
||||
- Search and select "Skype and Teams Tenant Admin API" in **APIs my organization uses**
|
||||
|
||||

|
||||
|
||||
- Select "application_access" permission and click "Add permissions"
|
||||
|
||||

|
||||
|
||||
3. Click "Grant admin consent for <your-tenant-name>" to grant admin consent
|
||||
|
||||

|
||||
|
||||
|
||||
## Service Principal Authentication (Recommended)
|
||||
|
||||
*Available for both Prowler App and Prowler CLI*
|
||||
|
||||
**Authentication flag for CLI:** `--sp-env-auth`
|
||||
|
||||
Authenticate using the **Service Principal Application** by configuring the following environment variables:
|
||||
|
||||
```console
|
||||
export AZURE_CLIENT_ID="XXXXXXXXX"
|
||||
@@ -24,122 +142,27 @@ export AZURE_TENANT_ID="XXXXXXXXX"
|
||||
|
||||
If these variables are not set or exported, execution using `--sp-env-auth` will fail.
|
||||
|
||||
Refer to the [Create Prowler Service Principal](getting-started-m365.md#create-the-service-principal-app) guide for setup instructions.
|
||||
Refer to the [Step-by-Step Permission Assignment](#step-by-step-permission-assignment) section below for setup instructions.
|
||||
|
||||
If the external API permissions described in the mentioned section above are not added only checks that work through MS Graph will be executed. This means that the full provider will not be executed.
|
||||
|
||||
???+ note
|
||||
In order to scan all the checks from M365 required permissions to the service principal application must be added. Refer to the [External API Permissions Assignment](getting-started-m365.md#grant-powershell-modules-permissions) section for more information.
|
||||
|
||||
### Service Principal and User Credentials Authentication
|
||||
|
||||
Authentication flag: `--env-auth`
|
||||
|
||||
???+ warning
|
||||
This method is not recommended anymore, we recommend just use the **Service Principal Application** authentication method instead.
|
||||
|
||||
This method builds upon the Service Principal authentication by adding User Credentials. Configure the following environment variables: `M365_USER` and `M365_PASSWORD`.
|
||||
|
||||
```console
|
||||
export AZURE_CLIENT_ID="XXXXXXXXX"
|
||||
export AZURE_CLIENT_SECRET="XXXXXXXXX"
|
||||
export AZURE_TENANT_ID="XXXXXXXXX"
|
||||
export M365_USER="your_email@example.com"
|
||||
export M365_PASSWORD="examplepassword"
|
||||
```
|
||||
|
||||
These two new environment variables are **required** in this authentication method to execute the PowerShell modules needed to retrieve information from M365 services. Prowler uses Service Principal authentication to access Microsoft Graph and user credentials to authenticate to Microsoft PowerShell modules.
|
||||
|
||||
- `M365_USER` should be your Microsoft account email using the **assigned domain in the tenant**. This means it must look like `example@YourCompany.onmicrosoft.com` or `example@YourCompany.com`, but it must be the exact domain assigned to that user in the tenant.
|
||||
|
||||
???+ warning
|
||||
Newly created users must sign in with the account first, as Microsoft prompts for password change. Without completing this step, user authentication fails because Microsoft marks the initial password as expired.
|
||||
|
||||
???+ warning
|
||||
The user must not be MFA capable. Microsoft does not allow MFA capable users to authenticate programmatically. See [Microsoft documentation](https://learn.microsoft.com/en-us/entra/identity-platform/scenario-desktop-acquire-token-username-password?tabs=dotnet) for more information.
|
||||
|
||||
???+ warning
|
||||
Using a tenant domain other than the one assigned — even if it belongs to the same tenant — will cause Prowler to fail, as Microsoft authentication will not succeed.
|
||||
|
||||
Ensure the correct domain is used for the authenticating user.
|
||||
|
||||

|
||||
|
||||
- `M365_PASSWORD` must be the user password.
|
||||
|
||||
???+ note
|
||||
Previously an encrypted password was required, but now the user password is accepted directly. Prowler handles the password encryption.
|
||||
In order to scan all the checks from M365 required permissions to the service principal application must be added. Refer to the [PowerShell Module Permissions](#grant-powershell-module-permissions-for-service-principal-authentication) section for more information.
|
||||
|
||||
|
||||
## Interactive Browser Authentication
|
||||
|
||||
### Interactive Browser Authentication
|
||||
*Available only for Prowler CLI*
|
||||
|
||||
**Authentication flag:** `--browser-auth`
|
||||
|
||||
This authentication method requires authentication against Azure using the default browser to start the scan. The `--tenant-id` flag is also required.
|
||||
Authenticate against Azure using the default browser to start the scan. The `--tenant-id` flag is also required.
|
||||
|
||||
These credentials only enable checks that rely on Microsoft Graph. The entire provider cannot be run with this method. To perform a full M365 security scan, use the **recommended authentication method**.
|
||||
|
||||
Since this is a **delegated permission** authentication method, necessary permissions should be assigned to the user rather than the application.
|
||||
|
||||
### Required Permissions
|
||||
|
||||
To run the full Prowler provider, including PowerShell checks, two types of permission scopes must be set in **Microsoft Entra ID**.
|
||||
|
||||
#### Service Principal Authentication (`--sp-env-auth`) - Recommended
|
||||
|
||||
When using service principal authentication, add the following **Application Permissions**:
|
||||
|
||||
**Microsoft Graph API Permissions:**
|
||||
|
||||
- `AuditLog.Read.All`: Required for Entra service.
|
||||
- `Directory.Read.All`: Required for all services.
|
||||
- `Policy.Read.All`: Required for all services.
|
||||
- `SharePointTenantSettings.Read.All`: Required for SharePoint service.
|
||||
- `User.Read` (IMPORTANT: this must be set as **delegated**): Required for the sign-in.
|
||||
|
||||
**External API Permissions:**
|
||||
|
||||
- `Exchange.ManageAsApp` from external API `Office 365 Exchange Online`: Required for Exchange PowerShell module app authentication. You also need to assign the `Global Reader` role to the app.
|
||||
- `application_access` from external API `Skype and Teams Tenant Admin API`: Required for Teams PowerShell module app authentication.
|
||||
|
||||
???+ note
|
||||
`Directory.Read.All` can be replaced with `Domain.Read.All` that is a more restrictive permission but you won't be able to run the Entra checks related with DirectoryRoles and GetUsers.
|
||||
|
||||
> If you do this you will need to add also the `Organization.Read.All` permission to the service principal application in order to authenticate.
|
||||
|
||||
???+ note
|
||||
This is the **recommended authentication method** because it allows you to run the full M365 provider including PowerShell checks, providing complete coverage of all available security checks, same as the Service Principal Authentication + User Credentials Authentication but this last one will be deprecated in October once Microsoft will enforce MFA in all tenants not allowing User authentication without interactive method.
|
||||
|
||||
|
||||
#### Service Principal + User Credentials Authentication (`--env-auth`)
|
||||
|
||||
When using service principal with user credentials authentication, you need **both** sets of permissions:
|
||||
|
||||
**1. Service Principal Application Permissions**:
|
||||
- You **will need** all the Microsoft Graph API permissions listed above.
|
||||
- You **won't need** the External API permissions listed above.
|
||||
|
||||
**2. User-Level Permissions**: These are set at the `M365_USER` level, so the user used to run Prowler must have one of the following roles:
|
||||
|
||||
- `Global Reader` (recommended): this allows you to read all roles needed.
|
||||
- `Exchange Administrator` and `Teams Administrator`: user needs both roles but with this [roles](https://learn.microsoft.com/en-us/exchange/permissions-exo/permissions-exo#microsoft-365-permissions-in-exchange-online) you can access to the same information as a Global Reader (since only read access is needed, Global Reader is recommended).
|
||||
|
||||
|
||||
#### Browser Authentication (`--browser-auth`)
|
||||
|
||||
When using browser authentication, permissions are delegated to the user, so the user must have the appropriate permissions rather than the application.
|
||||
|
||||
???+ warning
|
||||
With browser authentication, you will only be able to run checks that work through MS Graph API. PowerShell module checks will not be executed.
|
||||
|
||||
### Assigning Permissions and Roles
|
||||
|
||||
For guidance on assigning the necessary permissions and roles, follow these instructions:
|
||||
- [Grant API Permissions](getting-started-m365.md#grant-required-graph-api-permissions)
|
||||
- [Assign Required Roles](getting-started-m365.md#if-using-user-authentication)
|
||||
|
||||
### Supported PowerShell Versions
|
||||
## Supported PowerShell Versions
|
||||
|
||||
PowerShell is required to run certain M365 checks.
|
||||
|
||||
@@ -156,26 +179,32 @@ PowerShell is required to run certain M365 checks.
|
||||
|
||||
### Installing PowerShell
|
||||
|
||||
Installing PowerShell is different depending on your OS.
|
||||
Installing PowerShell is different depending on your OS:
|
||||
|
||||
- [Windows](https://learn.microsoft.com/es-es/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.5#install-powershell-using-winget-recommended): you will need to update PowerShell to +7.4 to be able to run prowler, if not some checks will not show findings and the provider could not work as expected. This version of PowerShell is [supported](https://learn.microsoft.com/es-es/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.4#supported-versions-of-windows) on Windows 10, Windows 11, Windows Server 2016 and higher versions.
|
||||
=== "Windows"
|
||||
|
||||
```console
|
||||
winget install --id Microsoft.PowerShell --source winget
|
||||
```
|
||||
[Windows](https://learn.microsoft.com/es-es/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.5#install-powershell-using-winget-recommended): PowerShell must be updated to version 7.4+ for Prowler to function properly. Otherwise, some checks will not show findings and the provider may not function properly. This version of PowerShell is [supported](https://learn.microsoft.com/es-es/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.4#supported-versions-of-windows) on Windows 10, Windows 11, Windows Server 2016 and higher versions.
|
||||
|
||||
```console
|
||||
winget install --id Microsoft.PowerShell --source winget
|
||||
```
|
||||
|
||||
- [MacOS](https://learn.microsoft.com/es-es/powershell/scripting/install/installing-powershell-on-macos?view=powershell-7.5#install-the-latest-stable-release-of-powershell): installing PowerShell on MacOS needs to have installed [brew](https://brew.sh/), once you have it is just running the command above, Pwsh is only supported in macOS 15 (Sequoia) x64 and Arm64, macOS 14 (Sonoma) x64 and Arm64, macOS 13 (Ventura) x64 and Arm64
|
||||
=== "MacOS"
|
||||
|
||||
```console
|
||||
brew install powershell/tap/powershell
|
||||
```
|
||||
[MacOS](https://learn.microsoft.com/es-es/powershell/scripting/install/installing-powershell-on-macos?view=powershell-7.5#install-the-latest-stable-release-of-powershell): installing PowerShell on MacOS needs to have installed [brew](https://brew.sh/), once installed, simply run the command shown above, Pwsh is only supported in macOS 15 (Sequoia) x64 and Arm64, macOS 14 (Sonoma) x64 and Arm64, macOS 13 (Ventura) x64 and Arm64
|
||||
|
||||
Once it's installed run `pwsh` on your terminal to verify it's working.
|
||||
```console
|
||||
brew install powershell/tap/powershell
|
||||
```
|
||||
|
||||
- Linux: installing PowerShell on Linux depends on the distro you are using:
|
||||
Once it's installed run `pwsh` on your terminal to verify it's working.
|
||||
|
||||
- [Ubuntu](https://learn.microsoft.com/es-es/powershell/scripting/install/install-ubuntu?view=powershell-7.5#installation-via-package-repository-the-package-repository): The required version for installing PowerShell +7.4 on Ubuntu are Ubuntu 22.04 and Ubuntu 24.04. The recommended way to install it is downloading the package available on PMC. You just need to follow the following steps:
|
||||
=== "Linux (Ubuntu)"
|
||||
|
||||
[Ubuntu](https://learn.microsoft.com/es-es/powershell/scripting/install/install-ubuntu?view=powershell-7.5#installation-via-package-repository-the-package-repository): The required version for installing PowerShell +7.4 on Ubuntu are Ubuntu 22.04 and Ubuntu 24.04.
|
||||
The recommended way to install it is downloading the package available on PMC.
|
||||
|
||||
Follow these steps:
|
||||
|
||||
```console
|
||||
###################################
|
||||
@@ -210,7 +239,11 @@ Once it's installed run `pwsh` on your terminal to verify it's working.
|
||||
pwsh
|
||||
```
|
||||
|
||||
- [Alpine](https://learn.microsoft.com/es-es/powershell/scripting/install/install-alpine?view=powershell-7.5#installation-steps): The only supported version for installing PowerShell +7.4 on Alpine is Alpine 3.20. The unique way to install it is downloading the tar.gz package available on [PowerShell github](https://github.com/PowerShell/PowerShell/releases/download/v7.5.0/powershell-7.5.0-linux-musl-x64.tar.gz). You just need to follow the following steps:
|
||||
=== "Linux (Alpine)"
|
||||
|
||||
[Alpine](https://learn.microsoft.com/es-es/powershell/scripting/install/install-alpine?view=powershell-7.5#installation-steps): The only supported version for installing PowerShell +7.4 on Alpine is Alpine 3.20. The unique way to install it is downloading the tar.gz package available on [PowerShell github](https://github.com/PowerShell/PowerShell/releases/download/v7.5.0/powershell-7.5.0-linux-musl-x64.tar.gz).
|
||||
|
||||
Follow these steps:
|
||||
|
||||
```console
|
||||
# Install the requirements
|
||||
@@ -252,7 +285,11 @@ Once it's installed run `pwsh` on your terminal to verify it's working.
|
||||
pwsh
|
||||
```
|
||||
|
||||
- [Debian](https://learn.microsoft.com/es-es/powershell/scripting/install/install-debian?view=powershell-7.5#installation-on-debian-11-or-12-via-the-package-repository): The required version for installing PowerShell +7.4 on Debian are Debian 11 and Debian 12. The recommended way to install it is downloading the package available on PMC. You just need to follow the following steps:
|
||||
=== "Linux (Debian)"
|
||||
|
||||
[Debian](https://learn.microsoft.com/es-es/powershell/scripting/install/install-debian?view=powershell-7.5#installation-on-debian-11-or-12-via-the-package-repository): The required version for installing PowerShell +7.4 on Debian are Debian 11 and Debian 12. The recommended way to install it is downloading the package available on PMC.
|
||||
|
||||
Follow these steps:
|
||||
|
||||
```console
|
||||
###################################
|
||||
@@ -287,7 +324,12 @@ Once it's installed run `pwsh` on your terminal to verify it's working.
|
||||
pwsh
|
||||
```
|
||||
|
||||
- [Rhel](https://learn.microsoft.com/es-es/powershell/scripting/install/install-rhel?view=powershell-7.5#installation-via-the-package-repository): The required version for installing PowerShell +7.4 on Red Hat are RHEL 8 and RHEL 9. The recommended way to install it is downloading the package available on PMC. You just need to follow the following steps:
|
||||
|
||||
=== "Linux (RHEL)"
|
||||
|
||||
[Rhel](https://learn.microsoft.com/es-es/powershell/scripting/install/install-rhel?view=powershell-7.5#installation-via-the-package-repository): The required version for installing PowerShell +7.4 on Red Hat are RHEL 8 and RHEL 9. The recommended way to install it is downloading the package available on PMC.
|
||||
|
||||
Follow these steps:
|
||||
|
||||
```console
|
||||
###################################
|
||||
@@ -317,7 +359,9 @@ Once it's installed run `pwsh` on your terminal to verify it's working.
|
||||
sudo dnf install powershell -y
|
||||
```
|
||||
|
||||
- [Docker](https://learn.microsoft.com/es-es/powershell/scripting/install/powershell-in-docker?view=powershell-7.5#use-powershell-in-a-container): The following command download the latest stable versions of PowerShell:
|
||||
=== "Docker"
|
||||
|
||||
[Docker](https://learn.microsoft.com/es-es/powershell/scripting/install/powershell-in-docker?view=powershell-7.5#use-powershell-in-a-container): The following command download the latest stable versions of PowerShell:
|
||||
|
||||
```console
|
||||
docker pull mcr.microsoft.com/dotnet/sdk:9.0
|
||||
@@ -329,6 +373,7 @@ Once it's installed run `pwsh` on your terminal to verify it's working.
|
||||
docker run -it mcr.microsoft.com/dotnet/sdk:9.0 pwsh
|
||||
```
|
||||
|
||||
|
||||
### Required PowerShell Modules
|
||||
|
||||
Prowler relies on several PowerShell cmdlets to retrieve necessary data.
|
||||
@@ -341,7 +386,7 @@ The required modules are automatically installed when running Prowler with the `
|
||||
Example command:
|
||||
|
||||
```console
|
||||
python3 prowler-cli.py m365 --verbose --log-level ERROR --env-auth --init-modules
|
||||
python3 prowler-cli.py m365 --verbose --log-level ERROR --sp-env-auth --init-modules
|
||||
```
|
||||
If the modules are already installed, running this command will not cause issues—it will simply verify that the necessary modules are available.
|
||||
|
||||
|
||||
@@ -1,275 +1,99 @@
|
||||
# Getting Started with M365 on Prowler Cloud/App
|
||||
|
||||
Set up your M365 account to enable security scanning using Prowler Cloud/App.
|
||||
# Getting Started With Microsoft 365 on Prowler
|
||||
|
||||
???+ note "Government Cloud Support"
|
||||
Government cloud accounts or tenants (Microsoft 365 Government) are not currently supported, but we expect to add support for them in the near future.
|
||||
Government cloud accounts or tenants (Microsoft 365 Government) are currently unsupported, but we expect to add support for them in the near future.
|
||||
|
||||
## Requirements
|
||||
## Prerequisites
|
||||
|
||||
To configure your M365 account, you'll need:
|
||||
Configure authentication for Microsoft 365 by following the [Microsoft 365 Authentication](authentication.md) guide. This includes:
|
||||
|
||||
1. Obtain a domain from the Entra ID portal.
|
||||
- Creating a Service Principal Application
|
||||
- Granting required Microsoft Graph API permissions
|
||||
- Setting up PowerShell module permissions (for full security coverage)
|
||||
- Assigning appropriate roles to users (if using user authentication)
|
||||
|
||||
2. Access Prowler Cloud/App and add a new cloud provider `Microsoft 365`.
|
||||
## Prowler App
|
||||
|
||||
3. Configure your M365 account:
|
||||
### Step 1: Obtain Domain ID
|
||||
|
||||
3.1 Create the Service Principal app.
|
||||
1. Go to the Entra ID portal, then search for "Domain" or go to Identity > Settings > Domain Names
|
||||
|
||||
3.2 Grant the required API permissions.
|
||||

|
||||
|
||||
3.3 Assign the required roles to your user.
|
||||

|
||||
|
||||
4. Add the credentials to Prowler Cloud/App.
|
||||
2. Select the domain to use as unique identifier for the Microsoft 365 account in Prowler App
|
||||
|
||||
## Step 1: Obtain your Domain
|
||||
|
||||
Go to the Entra ID portal, then you can search for `Domain` or go to Identity > Settings > Domain Names.
|
||||
|
||||

|
||||
|
||||
<br>
|
||||
|
||||

|
||||
|
||||
Once you are there just select the domain you want to use as unique identifier for your M365 account in Prowler Cloud/App.
|
||||
|
||||
---
|
||||
|
||||
## Step 2: Access Prowler Cloud/App
|
||||
### Step 2: Access Prowler App
|
||||
|
||||
1. Go to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](../prowler-app.md)
|
||||
2. Navigate to `Configuration` > `Cloud Providers`
|
||||
2. Navigate to "Configuration" > "Cloud Providers"
|
||||
|
||||

|
||||
|
||||
3. Click on `Add Cloud Provider`
|
||||
3. Click on "Add Cloud Provider"
|
||||
|
||||

|
||||
|
||||
4. Select `Microsoft 365`
|
||||
4. Select "Microsoft 365"
|
||||
|
||||

|
||||
|
||||
5. Add the Domain ID and an optional alias, then click `Next`
|
||||
5. Add the Domain ID and an optional alias, then click "Next"
|
||||
|
||||

|
||||
|
||||
---
|
||||
### Step 3: Add Credentials to Prowler App
|
||||
|
||||
## Step 3: Configure your M365 account
|
||||
|
||||
|
||||
### Create the Service Principal app
|
||||
|
||||
A Service Principal is required to grant Prowler the necessary privileges.
|
||||
|
||||
1. Access **Microsoft Entra ID**
|
||||
|
||||

|
||||
|
||||
2. Navigate to `Applications` > `App registrations`
|
||||
|
||||

|
||||
|
||||
3. Click `+ New registration`, complete the form, and click `Register`
|
||||
|
||||

|
||||
|
||||
4. Go to `Certificates & secrets` > `Client secrets` > `+ New client secret`
|
||||
|
||||

|
||||
|
||||
5. Fill in the required fields and click `Add`, then copy the generated `value` (that value will be `AZURE_CLIENT_SECRET`)
|
||||
|
||||

|
||||
|
||||
With this done you will have all the needed keys, summarized in the following table
|
||||
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| Client ID | Application (client) ID |
|
||||
| Client Secret | AZURE_CLIENT_SECRET |
|
||||
| Tenant ID | Directory (tenant) ID |
|
||||
|
||||
---
|
||||
|
||||
### Grant required Graph API permissions
|
||||
|
||||
Assign the following Microsoft Graph permissions:
|
||||
|
||||
- `AuditLog.Read.All`: Required for Entra service.
|
||||
- `Directory.Read.All`: Required for all services.
|
||||
- `Policy.Read.All`: Required for all services.
|
||||
- `SharePointTenantSettings.Read.All`: Required for SharePoint service.
|
||||
- `User.Read` (IMPORTANT: this is set as **delegated**): Required for the sign-in only if using user authentication.
|
||||
|
||||
???+ note
|
||||
You can replace `Directory.Read.All` with `Domain.Read.All` is a more restrictive permission but you won't be able to run the Entra checks related with DirectoryRoles and GetUsers.
|
||||
|
||||
> If you do this you will need to add also the `Organization.Read.All` permission to the service principal application in order to authenticate.
|
||||
|
||||
Follow these steps to assign the permissions:
|
||||
|
||||
1. Go to your App Registration > Select your Prowler App created before > click on `API permissions`
|
||||
|
||||

|
||||
|
||||
2. Click `+ Add a permission` > `Microsoft Graph` > `Application permissions`
|
||||
|
||||

|
||||
|
||||
3. Search and select every permission below and once all are selected click on `Add permissions`:
|
||||
- `AuditLog.Read.All`: Required for Entra service.
|
||||
- `Directory.Read.All`
|
||||
- `Policy.Read.All`
|
||||
- `SharePointTenantSettings.Read.All`
|
||||
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
|
||||
### Grant PowerShell modules permissions
|
||||
|
||||
The permissions you need to grant depends on whether you are using user credentials or service principal to authenticate to the M365 modules.
|
||||
|
||||
???+ warning "Warning"
|
||||
Make sure you add the correct set of permissions for the authentication method you are using.
|
||||
|
||||
|
||||
#### If using application(service principal) authentication (Recommended)
|
||||
|
||||
To grant the permissions for the PowerShell modules via application authentication, you need to add the necessary APIs to your app registration. All of this assignments are done through Entra ID.
|
||||
|
||||
???+ warning "Warning"
|
||||
You need to have a license that allows you to use the APIs.
|
||||
|
||||
1. Add Exchange API:
|
||||
|
||||
- Search and select`Office 365 Exchange Online` API in **APIs my organization uses**.
|
||||
|
||||

|
||||
|
||||
- Select `Exchange.ManageAsApp` permission and click on `Add permissions`.
|
||||
|
||||

|
||||
|
||||
You also need to assign the `Global Reader` role to the app. For that go to `Roles and administrators` and in the `Administrative roles` section click `here` to go to the directory level assignment:
|
||||
|
||||

|
||||
|
||||
Once in the directory level assignment, search for `Global Reader` and click on it to open the assginments page of that role.
|
||||
|
||||

|
||||
|
||||
Click on `Add assignments`, search for your app and click on `Assign`.
|
||||
|
||||
You have to select it as `Active` and click on `Assign` to assign the role to the app.
|
||||
|
||||

|
||||
|
||||
For more information about the need of adding this role, see [Microsoft documentation](https://learn.microsoft.com/en-us/powershell/exchange/app-only-auth-powershell-v2?view=exchange-ps#step-5-assign-microsoft-entra-roles-to-the-application). You can select any other role of the specified.
|
||||
|
||||
2. Add Teams API:
|
||||
|
||||
- Search and select `Skype and Teams Tenant Admin API` API in **APIs my organization uses**.
|
||||
|
||||

|
||||
|
||||
- Select `application_access` permission and click on `Add permissions`.
|
||||
|
||||

|
||||
|
||||
3. Click on `Grant admin consent for <your-tenant-name>` to grant admin consent.
|
||||
|
||||

|
||||
|
||||
The final result of permission assignment should be this:
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
#### If using user authentication
|
||||
|
||||
This method is not recommended because it requires a user with MFA enabled and Microsoft will not allow MFA capable users to authenticate programmatically after 1st October 2025. See [Microsoft documentation](https://learn.microsoft.com/en-us/entra/identity/authentication/concept-mandatory-multifactor-authentication?tabs=dotnet) for more information.
|
||||
|
||||
???+ warning
|
||||
Remember that if the user is newly created, you need to sign in with that account first, as Microsoft will prompt you to change the password. If you don’t complete this step, user authentication will fail because Microsoft marks the initial password as expired.
|
||||
|
||||
|
||||
1. Search and select:
|
||||
|
||||
- `User.Read`
|
||||
|
||||

|
||||
|
||||
2. Click `Add permissions`, then **grant admin consent**
|
||||
|
||||

|
||||
|
||||
The final result of permission assignment should be this:
|
||||
|
||||

|
||||
|
||||
3. Assign **required roles** to your **user**
|
||||
|
||||
Assign one of the following roles to your User:
|
||||
|
||||
- `Global Reader` (recommended): this allows you to read all roles needed.
|
||||
- `Exchange Administrator` and `Teams Administrator`: user needs both roles but with this [roles](https://learn.microsoft.com/en-us/exchange/permissions-exo/permissions-exo#microsoft-365-permissions-in-exchange-online) you can access to the same information as a Global Reader (here you only read so that's why we recomend that role).
|
||||
|
||||
Follow these steps to assign the role:
|
||||
|
||||
1. Go to Users > All Users > Click on the email for the user you will use
|
||||
|
||||

|
||||
|
||||
2. Click `Assigned Roles`
|
||||
|
||||

|
||||
|
||||
3. Click on `Add assignments`, then search and select:
|
||||
|
||||
- `Global Reader` This is the recommended, if you want to use the others just search for them
|
||||
|
||||

|
||||
|
||||
4. Click on next, then assign the role as `Active`, and click on `Assign` to grant admin consent
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Step 4: Add credentials to Prowler Cloud/App
|
||||
|
||||
1. Go to your App Registration overview and copy the `Client ID` and `Tenant ID`
|
||||
1. Go to App Registration overview and copy the Client ID and Tenant ID
|
||||
|
||||

|
||||
|
||||
2. Go to Prowler App and paste:
|
||||
|
||||
2. Go to Prowler Cloud/App and paste:
|
||||
|
||||
- `Client ID`
|
||||
- `Tenant ID`
|
||||
- `AZURE_CLIENT_SECRET` from earlier
|
||||
|
||||
If you are using user authentication, also add:
|
||||
|
||||
- `M365_USER` the user using the correct assigned domain, more info [here](../../tutorials/microsoft365/authentication.md#service-principal-and-user-credentials-authentication)
|
||||
- `M365_PASSWORD` the password of the user
|
||||
- Client ID
|
||||
- Tenant ID
|
||||
- `AZURE_CLIENT_SECRET` from the Service Principal setup
|
||||
|
||||

|
||||
|
||||
3. Click `Next`
|
||||
3. Click "Next"
|
||||
|
||||

|
||||
|
||||
4. Click `Launch Scan`
|
||||
4. Click "Launch Scan"
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
Use Prowler CLI to scan Microsoft 365 environments.
|
||||
|
||||
### PowerShell Requirements
|
||||
|
||||
PowerShell 7.4+ is required for comprehensive Microsoft 365 security coverage. Installation instructions are available in the [Authentication guide](authentication.md#supported-powershell-versions).
|
||||
|
||||
### Authentication Options
|
||||
|
||||
Select an authentication method from the [Microsoft 365 Authentication](authentication.md) guide:
|
||||
|
||||
- **Service Principal Application** (recommended): `--sp-env-auth`
|
||||
- **Interactive Browser Authentication**: `--browser-auth`
|
||||
|
||||
### Basic Usage
|
||||
|
||||
After configuring authentication, run a basic scan:
|
||||
|
||||
```console
|
||||
prowler m365 --sp-env-auth
|
||||
```
|
||||
|
||||
For comprehensive scans including PowerShell checks:
|
||||
|
||||
```console
|
||||
prowler m365 --sp-env-auth --init-modules
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -2,44 +2,49 @@
|
||||
|
||||
MongoDB Atlas provider uses [HTTP Digest Authentication with API key pairs consisting of a public key and private key](https://www.mongodb.com/docs/atlas/configure-api-access/#grant-programmatic-access-to-service).
|
||||
|
||||
## Authentication Methods
|
||||
|
||||
### Command-Line Arguments
|
||||
## Required Permissions
|
||||
|
||||
```bash
|
||||
prowler mongodbatlas --atlas-public-key <public_key> --atlas-private-key <private_key>
|
||||
```
|
||||
MongoDB Atlas API keys require appropriate permissions to perform security checks:
|
||||
|
||||
### Environment Variables
|
||||
- **Organization Read Only**: Provides read-only access to everything in the organization, including all projects in the organization.
|
||||
- To [audit the Auditing configuration for the project](https://www.mongodb.com/docs/api/doc/atlas-admin-api-v2/group/endpoint-auditing), **Organization Owner** permission is required.
|
||||
|
||||
```bash
|
||||
export ATLAS_PUBLIC_KEY=<public_key>
|
||||
export ATLAS_PRIVATE_KEY=<private_key>
|
||||
prowler mongodbatlas
|
||||
```
|
||||
The IP address where Prowler runs must be added to the IP Access List of the MongoDB Atlas organization API key. To skip this step and use the API key across all IP address types, uncheck the "Require IP Access List for the Atlas Administration API" button in Organization Settings. This setting is [enabled by default](https://www.mongodb.com/docs/atlas/configure-api-access/#optional--require-an-ip-access-list-for-the-atlas-administration-api).
|
||||
|
||||
## Creating API Keys
|
||||
???+ warning
|
||||
To ensure the check `organizations_api_access_list_required` passes, enable the API access list for the organization and add the execution IP to the organization's IP Access List. When running checks from Prowler Cloud, add our IP to the IP Access List.
|
||||
|
||||
### Step-by-Step Guide
|
||||

|
||||
|
||||
1. **Log into MongoDB Atlas**
|
||||
- Access the MongoDB Atlas console
|
||||
|
||||
2. **Navigate to Access Manager**
|
||||
- Go to the organization or project access management section
|
||||
## API Key
|
||||
|
||||
3. **Select API Keys Tab**
|
||||
- Click on the "API Keys" tab
|
||||
1. **Log into MongoDB Atlas**: Access the MongoDB Atlas console
|
||||
2. **Navigate to Access Manager**: Go to the organization access management section:
|
||||
|
||||
4. **Create API Key**
|
||||
- Click "Create API Key"
|
||||
- Provide a description for the key
|
||||
- Click "Access Manager" and "Organization Access":
|
||||
|
||||
5. **Set Permissions**
|
||||
- Grant minimum required permissions
|
||||

|
||||
|
||||
6. **Save Credentials**
|
||||
- Note the public key and private key
|
||||
- Store credentials securely
|
||||
- Then click the "Applications" tab inside the Access Manager:
|
||||
|
||||
For more details about MongoDB Atlas, see the [MongoDB Atlas Tutorial](./getting-started-mongodbatlas.md).
|
||||

|
||||
|
||||
3. **Select API Keys Tab**: Click the "API Keys" tab that appears in the image above
|
||||
|
||||
4. **Create API Key**: Click "Create API Key" and provide a description
|
||||
|
||||

|
||||
|
||||
5. **Set Permissions**: Recommend project permissions for enhanced security; modify them after creating the key
|
||||
|
||||

|
||||
|
||||
6. **Save Credentials**: Record both the public and private keys, then store them securely
|
||||
|
||||

|
||||
|
||||
7. **Add IP Access List**: Add the IP address where Prowler runs to the API Key's IP Access List. To skip this step and use the API key for all IP addresses, uncheck the "Require IP Access List for the Atlas Administration API" button in [Organization Settings](#required-permissions), though this is not recommended.
|
||||
|
||||

|
||||
|
||||
@@ -1,64 +1,42 @@
|
||||
# Getting Started with MongoDB Atlas
|
||||
|
||||
MongoDB Atlas provider enables security assessments of MongoDB Atlas cloud database deployments.
|
||||
## Prowler CLI
|
||||
|
||||
## Features
|
||||
### Authentication Methods
|
||||
|
||||
- **Authentication**: Supports MongoDB Atlas API key authentication
|
||||
- **Services**: Projects and clusters services
|
||||
- **Checks**: Network access security and encryption at rest validation
|
||||
#### Command-Line Arguments
|
||||
|
||||
## Creating API Keys
|
||||
|
||||
To create MongoDB Atlas API keys:
|
||||
```bash
|
||||
prowler mongodbatlas --atlas-public-key <public_key> --atlas-private-key <private_key>
|
||||
```
|
||||
|
||||
1. **Log into MongoDB Atlas**: Access the MongoDB Atlas console
|
||||
2. **Navigate to Access Manager**: Go to the organization access management section:
|
||||
#### Environment Variables
|
||||
|
||||
- Click on Access Manager and Organization Access:
|
||||
```bash
|
||||
export ATLAS_PUBLIC_KEY=<public_key>
|
||||
export ATLAS_PRIVATE_KEY=<private_key>
|
||||
prowler mongodbatlas
|
||||
```
|
||||
|
||||

|
||||
|
||||
- After that click on the Applications tab inside the Access Manager:
|
||||
|
||||

|
||||
|
||||
3. **Select API Keys Tab**: Click on the "API Keys" tab that appears in the image above
|
||||
|
||||
4. **Create API Key**: Click "Create API Key" and provide a description
|
||||
|
||||

|
||||
|
||||
5. **Set Permissions**: Project permissions are recommended for security, you can modify them after creating the key
|
||||
|
||||

|
||||
|
||||
6. **Save Credentials**: Note the public key and private key and store them securely
|
||||
|
||||

|
||||
|
||||
7. **Add IP Access List**: Add the IP where you are running Prowler to the IP Access List of the API Key. If you want to skip this step and use your API key in all type of IP addresses you need to uncheck the `Require IP Access List for the Atlas Administration API` button on the [Organization Settings](#needed-permissions), but this is not recommended.
|
||||
|
||||

|
||||
|
||||
## Basic Usage
|
||||
|
||||
### Scan All Projects and Clusters
|
||||
|
||||
After storing your API keys, you can run Prowler with the following command:
|
||||
After storing API keys, run Prowler with the following command:
|
||||
|
||||
```bash
|
||||
prowler mongodbatlas --atlas-public-key <key> --atlas-private-key <secret>
|
||||
```
|
||||
|
||||
Also, you can set your API keys as environment variables:
|
||||
Alternatively, set API keys as environment variables:
|
||||
|
||||
```bash
|
||||
export ATLAS_PUBLIC_KEY=<key>
|
||||
export ATLAS_PRIVATE_KEY=<secret>
|
||||
```
|
||||
|
||||
And then just run Prowler with the following command:
|
||||
Then run Prowler with the following command:
|
||||
|
||||
```bash
|
||||
prowler mongodbatlas
|
||||
@@ -66,22 +44,8 @@ prowler mongodbatlas
|
||||
|
||||
### Scanning a Specific Project
|
||||
|
||||
If you want to scan a specific project, you can use the following argument added to the command above:
|
||||
To scan a specific project, add the following argument to the command above:
|
||||
|
||||
```bash
|
||||
prowler mongodbatlas --atlas-project-id <project-id>
|
||||
```
|
||||
|
||||
### Needed Permissions
|
||||
|
||||
MongoDB Atlas API keys require appropriate permissions to perform security checks:
|
||||
|
||||
- **Organization Read Only**: Provides read-only access to everything in the organization, including all projects in the organization.
|
||||
- If you want to be able to [audit the Auditing configuration for the project](https://www.mongodb.com/docs/api/doc/atlas-admin-api-v2/group/endpoint-auditing), **Organization Owner** is needed.
|
||||
|
||||
Also, it's important to note that the IP where you are running Prowler must be added to the IP Access List of the MongoDB Atlas organization API key. If you want to skip this step and use your API key in all type of IP addresses you need to uncheck the `Require IP Access List for the Atlas Administration API` button on the Organization Settings, that setting is [enabled by default](https://www.mongodb.com/docs/atlas/configure-api-access/#optional--require-an-ip-access-list-for-the-atlas-administration-api).
|
||||
|
||||
???+ warning
|
||||
If you want the check `organizations_api_access_list_required` to pass you will need to enable the API access list for the organization, so to make sure that your API Key is working you need to add your IP to the IP Access List of the organization. If you are running the check from Prowler Cloud, you will need to add our IP to the IP Access List.
|
||||
|
||||

|
||||
|
||||
@@ -194,10 +194,10 @@ If you are adding an **EKS**, **GKE**, **AKS** or external cluster, follow these
|
||||
For M365, you must enter your Domain ID and choose the authentication method you want to use:
|
||||
|
||||
- Service Principal Authentication (Recommended)
|
||||
- User Authentication (only works if the user does not have MFA enabled)
|
||||
- User Authentication (Deprecated)
|
||||
|
||||
???+ note
|
||||
User authentication with M365_USER and M365_PASSWORD is optional and will only work if the account does not enforce MFA.
|
||||
???+ warning
|
||||
User authentication with M365_USER and M365_PASSWORD is deprecated and will be removed.
|
||||
|
||||
For full setup instructions and requirements, check the [Microsoft 365 provider requirements](./microsoft365/getting-started-m365.md).
|
||||
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
|
||||
.env
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
.git/
|
||||
.gitignore
|
||||
.gitattributes
|
||||
.github/
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
LICENSE
|
||||
docs/
|
||||
tests/
|
||||
examples/
|
||||
.pre-commit-config.yaml
|
||||
Makefile
|
||||
docker-compose.yml
|
||||
docker-compose.yaml
|
||||
@@ -1,4 +1,3 @@
|
||||
PROWLER_APP_EMAIL="your_registered@email.com"
|
||||
PROWLER_APP_PASSWORD="your_user_pass"
|
||||
PROWLER_APP_TENANT_ID="optional_tenant_to_login"
|
||||
PROWLER_API_BASE_URL=https://api.prowler.com
|
||||
PROWLER_APP_API_KEY="pk_your_api_key_here"
|
||||
PROWLER_API_BASE_URL="https://api.prowler.com"
|
||||
PROWLER_MCP_MODE="stdio"
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# =============================================================================
|
||||
# Build stage - Install dependencies and build the application
|
||||
# =============================================================================
|
||||
FROM ghcr.io/astral-sh/uv:python3.13-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Performance optimizations for uv:
|
||||
# UV_COMPILE_BYTECODE=1: Pre-compile Python files to .pyc for faster startup
|
||||
# UV_LINK_MODE=copy: Use copy instead of symlinks to avoid potential issues
|
||||
ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
|
||||
|
||||
# Install dependencies first for better Docker layer caching
|
||||
# This allows dependency layer to be reused when only source code changes
|
||||
COPY uv.lock pyproject.toml ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-install-project
|
||||
|
||||
# Copy all source code and install the project
|
||||
# --frozen ensures reproducible builds by using exact versions from uv.lock
|
||||
COPY . .
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen
|
||||
|
||||
# =============================================================================
|
||||
# Final stage - Minimal runtime environment
|
||||
# =============================================================================
|
||||
FROM python:3.13-alpine
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud"
|
||||
|
||||
# Create non-root user for security
|
||||
# Using specific UID/GID for consistency across environments
|
||||
RUN addgroup -g 1001 prowler && \
|
||||
adduser -D -u 1001 -G prowler prowler
|
||||
|
||||
WORKDIR /app
|
||||
USER prowler
|
||||
|
||||
# Copy only the necessary files from builder stage to minimize image size:
|
||||
# 1. Virtual environment with all dependencies and the installed package
|
||||
COPY --from=builder --chown=prowler /app/.venv /app/.venv
|
||||
|
||||
# 2. Source code needed at runtime (for imports and module resolution)
|
||||
COPY --from=builder --chown=prowler /app/prowler_mcp_server /app/prowler_mcp_server
|
||||
|
||||
# 3. Project metadata file (may be needed by some packages at runtime)
|
||||
COPY --from=builder --chown=prowler /app/pyproject.toml /app/pyproject.toml
|
||||
|
||||
# Add virtual environment to PATH so prowler-mcp command is available
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
# Entry point for the MCP server
|
||||
# Default to stdio mode, but allow overriding via command arguments
|
||||
# Examples:
|
||||
# docker run -p 8000:8000 prowler-mcp --transport http --host 0.0.0.0 --port 8000
|
||||
# docker run prowler-mcp --transport stdio
|
||||
ENTRYPOINT ["prowler-mcp"]
|
||||
CMD ["--transport", "stdio"]
|
||||
+222
-12
@@ -1,5 +1,7 @@
|
||||
# Prowler MCP Server
|
||||
|
||||
> ⚠️ **Preview Feature**: This MCP server is currently in preview and under active development. Features and functionality may change. We welcome your feedback—please report any issues on [GitHub](https://github.com/prowler-cloud/prowler/issues) or join our [Slack community](https://goto.prowler.com/slack) to discuss and share your thoughts.
|
||||
|
||||
Access the entire Prowler ecosystem through the Model Context Protocol (MCP). This server provides two main capabilities:
|
||||
|
||||
- **Prowler Cloud and Prowler App (Self-Managed)**: Full access to Prowler Cloud platform and Prowler Self-Managed for managing providers, running scans, and analyzing security findings
|
||||
@@ -23,8 +25,63 @@ It is needed to have [uv](https://docs.astral.sh/uv/) installed.
|
||||
git clone https://github.com/prowler-cloud/prowler.git
|
||||
```
|
||||
|
||||
### Using Docker
|
||||
|
||||
Alternatively, you can build and run the MCP server using Docker:
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/prowler-cloud/prowler.git
|
||||
cd prowler/mcp_server
|
||||
|
||||
# Build the Docker image
|
||||
docker build -t prowler-mcp .
|
||||
|
||||
# Run the container with environment variables
|
||||
docker run --rm --env-file ./.env -it prowler-mcp
|
||||
```
|
||||
|
||||
## Running
|
||||
|
||||
The Prowler MCP server supports two transport modes:
|
||||
- **STDIO mode** (default): For direct integration with MCP clients like Claude Desktop
|
||||
- **HTTP mode**: For remote access over HTTP with Bearer token authentication
|
||||
|
||||
### Transport Modes
|
||||
|
||||
#### STDIO Mode (Default)
|
||||
|
||||
STDIO mode is the standard MCP transport for direct client integration:
|
||||
|
||||
```bash
|
||||
cd prowler/mcp_server
|
||||
uv run prowler-mcp
|
||||
# or
|
||||
uv run prowler-mcp --transport stdio
|
||||
```
|
||||
|
||||
#### HTTP Mode (Remote Server)
|
||||
|
||||
HTTP mode allows the server to run as a remote service accessible over HTTP:
|
||||
|
||||
```bash
|
||||
cd prowler/mcp_server
|
||||
# Run on default host and port (127.0.0.1:8000)
|
||||
uv run prowler-mcp --transport http
|
||||
|
||||
# Run on custom host and port
|
||||
uv run prowler-mcp --transport http --host 0.0.0.0 --port 8080
|
||||
```
|
||||
|
||||
For self-deployed MCP remote server, you can use also configure the server to use a custom API base URL with the environment variable `PROWLER_API_BASE_URL`; and the transport mode with the environment variable `PROWLER_MCP_MODE`.
|
||||
|
||||
```bash
|
||||
export PROWLER_API_BASE_URL="https://api.prowler.com"
|
||||
export PROWLER_MCP_MODE="http"
|
||||
```
|
||||
|
||||
### Using uv directly
|
||||
|
||||
After installation, start the MCP server via the console script:
|
||||
|
||||
```bash
|
||||
@@ -38,6 +95,63 @@ Alternatively, you can run from wherever you want using `uvx` command:
|
||||
uvx /path/to/prowler/mcp_server/
|
||||
```
|
||||
|
||||
### Using Docker
|
||||
|
||||
#### STDIO Mode (Default)
|
||||
|
||||
Run the pre-built Docker container in STDIO mode:
|
||||
|
||||
```bash
|
||||
cd prowler/mcp_server
|
||||
docker run --rm --env-file ./.env -it prowler-mcp
|
||||
```
|
||||
|
||||
#### HTTP Mode (Remote Server)
|
||||
|
||||
Run as a remote HTTP server:
|
||||
|
||||
```bash
|
||||
cd prowler/mcp_server
|
||||
# Run on port 8000 (accessible from host)
|
||||
docker run --rm --env-file ./.env -p 8000:8000 -it prowler-mcp --transport http --host 0.0.0.0 --port 8000
|
||||
|
||||
# Run on custom port
|
||||
docker run --rm --env-file ./.env -p 8080:8080 -it prowler-mcp --transport http --host 0.0.0.0 --port 8080
|
||||
```
|
||||
|
||||
## Command Line Arguments
|
||||
|
||||
The Prowler MCP server supports the following command line arguments:
|
||||
|
||||
```
|
||||
prowler-mcp [--transport {stdio,http}] [--host HOST] [--port PORT]
|
||||
```
|
||||
|
||||
**Arguments:**
|
||||
- `--transport {stdio,http}`: Transport method (default: stdio)
|
||||
- `stdio`: Standard input/output transport for direct MCP client integration
|
||||
- `http`: HTTP transport for remote server access
|
||||
- `--host HOST`: Host to bind to for HTTP transport (default: 127.0.0.1)
|
||||
- `--port PORT`: Port to bind to for HTTP transport (default: 8000)
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
# Default STDIO mode
|
||||
prowler-mcp
|
||||
|
||||
# Explicit STDIO mode
|
||||
prowler-mcp --transport stdio
|
||||
|
||||
# HTTP mode with default host and port (127.0.0.1:8000)
|
||||
prowler-mcp --transport http
|
||||
|
||||
# HTTP mode accessible from any network interface
|
||||
prowler-mcp --transport http --host 0.0.0.0
|
||||
|
||||
# HTTP mode with custom port
|
||||
prowler-mcp --transport http --host 0.0.0.0 --port 8080
|
||||
```
|
||||
|
||||
## Available Tools
|
||||
|
||||
### Prowler Hub
|
||||
@@ -46,6 +160,9 @@ All tools are exposed under the `prowler_hub` prefix.
|
||||
|
||||
- `prowler_hub_get_check_filters`: Return available filter values for checks (providers, services, severities, categories, compliances). Call this before `prowler_hub_get_checks` to build valid queries.
|
||||
- `prowler_hub_get_checks`: List checks with option of advanced filtering.
|
||||
- `prowler_hub_get_check_raw_metadata`: Fetch raw check metadata JSON (low-level version of get_checks).
|
||||
- `prowler_hub_get_check_code`: Fetch check implementation Python code from Prowler.
|
||||
- `prowler_hub_get_check_fixer`: Fetch check fixer Python code from Prowler (if it exists).
|
||||
- `prowler_hub_search_checks`: Full‑text search across check metadata.
|
||||
- `prowler_hub_get_compliance_frameworks`: List/filter compliance frameworks.
|
||||
- `prowler_hub_search_compliance_frameworks`: Full-text search across frameworks.
|
||||
@@ -98,25 +215,64 @@ All tools are exposed under the `prowler_app` prefix.
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
### Prowler Cloud and Prowler App (Self-Managed) Authentication
|
||||
|
||||
For Prowler Cloud and Prowler App (Self-Managed) features, you need to set the following environment variables:
|
||||
> [!IMPORTANT]
|
||||
> Authentication is not needed for using Prowler Hub features.
|
||||
|
||||
The Prowler MCP server supports different authentication in Prowler Cloud and Prowler App (Self-Managed) methods depending on the transport mode:
|
||||
|
||||
#### STDIO Mode Authentication
|
||||
|
||||
For STDIO mode, authentication is handled via environment variables using an API key:
|
||||
|
||||
```bash
|
||||
# Required for Prowler Cloud and Prowler App (Self-Managed) authentication
|
||||
export PROWLER_APP_EMAIL="your-email@example.com"
|
||||
export PROWLER_APP_PASSWORD="your-password"
|
||||
|
||||
# Optional - in case not provided the first membership that was added to the user will be used. This can be found as `Organization ID` in your User Profile in Prowler App
|
||||
export PROWLER_APP_TENANT_ID="your-tenant-id"
|
||||
export PROWLER_APP_API_KEY="pk_your_api_key_here"
|
||||
|
||||
# Optional - for custom API endpoint, in case not provided Prowler Cloud API will be used
|
||||
export PROWLER_API_BASE_URL="https://api.prowler.com"
|
||||
```
|
||||
|
||||
#### HTTP Mode Authentication
|
||||
|
||||
For HTTP mode (remote server), authentication is handled via Bearer tokens. The MCP server supports both JWT tokens and API keys:
|
||||
|
||||
**Option 1: Using API Keys (Recommended)**
|
||||
Use your Prowler API key directly in the MCP client configuration with Bearer token format:
|
||||
```
|
||||
Authorization: Bearer pk_your_api_key_here
|
||||
```
|
||||
|
||||
**Option 2: Using JWT Tokens**
|
||||
You need to obtain a JWT token from Prowler Cloud/App and include the generated token in the MCP client configuration. To get a valid token, you can use the following command (replace the email and password with your own credentials):
|
||||
|
||||
```bash
|
||||
curl -X POST https://api.prowler.com/api/v1/tokens \
|
||||
-H "Content-Type: application/vnd.api+json" \
|
||||
-H "Accept: application/vnd.api+json" \
|
||||
-d '{
|
||||
"data": {
|
||||
"type": "tokens",
|
||||
"attributes": {
|
||||
"email": "your-email@example.com",
|
||||
"password": "your-password"
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
The response will be a JWT token that you can use to [authenticate your MCP client](#http-mode-configuration-remote-server).
|
||||
|
||||
### MCP Client Configuration
|
||||
|
||||
Configure your MCP client, like Claude Desktop, Cursor, etc, to launch the server with the `uvx` command. Below is a generic snippet; consult your client's documentation for exact locations.
|
||||
Configure your MCP client, like Claude Desktop, Cursor, etc, to connect to the server. The configuration depends on whether you're running in STDIO mode (local) or HTTP mode (remote).
|
||||
|
||||
#### STDIO Mode Configuration
|
||||
|
||||
For local execution, configure your MCP client to launch the server directly. Below are examples for both direct execution and Docker deployment; consult your client's documentation for exact locations.
|
||||
|
||||
##### Using uvx (Direct Execution)
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -125,10 +281,64 @@ Configure your MCP client, like Claude Desktop, Cursor, etc, to launch the serve
|
||||
"command": "uvx",
|
||||
"args": ["/path/to/prowler/mcp_server/"],
|
||||
"env": {
|
||||
"PROWLER_APP_EMAIL": "your-email@example.com",
|
||||
"PROWLER_APP_PASSWORD": "your-password",
|
||||
"PROWLER_APP_TENANT_ID": "your-tenant-id", // Optional, this can be found as `Organization ID` in your User Profile in Prowler App
|
||||
"PROWLER_API_BASE_URL": "https://api.prowler.com" // Optional
|
||||
"PROWLER_APP_API_KEY": "pk_your_api_key_here",
|
||||
"PROWLER_API_BASE_URL": "https://api.prowler.com" // Optional, in case not provided Prowler Cloud API will be used
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
##### Using Docker
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"prowler": {
|
||||
"command": "docker",
|
||||
"args": [
|
||||
"run", "--rm", "-i",
|
||||
"--env", "PROWLER_APP_API_KEY=pk_your_api_key_here",
|
||||
"--env", "PROWLER_API_BASE_URL=https://api.prowler.com", // Optional, in case not provided Prowler Cloud API will be used
|
||||
"prowler-mcp"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### HTTP Mode Configuration (Remote Server)
|
||||
|
||||
For HTTP mode, you can configure your MCP client to connect to a remote Prowler MCP server.
|
||||
|
||||
**Important Limitations:**
|
||||
- HTTP mode support varies by client - some clients may not support HTTP transport yet.
|
||||
- Some MCP clients like Claude Desktop only support OAuth authentication for HTTP connections, which is not currently supported by our MCP server.
|
||||
|
||||
Example configuration for clients that support HTTP transport:
|
||||
|
||||
**Using API Key (Recommended):**
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"prowler": {
|
||||
"url": "http://mcp.prowler.com/mcp", // Replace with your own MCP server URL, by default when server is run in local it is http://localhost:8000/mcp
|
||||
"headers": {
|
||||
"Authorization": "Bearer pk_your_api_key_here"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Using JWT Token:**
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"prowler": {
|
||||
"url": "http://mcp.prowler.com/mcp", // Replace with your own MCP server URL, by default when server is run in local it is http://localhost:8000/mcp
|
||||
"headers": {
|
||||
"Authorization": "Bearer <your-jwt-token-here>"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,15 +1,48 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
from prowler_mcp_server.lib.logger import logger
|
||||
from prowler_mcp_server.server import prowler_mcp_server, setup_main_server
|
||||
from prowler_mcp_server.server import setup_main_server
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
"""Parse command line arguments."""
|
||||
parser = argparse.ArgumentParser(description="Prowler MCP Server")
|
||||
parser.add_argument(
|
||||
"--transport",
|
||||
choices=["stdio", "http"],
|
||||
default=os.getenv("PROWLER_MCP_MODE", "stdio"),
|
||||
help="Transport method (default: stdio)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
default="127.0.0.1",
|
||||
help="Host to bind to for HTTP transport (default: 127.0.0.1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
default=8000,
|
||||
help="Port to bind to for HTTP transport (default: 8000)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the MCP server."""
|
||||
try:
|
||||
asyncio.run(setup_main_server())
|
||||
prowler_mcp_server.run()
|
||||
args = parse_arguments()
|
||||
|
||||
# Set up server with configuration
|
||||
prowler_mcp_server = asyncio.run(setup_main_server(transport=args.transport))
|
||||
|
||||
if args.transport == "stdio":
|
||||
prowler_mcp_server.run(transport="stdio")
|
||||
elif args.transport == "http":
|
||||
prowler_mcp_server.run(transport="http", host=args.host, port=args.port)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Shutting down Prowler MCP server...")
|
||||
sys.exit(0)
|
||||
|
||||
@@ -1,38 +1,36 @@
|
||||
"""Authentication manager for Prowler App API."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
|
||||
import httpx
|
||||
from fastmcp.server.dependencies import get_http_headers
|
||||
from prowler_mcp_server import __version__
|
||||
from prowler_mcp_server.lib.logger import logger
|
||||
|
||||
|
||||
class ProwlerAppAuth:
|
||||
"""Handles authentication and token management for Prowler App API."""
|
||||
|
||||
def __init__(self):
|
||||
self.base_url = os.getenv(
|
||||
"PROWLER_API_BASE_URL", "https://api.prowler.com"
|
||||
).rstrip("/")
|
||||
self.email = os.getenv("PROWLER_APP_EMAIL")
|
||||
self.password = os.getenv("PROWLER_APP_PASSWORD")
|
||||
self.tenant_id = os.getenv("PROWLER_APP_TENANT_ID", None)
|
||||
"""Handles authentication for Prowler App API using API keys or JWT tokens."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mode: str = os.getenv("PROWLER_MCP_MODE", "stdio"),
|
||||
base_url: str = os.getenv("PROWLER_API_BASE_URL", "https://api.prowler.com"),
|
||||
):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
logger.info(f"Using Prowler App API base URL: {self.base_url}")
|
||||
self.mode = mode
|
||||
self.access_token: Optional[str] = None
|
||||
self.refresh_token: Optional[str] = None
|
||||
self.api_key: Optional[str] = None
|
||||
|
||||
self._validate_credentials()
|
||||
if mode == "stdio": # STDIO mode
|
||||
self.api_key = os.getenv("PROWLER_APP_API_KEY")
|
||||
|
||||
def _validate_credentials(self):
|
||||
"""Validate that all required credentials are present."""
|
||||
if not self.email:
|
||||
raise ValueError("PROWLER_APP_EMAIL environment variable is required")
|
||||
if not self.password:
|
||||
raise ValueError("PROWLER_APP_PASSWORD environment variable is required")
|
||||
if not self.api_key:
|
||||
raise ValueError("PROWLER_APP_API_KEY environment variable is required")
|
||||
|
||||
if not self.api_key.startswith("pk_"):
|
||||
raise ValueError("Prowler App API key format is incorrect")
|
||||
|
||||
def _parse_jwt(self, token: str) -> Optional[Dict]:
|
||||
"""Parse JWT token and return payload, similar to JS parseJwt function."""
|
||||
@@ -61,140 +59,61 @@ class ProwlerAppAuth:
|
||||
return None
|
||||
|
||||
async def authenticate(self) -> str:
|
||||
"""Authenticate with Prowler App API and return access token."""
|
||||
logger.info("Starting authentication with Prowler App API")
|
||||
async with httpx.AsyncClient() as client:
|
||||
try:
|
||||
# Prepare JSON:API formatted request body
|
||||
auth_attributes = {"email": self.email, "password": self.password}
|
||||
if self.tenant_id:
|
||||
auth_attributes["tenant_id"] = self.tenant_id
|
||||
"""Authenticate and return token (API key for STDIO, API key or JWT for HTTP)."""
|
||||
if self.mode == "http":
|
||||
headers = get_http_headers()
|
||||
authorization_header = headers.get("authorization", None)
|
||||
|
||||
request_body = {
|
||||
"data": {
|
||||
"type": "tokens",
|
||||
"attributes": auth_attributes,
|
||||
}
|
||||
}
|
||||
if not authorization_header:
|
||||
raise ValueError("No authorization header provided")
|
||||
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/v1/tokens",
|
||||
json=request_body,
|
||||
headers={
|
||||
"Content-Type": "application/vnd.api+json",
|
||||
"Accept": "application/vnd.api+json",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
# Extract token from JSON:API response format
|
||||
self.access_token = (
|
||||
data.get("data", {}).get("attributes", {}).get("access")
|
||||
)
|
||||
self.refresh_token = (
|
||||
data.get("data", {}).get("attributes", {}).get("refresh")
|
||||
# Extract token from Bearer header
|
||||
if authorization_header.startswith("Bearer "):
|
||||
token = authorization_header.replace("Bearer ", "")
|
||||
else:
|
||||
raise ValueError(
|
||||
"Invalid authorization header format. Expected 'Bearer <token>'"
|
||||
)
|
||||
|
||||
logger.debug(f"Access token: {self.access_token}")
|
||||
# Check if it's an API key or JWT token
|
||||
if token.startswith("pk_"):
|
||||
# API key - no expiration check needed
|
||||
return token
|
||||
else:
|
||||
# JWT token - validate and check expiration
|
||||
payload = self._parse_jwt(token)
|
||||
if not payload:
|
||||
raise ValueError("Invalid JWT token format")
|
||||
|
||||
if not self.access_token:
|
||||
raise ValueError("Token not found in response")
|
||||
# Check if token is expired
|
||||
now = int(datetime.now().timestamp())
|
||||
exp = payload.get("exp", 0)
|
||||
if exp <= now:
|
||||
raise ValueError("Token has expired")
|
||||
|
||||
logger.info("Authentication successful")
|
||||
|
||||
return self.access_token
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.error(
|
||||
f"Authentication failed with HTTP status {e.response.status_code}: {e.response.text}"
|
||||
)
|
||||
raise ValueError(f"Authentication failed: {e.response.text}")
|
||||
except Exception as e:
|
||||
logger.error(f"Authentication failed with error: {e}")
|
||||
raise ValueError(f"Authentication failed: {e}")
|
||||
|
||||
async def refresh_access_token(self) -> str:
|
||||
"""Refresh the access token using the refresh token."""
|
||||
if not self.refresh_token:
|
||||
logger.info("No refresh token available, performing full authentication")
|
||||
return await self.authenticate()
|
||||
|
||||
logger.info("Refreshing access token")
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
try:
|
||||
# Prepare JSON:API formatted request body for refresh
|
||||
request_body = {
|
||||
"data": {
|
||||
"type": "tokens",
|
||||
"attributes": {"refresh": self.refresh_token},
|
||||
}
|
||||
}
|
||||
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/v1/tokens/refresh",
|
||||
json=request_body,
|
||||
headers={
|
||||
"Content-Type": "application/vnd.api+json",
|
||||
"Accept": "application/vnd.api+json",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
# Extract new access token from JSON:API response
|
||||
self.access_token = (
|
||||
data.get("data", {}).get("attributes", {}).get("access")
|
||||
)
|
||||
logger.info("Token refresh successful")
|
||||
|
||||
return self.access_token
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(
|
||||
f"Token refresh failed, attempting re-authentication: {e}"
|
||||
)
|
||||
# If refresh fails, re-authenticate
|
||||
return await self.authenticate()
|
||||
return token
|
||||
else:
|
||||
raise ValueError(f"Invalid mode: {self.mode}")
|
||||
|
||||
async def get_valid_token(self) -> str:
|
||||
"""Get a valid access token, checking JWT expiry."""
|
||||
|
||||
current_token = self.access_token
|
||||
need_new_token = True
|
||||
|
||||
if current_token:
|
||||
payload = self._parse_jwt(current_token)
|
||||
|
||||
if payload:
|
||||
now = int(datetime.now().timestamp())
|
||||
time_left = payload.get("exp", 0) - now
|
||||
|
||||
if time_left > 120: # 2 minutes margin
|
||||
need_new_token = False
|
||||
|
||||
if need_new_token:
|
||||
token = await self.authenticate()
|
||||
|
||||
# Verify the new token
|
||||
payload = self._parse_jwt(token)
|
||||
|
||||
return token
|
||||
"""Get a valid token (API key or JWT token)."""
|
||||
if self.mode == "stdio" and self.api_key:
|
||||
return self.api_key
|
||||
else:
|
||||
return current_token
|
||||
return await self.authenticate()
|
||||
|
||||
def get_headers(self, token: str) -> Dict[str, str]:
|
||||
"""Get headers for API requests with authentication."""
|
||||
if token.startswith("pk_"):
|
||||
authorization_header = f"Api-Key {token}"
|
||||
else:
|
||||
authorization_header = f"Bearer {token}"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Authorization": authorization_header,
|
||||
"Content-Type": "application/vnd.api+json",
|
||||
"Accept": "application/vnd.api+json",
|
||||
"User-Agent": f"prowler-mcp-server/{__version__}",
|
||||
}
|
||||
|
||||
# Add tenant ID header if available
|
||||
if self.tenant_id:
|
||||
headers["X-Tenant-Id"] = self.tenant_id
|
||||
|
||||
return headers
|
||||
|
||||
@@ -11,11 +11,10 @@ import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
from prowler_mcp_server.lib.logger import logger
|
||||
|
||||
|
||||
class OpenAPIToMCPGenerator:
|
||||
@@ -23,10 +22,10 @@ class OpenAPIToMCPGenerator:
|
||||
self,
|
||||
spec_file: str,
|
||||
custom_auth_module: Optional[str] = None,
|
||||
exclude_patterns: Optional[List[str]] = None,
|
||||
exclude_operations: Optional[List[str]] = None,
|
||||
exclude_tags: Optional[List[str]] = None,
|
||||
include_only_tags: Optional[List[str]] = None,
|
||||
exclude_patterns: Optional[list[str]] = None,
|
||||
exclude_operations: Optional[list[str]] = None,
|
||||
exclude_tags: Optional[list[str]] = None,
|
||||
include_only_tags: Optional[list[str]] = None,
|
||||
config_file: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
@@ -35,9 +34,9 @@ class OpenAPIToMCPGenerator:
|
||||
Args:
|
||||
spec_file: Path to OpenAPI specification file
|
||||
custom_auth_module: Module path for custom authentication
|
||||
exclude_patterns: List of regex patterns to exclude endpoints (matches against path)
|
||||
exclude_operations: List of operation IDs to exclude
|
||||
exclude_tags: List of tags to exclude
|
||||
exclude_patterns: list of regex patterns to exclude endpoints (matches against path)
|
||||
exclude_operations: list of operation IDs to exclude
|
||||
exclude_tags: list of tags to exclude
|
||||
include_only_tags: If specified, only include endpoints with these tags
|
||||
config_file: Path to JSON configuration file for custom mappings
|
||||
"""
|
||||
@@ -54,26 +53,24 @@ class OpenAPIToMCPGenerator:
|
||||
self.imports = set()
|
||||
self.type_mapping = {
|
||||
"string": "str",
|
||||
"integer": "int",
|
||||
"number": "float",
|
||||
"boolean": "bool",
|
||||
"array": "str",
|
||||
"object": "Dict[str, Any]",
|
||||
"integer": "int | str",
|
||||
"number": "float | str",
|
||||
"boolean": "bool | str",
|
||||
"array": "list[Any] | str",
|
||||
"object": "dict[str, Any] | str",
|
||||
}
|
||||
|
||||
def _load_config(self) -> Dict:
|
||||
def _load_config(self) -> dict:
|
||||
"""Load configuration from JSON file."""
|
||||
try:
|
||||
with open(self.config_file, "r") as f:
|
||||
return json.load(f)
|
||||
except FileNotFoundError:
|
||||
# print(f"Warning: Config file {self.config_file} not found. Using defaults.")
|
||||
return {}
|
||||
except json.JSONDecodeError:
|
||||
# print(f"Warning: Error parsing config file: {e}. Using defaults.")
|
||||
return {}
|
||||
|
||||
def _load_spec(self) -> Dict:
|
||||
def _load_spec(self) -> dict:
|
||||
"""Load OpenAPI specification from file."""
|
||||
with open(self.spec_file, "r") as f:
|
||||
if self.spec_file.endswith(".yaml") or self.spec_file.endswith(".yml"):
|
||||
@@ -81,7 +78,7 @@ class OpenAPIToMCPGenerator:
|
||||
else:
|
||||
return json.load(f)
|
||||
|
||||
def _get_endpoint_config(self, path: str, method: str) -> Dict:
|
||||
def _get_endpoint_config(self, path: str, method: str) -> dict:
|
||||
"""Get endpoint configuration from config file with pattern matching and inheritance.
|
||||
|
||||
Configuration resolution order (most to least specific):
|
||||
@@ -153,7 +150,7 @@ class OpenAPIToMCPGenerator:
|
||||
|
||||
return merged_config
|
||||
|
||||
def _merge_configs(self, base_config: Dict, override_config: Dict) -> Dict:
|
||||
def _merge_configs(self, base_config: dict, override_config: dict) -> dict:
|
||||
"""Merge two configurations, with override_config taking precedence.
|
||||
|
||||
Special handling for parameters: merges parameter configurations deeply.
|
||||
@@ -194,15 +191,19 @@ class OpenAPIToMCPGenerator:
|
||||
name = f"op_{name}"
|
||||
return name.lower()
|
||||
|
||||
def _get_python_type(self, schema: Dict) -> str:
|
||||
"""Convert OpenAPI schema to Python type hint."""
|
||||
def _get_python_type(self, schema: dict) -> tuple[str, str]:
|
||||
"""Convert OpenAPI schema to Python type hint.
|
||||
|
||||
Returns:
|
||||
Tuple of (type_hint, original_type) where original_type is used for casting
|
||||
"""
|
||||
if not schema:
|
||||
return "Any"
|
||||
return "Any", "any"
|
||||
|
||||
# Handle oneOf/anyOf/allOf schemas - these are typically objects
|
||||
if "oneOf" in schema or "anyOf" in schema or "allOf" in schema:
|
||||
# These are complex schemas, typically representing different object variants
|
||||
return "Dict[str, Any]"
|
||||
return "dict[str, Any] | str", "object"
|
||||
|
||||
schema_type = schema.get("type", "string")
|
||||
|
||||
@@ -210,30 +211,26 @@ class OpenAPIToMCPGenerator:
|
||||
if "enum" in schema:
|
||||
enum_values = schema["enum"]
|
||||
if all(isinstance(v, str) for v in enum_values):
|
||||
# Create Literal type for string enums
|
||||
# Create Literal type for string enums - already strings, no casting needed
|
||||
self.imports.add("from typing import Literal")
|
||||
enum_str = ", ".join(f'"{v}"' for v in enum_values)
|
||||
return f"Literal[{enum_str}]"
|
||||
return f"Literal[{enum_str}]", "string"
|
||||
else:
|
||||
return self.type_mapping.get(schema_type, "Any")
|
||||
return self.type_mapping.get(schema_type, "Any"), schema_type
|
||||
|
||||
# Handle arrays
|
||||
if schema_type == "array":
|
||||
return "str"
|
||||
return "list[Any] | str", "array"
|
||||
|
||||
# Handle format specifications
|
||||
if schema_type == "string":
|
||||
format_type = schema.get("format", "")
|
||||
if format_type in ["date", "date-time"]:
|
||||
return "str" # Keep as string for API calls
|
||||
elif format_type == "uuid":
|
||||
return "str"
|
||||
elif format_type == "email":
|
||||
return "str"
|
||||
if format_type in ["date", "date-time", "uuid", "email"]:
|
||||
return "str", "string"
|
||||
|
||||
return self.type_mapping.get(schema_type, "Any")
|
||||
return self.type_mapping.get(schema_type, "Any"), schema_type
|
||||
|
||||
def _resolve_ref(self, ref: str) -> Dict:
|
||||
def _resolve_ref(self, ref: str) -> dict:
|
||||
"""Resolve a $ref reference in the OpenAPI spec."""
|
||||
if not ref.startswith("#/"):
|
||||
return {}
|
||||
@@ -249,8 +246,8 @@ class OpenAPIToMCPGenerator:
|
||||
return resolved
|
||||
|
||||
def _extract_parameters(
|
||||
self, operation: Dict, endpoint_config: Optional[Dict] = None
|
||||
) -> List[Dict]:
|
||||
self, operation: dict, endpoint_config: Optional[dict] = None
|
||||
) -> list[dict]:
|
||||
"""Extract and process parameters from an operation."""
|
||||
parameters = []
|
||||
|
||||
@@ -264,13 +261,15 @@ class OpenAPIToMCPGenerator:
|
||||
.replace("-", "_")
|
||||
) # Also replace hyphens
|
||||
|
||||
type_hint, original_type = self._get_python_type(param.get("schema", {}))
|
||||
param_info = {
|
||||
"name": param.get("name", ""),
|
||||
"python_name": python_name,
|
||||
"in": param.get("in", "query"),
|
||||
"required": param.get("required", False),
|
||||
"description": param.get("description", ""),
|
||||
"type": self._get_python_type(param.get("schema", {})),
|
||||
"type": type_hint,
|
||||
"original_type": original_type,
|
||||
"original_schema": param.get("schema", {}),
|
||||
}
|
||||
|
||||
@@ -323,7 +322,7 @@ class OpenAPIToMCPGenerator:
|
||||
|
||||
return parameters
|
||||
|
||||
def _extract_body_parameters(self, schema: Dict, is_required: bool) -> List[Dict]:
|
||||
def _extract_body_parameters(self, schema: dict, is_required: bool) -> list[dict]:
|
||||
"""Extract individual parameters from request body schema."""
|
||||
parameters = []
|
||||
|
||||
@@ -346,6 +345,7 @@ class OpenAPIToMCPGenerator:
|
||||
# Check if this field is required
|
||||
is_field_required = prop_name in required_attrs
|
||||
|
||||
type_hint, original_type = self._get_python_type(prop_schema)
|
||||
param_info = {
|
||||
"name": prop_name, # Keep original name for API
|
||||
"python_name": python_name,
|
||||
@@ -355,7 +355,8 @@ class OpenAPIToMCPGenerator:
|
||||
"description",
|
||||
prop_schema.get("title", f"{prop_name} parameter"),
|
||||
),
|
||||
"type": self._get_python_type(prop_schema),
|
||||
"type": type_hint,
|
||||
"original_type": original_type,
|
||||
"original_schema": prop_schema,
|
||||
"resource_type": (
|
||||
data["properties"]
|
||||
@@ -383,6 +384,7 @@ class OpenAPIToMCPGenerator:
|
||||
"required": is_rel_required,
|
||||
"description": f"ID of the related {rel_name}",
|
||||
"type": "str",
|
||||
"original_type": "string",
|
||||
"original_schema": rel_schema,
|
||||
}
|
||||
parameters.append(param_info)
|
||||
@@ -396,7 +398,8 @@ class OpenAPIToMCPGenerator:
|
||||
"in": "body",
|
||||
"required": is_required,
|
||||
"description": "Request body data",
|
||||
"type": "Dict[str, Any]",
|
||||
"type": "dict[str, Any] | str",
|
||||
"original_type": "object",
|
||||
"original_schema": schema,
|
||||
}
|
||||
)
|
||||
@@ -405,11 +408,11 @@ class OpenAPIToMCPGenerator:
|
||||
|
||||
def _generate_docstring(
|
||||
self,
|
||||
operation: Dict,
|
||||
parameters: List[Dict],
|
||||
operation: dict,
|
||||
parameters: list[dict],
|
||||
path: str,
|
||||
method: str,
|
||||
endpoint_config: Optional[Dict] = None,
|
||||
endpoint_config: Optional[dict] = None,
|
||||
) -> str:
|
||||
"""Generate a comprehensive docstring for the tool function."""
|
||||
lines = []
|
||||
@@ -447,7 +450,7 @@ class OpenAPIToMCPGenerator:
|
||||
lines.append(" Args:")
|
||||
for param in parameters:
|
||||
# Use custom description if available
|
||||
param_desc = param["description"] or "No description provided"
|
||||
param_desc = param["description"] or "Self-explanatory parameter"
|
||||
|
||||
# Handle multi-line descriptions properly
|
||||
required_text = "(required)" if param["required"] else "(optional)"
|
||||
@@ -481,13 +484,13 @@ class OpenAPIToMCPGenerator:
|
||||
# Returns section
|
||||
lines.append("")
|
||||
lines.append(" Returns:")
|
||||
lines.append(" Dict containing the API response")
|
||||
lines.append(" dict containing the API response")
|
||||
|
||||
lines.append(' """')
|
||||
return "\n".join(lines)
|
||||
|
||||
def _generate_function_signature(
|
||||
self, func_name: str, parameters: List[Dict]
|
||||
self, func_name: str, parameters: list[dict]
|
||||
) -> str:
|
||||
"""Generate the function signature with proper type hints."""
|
||||
# Sort parameters: required first, then optional
|
||||
@@ -506,12 +509,38 @@ class OpenAPIToMCPGenerator:
|
||||
|
||||
if param_strings:
|
||||
params_str = ",\n".join(param_strings)
|
||||
return f"async def {func_name}(\n{params_str}\n) -> Dict[str, Any]:"
|
||||
return f"async def {func_name}(\n{params_str}\n) -> dict[str, Any]:"
|
||||
else:
|
||||
return f"async def {func_name}() -> Dict[str, Any]:"
|
||||
return f"async def {func_name}() -> dict[str, Any]:"
|
||||
|
||||
def _get_cast_expression(self, param: dict) -> str:
|
||||
"""Generate type casting expression for a parameter.
|
||||
|
||||
Args:
|
||||
param: Parameter dict with 'python_name' and 'original_type'
|
||||
|
||||
Returns:
|
||||
Expression string that casts the parameter value to the correct type
|
||||
"""
|
||||
python_name = param["python_name"]
|
||||
original_type = param.get("original_type", "string")
|
||||
|
||||
if original_type == "integer":
|
||||
return f"int({python_name}) if isinstance({python_name}, str) else {python_name}"
|
||||
elif original_type == "number":
|
||||
return f"float({python_name}) if isinstance({python_name}, str) else {python_name}"
|
||||
elif original_type == "boolean":
|
||||
return f"({python_name}.lower() in ['true', '1', 'yes'] if isinstance({python_name}, str) else bool({python_name}))"
|
||||
elif original_type == "array":
|
||||
return f"json.loads({python_name}) if isinstance({python_name}, str) else {python_name}"
|
||||
elif original_type == "object":
|
||||
return f"json.loads({python_name}) if isinstance({python_name}, str) else {python_name}"
|
||||
else:
|
||||
# string or any other type - no casting needed
|
||||
return python_name
|
||||
|
||||
def _generate_function_body(
|
||||
self, path: str, method: str, parameters: List[Dict], operation_id: str
|
||||
self, path: str, method: str, parameters: list[dict], operation_id: str
|
||||
) -> str:
|
||||
"""Generate the function body for making API calls."""
|
||||
lines = []
|
||||
@@ -529,26 +558,28 @@ class OpenAPIToMCPGenerator:
|
||||
path_params = [p for p in parameters if p["in"] == "path"]
|
||||
body_params = [p for p in parameters if p["in"] == "body"]
|
||||
|
||||
# Add json import if needed for object or array type casting
|
||||
if any(p.get("original_type") in ["object", "array"] for p in parameters):
|
||||
self.imports.add("import json")
|
||||
|
||||
# Build query parameters
|
||||
if query_params:
|
||||
lines.append(" params = {}")
|
||||
for param in query_params:
|
||||
cast_expr = self._get_cast_expression(param)
|
||||
if param["required"]:
|
||||
lines.append(
|
||||
f" params['{param['name']}'] = {param['python_name']}"
|
||||
)
|
||||
lines.append(f" params['{param['name']}'] = {cast_expr}")
|
||||
else:
|
||||
lines.append(f" if {param['python_name']} is not None:")
|
||||
lines.append(
|
||||
f" params['{param['name']}'] = {param['python_name']}"
|
||||
)
|
||||
lines.append(f" params['{param['name']}'] = {cast_expr}")
|
||||
lines.append("")
|
||||
|
||||
# Build path with path parameters
|
||||
final_path = path
|
||||
for param in path_params:
|
||||
cast_expr = self._get_cast_expression(param)
|
||||
lines.append(
|
||||
f" path = '{path}'.replace('{{{param['name']}}}', str({param['python_name']}))"
|
||||
f" path = '{path}'.replace('{{{param['name']}}}', str({cast_expr}))"
|
||||
)
|
||||
final_path = "path"
|
||||
|
||||
@@ -556,8 +587,9 @@ class OpenAPIToMCPGenerator:
|
||||
if body_params:
|
||||
# Check if we have individual params or a single body param
|
||||
if len(body_params) == 1 and body_params[0]["python_name"] == "body":
|
||||
# Single body parameter - use it directly
|
||||
lines.append(" request_body = body")
|
||||
# Single body parameter - use it directly with casting
|
||||
cast_expr = self._get_cast_expression(body_params[0])
|
||||
lines.append(f" request_body = {cast_expr}")
|
||||
else:
|
||||
# Get resource type from first body param (they should all have the same)
|
||||
resource_type = (
|
||||
@@ -598,16 +630,17 @@ class OpenAPIToMCPGenerator:
|
||||
lines.append("")
|
||||
lines.append(" # Add attributes")
|
||||
for param in attribute_params:
|
||||
cast_expr = self._get_cast_expression(param)
|
||||
if param["required"]:
|
||||
lines.append(
|
||||
f' request_body["data"]["attributes"]["{param["name"]}"] = {param["python_name"]}'
|
||||
f' request_body["data"]["attributes"]["{param["name"]}"] = {cast_expr}'
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
f" if {param['python_name']} is not None:"
|
||||
)
|
||||
lines.append(
|
||||
f' request_body["data"]["attributes"]["{param["name"]}"] = {param["python_name"]}'
|
||||
f' request_body["data"]["attributes"]["{param["name"]}"] = {cast_expr}'
|
||||
)
|
||||
|
||||
if relationship_params:
|
||||
@@ -616,15 +649,14 @@ class OpenAPIToMCPGenerator:
|
||||
lines.append(' request_body["data"]["relationships"] = {}')
|
||||
for param in relationship_params:
|
||||
rel_name = param["python_name"].replace("_id", "")
|
||||
cast_expr = self._get_cast_expression(param)
|
||||
if param["required"]:
|
||||
lines.append(
|
||||
f' request_body["data"]["relationships"]["{rel_name}"] = {{'
|
||||
)
|
||||
lines.append(' "data": {')
|
||||
lines.append(f' "type": "{rel_name}s",')
|
||||
lines.append(
|
||||
f' "id": {param["python_name"]}'
|
||||
)
|
||||
lines.append(f' "id": {cast_expr}')
|
||||
lines.append(" }")
|
||||
lines.append(" }")
|
||||
else:
|
||||
@@ -636,24 +668,22 @@ class OpenAPIToMCPGenerator:
|
||||
)
|
||||
lines.append(' "data": {')
|
||||
lines.append(f' "type": "{rel_name}s",')
|
||||
lines.append(
|
||||
f' "id": {param["python_name"]}'
|
||||
)
|
||||
lines.append(f' "id": {cast_expr}')
|
||||
lines.append(" }")
|
||||
lines.append(" }")
|
||||
lines.append("")
|
||||
|
||||
# Prepare HTTP client call
|
||||
lines.append(" async with httpx.AsyncClient() as client:")
|
||||
# Build the request URL
|
||||
url_line = (
|
||||
f'f"{{auth_manager.base_url}}{{{final_path}}}"'
|
||||
if final_path == "path"
|
||||
else f'f"{{auth_manager.base_url}}{path}"'
|
||||
)
|
||||
lines.append(f" url = {url_line}")
|
||||
lines.append("")
|
||||
|
||||
# Build the request
|
||||
request_params = [
|
||||
(
|
||||
f'f"{{auth_manager.base_url}}{{{final_path}}}"'
|
||||
if final_path == "path"
|
||||
else f'f"{{auth_manager.base_url}}{path}"'
|
||||
)
|
||||
]
|
||||
# Build request parameters
|
||||
request_params = ["url"]
|
||||
|
||||
if self.custom_auth_module:
|
||||
request_params.append("headers=auth_manager.get_headers(token)")
|
||||
@@ -664,24 +694,21 @@ class OpenAPIToMCPGenerator:
|
||||
if body_params:
|
||||
request_params.append("json=request_body")
|
||||
|
||||
request_params.append("timeout=30.0")
|
||||
params_str = ",\n ".join(request_params)
|
||||
|
||||
params_str = ",\n ".join(request_params)
|
||||
|
||||
lines.append(f" response = await client.{method}(")
|
||||
lines.append(f" {params_str}")
|
||||
lines.append(" )")
|
||||
lines.append(" response.raise_for_status()")
|
||||
lines.append(f" response = await prowler_app_client.{method}(")
|
||||
lines.append(f" {params_str}")
|
||||
lines.append(" )")
|
||||
lines.append(" response.raise_for_status()")
|
||||
lines.append("")
|
||||
|
||||
# Parse response
|
||||
lines.append(" data = response.json()")
|
||||
lines.append(" data = response.json()")
|
||||
lines.append("")
|
||||
lines.append(" return {")
|
||||
lines.append(' "success": True,')
|
||||
lines.append(' "data": data.get("data", data),')
|
||||
lines.append(' "meta": data.get("meta", {})')
|
||||
lines.append(" }")
|
||||
lines.append(" return {")
|
||||
lines.append(' "success": True,')
|
||||
lines.append(' "data": data.get("data", data),')
|
||||
lines.append(" }")
|
||||
lines.append("")
|
||||
|
||||
# Exception handling
|
||||
@@ -695,7 +722,7 @@ class OpenAPIToMCPGenerator:
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _should_exclude_endpoint(self, path: str, operation: Dict) -> bool:
|
||||
def _should_exclude_endpoint(self, path: str, operation: dict) -> bool:
|
||||
"""
|
||||
Determine if an endpoint should be excluded from generation.
|
||||
|
||||
@@ -730,7 +757,6 @@ class OpenAPIToMCPGenerator:
|
||||
|
||||
# Check excluded tags
|
||||
if any(tag in self.exclude_tags for tag in tags):
|
||||
logger.debug(f"Excluding endpoint {path} due to tag {tags}")
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -750,7 +776,7 @@ class OpenAPIToMCPGenerator:
|
||||
output_lines.append("")
|
||||
|
||||
# Add imports
|
||||
self.imports.add("from typing import Dict, Any, Optional")
|
||||
self.imports.add("from typing import Any, Optional")
|
||||
self.imports.add("import httpx")
|
||||
self.imports.add("from fastmcp import FastMCP")
|
||||
|
||||
@@ -848,6 +874,11 @@ class OpenAPIToMCPGenerator:
|
||||
output_lines.append("# Initialize authentication manager")
|
||||
output_lines.append("auth_manager = ProwlerAppAuth()")
|
||||
output_lines.append("")
|
||||
output_lines.append("# Initialize HTTP client")
|
||||
output_lines.append("prowler_app_client = httpx.AsyncClient(")
|
||||
output_lines.append(" timeout=30.0,")
|
||||
output_lines.append(")")
|
||||
output_lines.append("")
|
||||
|
||||
# Write tools grouped by tag
|
||||
for tag, tools in tools_by_tag.items():
|
||||
@@ -867,45 +898,6 @@ class OpenAPIToMCPGenerator:
|
||||
"""Save the generated code to a file."""
|
||||
generated_code = self.generate_tools()
|
||||
Path(output_file).write_text(generated_code)
|
||||
# print(f"Generated FastMCP server saved to: {output_file}")
|
||||
|
||||
# # Report statistics
|
||||
# paths = self.spec.get("paths", {})
|
||||
# total_endpoints = sum(
|
||||
# len(
|
||||
# [m for m in ["get", "post", "put", "patch", "delete"] if m in path_item]
|
||||
# )
|
||||
# for path_item in paths.values()
|
||||
# )
|
||||
|
||||
# # Count excluded endpoints by reason
|
||||
# excluded_count = 0
|
||||
# deprecated_count = 0
|
||||
# for path, path_item in paths.items():
|
||||
# for method in ["get", "post", "put", "patch", "delete"]:
|
||||
# if method in path_item:
|
||||
# operation = path_item[method]
|
||||
# if operation.get("deprecated", False):
|
||||
# deprecated_count += 1
|
||||
# if self._should_exclude_endpoint(path, operation):
|
||||
# excluded_count += 1
|
||||
|
||||
# generated_count = total_endpoints - excluded_count
|
||||
# print(f"Total endpoints in spec: {total_endpoints}")
|
||||
# print(f"Endpoints excluded: {excluded_count}")
|
||||
# if deprecated_count > 0:
|
||||
# print(f" - Deprecated: {deprecated_count}")
|
||||
# print(f"Endpoints generated: {generated_count}")
|
||||
|
||||
# Show exclusion rules if any
|
||||
# if self.exclude_patterns:
|
||||
# # print(f"Excluded patterns: {self.exclude_patterns}")
|
||||
# if self.exclude_operations:
|
||||
# # print(f"Excluded operations: {self.exclude_operations}")
|
||||
# if self.exclude_tags:
|
||||
# # print(f"Excluded tags: {self.exclude_tags}")
|
||||
# if self.include_only_tags:
|
||||
# # print(f"Including only tags: {self.include_only_tags}")
|
||||
|
||||
|
||||
def generate_server_file():
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user