diff --git a/.env b/.env index 186ae3a6d0..00d48ed6a2 100644 --- a/.env +++ b/.env @@ -10,13 +10,16 @@ NEXT_PUBLIC_API_BASE_URL=${API_BASE_URL} NEXT_PUBLIC_API_DOCS_URL=http://prowler-api:8080/api/v1/docs AUTH_TRUST_HOST=true UI_PORT=3000 -# Temp URL for feeds need to use actual -RSS_FEED_URL=https://prowler.com/blog/rss # openssl rand -base64 32 AUTH_SECRET="N/c6mnaS5+SWq81+819OrzQZlmx1Vxtp/orjttJSmw8=" # Google Tag Manager ID NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID="" +#### Code Review Configuration #### +# Enable Claude Code standards validation on pre-push hook +# Set to 'true' to validate changes against AGENTS.md standards via Claude Code +# Set to 'false' to skip validation +CODE_REVIEW_ENABLED=true #### Prowler API Configuration #### PROWLER_API_VERSION="stable" @@ -29,6 +32,14 @@ POSTGRES_ADMIN_PASSWORD=postgres POSTGRES_USER=prowler POSTGRES_PASSWORD=postgres POSTGRES_DB=prowler_db +# Read replica settings (optional) +# POSTGRES_REPLICA_HOST=postgres-db +# POSTGRES_REPLICA_PORT=5432 +# POSTGRES_REPLICA_USER=prowler +# POSTGRES_REPLICA_PASSWORD=postgres +# POSTGRES_REPLICA_DB=prowler_db +# POSTGRES_REPLICA_MAX_ATTEMPTS=3 +# POSTGRES_REPLICA_RETRY_BASE_DELAY=0.5 # Celery-Prowler task settings TASK_RETRY_DELAY_SECONDS=0.1 @@ -97,6 +108,8 @@ DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute # Sentry settings SENTRY_ENVIRONMENT=local SENTRY_RELEASE=local +NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT} + #### Prowler release version #### NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.12.2 @@ -118,3 +131,12 @@ LANGSMITH_TRACING=false LANGSMITH_ENDPOINT="https://api.smith.langchain.com" LANGSMITH_API_KEY="" LANGCHAIN_PROJECT="" + +# RSS Feed Configuration +# Multiple feed sources can be configured as a JSON array (must be valid JSON, no trailing commas) +# Each source requires: id, name, type (github_releases|blog|custom), url, and enabled flag +# IMPORTANT: Must be a single line with valid JSON (no newlines, no trailing commas) +# Example with one source: +RSS_FEED_SOURCES='[{"id":"prowler-releases","name":"Prowler Releases","type":"github_releases","url":"https://github.com/prowler-cloud/prowler/releases.atom","enabled":true}]' +# Example with multiple sources (no trailing comma after last item): +# RSS_FEED_SOURCES='[{"id":"prowler-releases","name":"Prowler Releases","type":"github_releases","url":"https://github.com/prowler-cloud/prowler/releases.atom","enabled":true},{"id":"prowler-blog","name":"Prowler Blog","type":"blog","url":"https://prowler.com/blog/rss","enabled":false}]' diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 74e9332e34..b953610fa1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1,28 @@ +# SDK /* @prowler-cloud/sdk -/.github/ @prowler-cloud/sdk -prowler @prowler-cloud/sdk @prowler-cloud/detection-and-remediation -tests @prowler-cloud/sdk @prowler-cloud/detection-and-remediation -api @prowler-cloud/api -ui @prowler-cloud/ui \ No newline at end of file +/prowler/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation +/tests/ @prowler-cloud/sdk @prowler-cloud/detection-and-remediation +/dashboard/ @prowler-cloud/sdk +/docs/ @prowler-cloud/sdk +/examples/ @prowler-cloud/sdk +/util/ @prowler-cloud/sdk +/contrib/ @prowler-cloud/sdk +/permissions/ @prowler-cloud/sdk +/codecov.yml @prowler-cloud/sdk @prowler-cloud/api + +# API +/api/ @prowler-cloud/api + +# UI +/ui/ @prowler-cloud/ui + +# AI +/mcp_server/ @prowler-cloud/ai + +# Platform +/.github/ @prowler-cloud/platform +/Makefile @prowler-cloud/platform +/kubernetes/ @prowler-cloud/platform +**/Dockerfile* @prowler-cloud/platform +**/docker-compose*.yml @prowler-cloud/platform +**/docker-compose*.yaml @prowler-cloud/platform diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index fa4c805d85..fcba29ca1f 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -3,6 +3,41 @@ description: Create a report to help us improve labels: ["bug", "status/needs-triage"] body: + - type: checkboxes + id: search + attributes: + label: Issue search + options: + - label: I have searched the existing issues and this bug has not been reported yet + required: true + - type: dropdown + id: component + attributes: + label: Which component is affected? + multiple: true + options: + - Prowler CLI/SDK + - Prowler API + - Prowler UI + - Prowler Dashboard + - Prowler MCP Server + - Documentation + - Other + validations: + required: true + - type: dropdown + id: provider + attributes: + label: Cloud Provider (if applicable) + multiple: true + options: + - AWS + - Azure + - GCP + - Kubernetes + - GitHub + - Microsoft 365 + - Not applicable - type: textarea id: reproduce attributes: @@ -78,6 +113,15 @@ body: prowler --version validations: required: true + - type: input + id: python-version + attributes: + label: Python version + description: Which Python version are you using? + placeholder: |- + python --version + validations: + required: true - type: input id: pip-version attributes: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 3ba13e0cec..6e7c21013b 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1 +1,11 @@ blank_issues_enabled: false +contact_links: + - name: 📖 Documentation + url: https://docs.prowler.com + about: Check our comprehensive documentation for guides and tutorials + - name: đŸ’Ŧ GitHub Discussions + url: https://github.com/prowler-cloud/prowler/discussions + about: Ask questions and discuss with the community + - name: 🌟 Prowler Community + url: https://goto.prowler.com/slack + about: Join our community for support and updates diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index 528723b717..3e3510496f 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -3,6 +3,42 @@ description: Suggest an idea for this project labels: ["feature-request", "status/needs-triage"] body: + - type: checkboxes + id: search + attributes: + label: Feature search + options: + - label: I have searched the existing issues and this feature has not been requested yet or is already in our [Public Roadmap](https://roadmap.prowler.com/roadmap) + required: true + - type: dropdown + id: component + attributes: + label: Which component would this feature affect? + multiple: true + options: + - Prowler CLI/SDK + - Prowler API + - Prowler UI + - Prowler Dashboard + - Prowler MCP Server + - Documentation + - New component/Integration + validations: + required: true + - type: dropdown + id: provider + attributes: + label: Related to specific cloud provider? + multiple: true + options: + - AWS + - Azure + - GCP + - Kubernetes + - GitHub + - Microsoft 365 + - All providers + - Not provider-specific - type: textarea id: Problem attributes: @@ -19,6 +55,14 @@ body: description: A clear and concise description of what you want to happen. validations: required: true + - type: textarea + id: use-case + attributes: + label: Use case and benefits + description: Who would benefit from this feature and how? + placeholder: This would help security teams by... + validations: + required: true - type: textarea id: Alternatives attributes: diff --git a/.github/actions/setup-python-poetry/action.yml b/.github/actions/setup-python-poetry/action.yml new file mode 100644 index 0000000000..790f3ef0e6 --- /dev/null +++ b/.github/actions/setup-python-poetry/action.yml @@ -0,0 +1,93 @@ +name: 'Setup Python with Poetry' +description: 'Setup Python environment with Poetry and install dependencies' +author: 'Prowler' + +inputs: + python-version: + description: 'Python version to use' + required: true + working-directory: + description: 'Working directory for Poetry' + required: false + default: '.' + poetry-version: + description: 'Poetry version to install' + required: false + default: '2.1.1' + install-dependencies: + description: 'Install Python dependencies with Poetry' + required: false + default: 'true' + +runs: + using: 'composite' + steps: + - name: Replace @master with current branch in pyproject.toml (prowler repo only) + if: github.event_name == 'pull_request' && github.base_ref == 'master' && github.repository == 'prowler-cloud/prowler' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + BRANCH_NAME="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" + echo "Using branch: $BRANCH_NAME" + sed -i "s|@master|@$BRANCH_NAME|g" pyproject.toml + + - name: Install poetry + shell: bash + run: | + python -m pip install --upgrade pip + pipx install poetry==${{ inputs.poetry-version }} + + - name: Update poetry.lock with latest Prowler commit + if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha') + echo "Latest commit hash: $LATEST_COMMIT" + sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / { + s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/ + }' poetry.lock + echo "Updated resolved_reference:" + grep -A2 -B2 "resolved_reference" poetry.lock + + - name: Update SDK resolved_reference to latest commit (prowler repo on push) + if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha') + echo "Latest commit hash: $LATEST_COMMIT" + sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / { + s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/ + }' poetry.lock + echo "Updated resolved_reference:" + grep -A2 -B2 "resolved_reference" poetry.lock + + - name: Update poetry.lock (prowler repo only) + if: github.repository == 'prowler-cloud/prowler' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: poetry lock + + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ inputs.python-version }} + cache: 'poetry' + cache-dependency-path: ${{ inputs.working-directory }}/poetry.lock + + - name: Install Python dependencies + if: inputs.install-dependencies == 'true' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + poetry install --no-root + poetry run pip list + + - name: Update Prowler Cloud API Client + if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + poetry remove prowler-cloud-api-client + poetry add ./prowler-cloud-api-client diff --git a/.github/actions/slack-notification/README.md b/.github/actions/slack-notification/README.md new file mode 100644 index 0000000000..ce8843df2a --- /dev/null +++ b/.github/actions/slack-notification/README.md @@ -0,0 +1,198 @@ +# Slack Notification Action + +A generic and flexible GitHub composite action for sending Slack notifications using JSON template files. Supports both standalone messages and message updates, with automatic status detection. + +## Features + +- **Template-based**: All messages use JSON template files for consistency +- **Automatic status detection**: Pass `step-outcome` to auto-calculate success/failure +- **Message updates**: Supports updating existing messages (using `chat.update`) +- **Simple API**: Clean and minimal interface +- **Reusable**: Use across all workflows and scenarios +- **Maintainable**: Centralized message templates + +## Use Cases + +1. **Container releases**: Track push start and completion with automatic status +2. **Deployments**: Track deployment progress with rich Block Kit formatting +3. **Custom notifications**: Any scenario where you need to notify Slack + +## Inputs + +| Input | Description | Required | Default | +|-------|-------------|----------|---------| +| `slack-bot-token` | Slack bot token for authentication | Yes | - | +| `payload-file-path` | Path to JSON file with the Slack message payload | Yes | - | +| `update-ts` | Message timestamp to update (leave empty for new messages) | No | `''` | +| `step-outcome` | Step outcome for automatic status detection (sets STATUS_EMOJI and STATUS_TEXT env vars) | No | `''` | + +## Outputs + +| Output | Description | +|--------|-------------| +| `ts` | Timestamp of the Slack message (use for updates) | + +## Usage Examples + +### Example 1: Container Release with Automatic Status Detection + +Using JSON template files with automatic status detection: + +```yaml +# Send start notification +- name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + +# Do the work +- name: Build and push container + if: github.event_name == 'release' + id: container-push + uses: docker/build-push-action@... + with: + push: true + tags: ... + +# Send completion notification with automatic status detection +- name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} +``` + +**Benefits:** +- No status calculation needed in workflow +- Reusable template files +- Clean and concise +- Automatic `STATUS_EMOJI` and `STATUS_TEXT` env vars set by action +- Consistent message format across all workflows + +### Example 2: Deployment with Message Update Pattern + +```yaml +# Send initial deployment message +- name: Notify deployment started + id: slack-start + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + ENVIRONMENT: PRODUCTION + COMMIT_HASH: ${{ github.sha }} + VERSION_DEPLOYED: latest + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/deployment-started.json" + +# Run deployment +- name: Deploy + id: deploy + run: terraform apply -auto-approve + +# Determine additional status variables +- name: Determine deployment status + if: always() + id: deploy-status + run: | + if [[ "${{ steps.deploy.outcome }}" == "success" ]]; then + echo "STATUS_COLOR=28a745" >> $GITHUB_ENV + echo "STATUS=Completed" >> $GITHUB_ENV + else + echo "STATUS_COLOR=fc3434" >> $GITHUB_ENV + echo "STATUS=Failed" >> $GITHUB_ENV + fi + +# Update the same message with final status +- name: Update deployment notification + if: always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + MESSAGE_TS: ${{ steps.slack-start.outputs.ts }} + COMPONENT: API + ENVIRONMENT: PRODUCTION + COMMIT_HASH: ${{ github.sha }} + VERSION_DEPLOYED: latest + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + STATUS: ${{ env.STATUS }} + STATUS_COLOR: ${{ env.STATUS_COLOR }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + update-ts: ${{ steps.slack-start.outputs.ts }} + payload-file-path: "./.github/scripts/slack-messages/deployment-completed.json" + step-outcome: ${{ steps.deploy.outcome }} +``` + +## Automatic Status Detection + +When you provide `step-outcome` input, the action automatically sets these environment variables: + +| Outcome | STATUS_EMOJI | STATUS_TEXT | +|---------|--------------|-------------| +| success | `[✓]` | `completed successfully!` | +| failure | `[✗]` | `failed` | + +These variables are then available in your payload template files. + +## Template File Format + +All template files must be valid JSON and support environment variable substitution. Example: + +```json +{ + "channel": "$SLACK_CHANNEL_ID", + "text": "$STATUS_EMOJI $COMPONENT container release $RELEASE_TAG push $STATUS_TEXT <$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID|View run>" +} +``` + +See available templates in [`.github/scripts/slack-messages/`](../../scripts/slack-messages/). + +## Requirements + +- Slack Bot Token with scopes: `chat:write`, `chat:write.public` +- Slack Channel ID where messages will be posted +- JSON template files for your messages + +## Benefits + +- **Consistency**: All notifications use standardized templates +- **Automatic status handling**: No need to calculate success/failure in workflows +- **Clean workflows**: Minimal boilerplate code +- **Reusable templates**: One template for all components +- **Easy to maintain**: Change template once, applies everywhere +- **Version controlled**: All message formats in git + +## Related Resources + +- [Slack Block Kit Builder](https://app.slack.com/block-kit-builder) +- [Slack API Method Documentation](https://docs.slack.dev/tools/slack-github-action/sending-techniques/sending-data-slack-api-method/) +- [Message templates documentation](../../scripts/slack-messages/README.md) diff --git a/.github/actions/slack-notification/action.yml b/.github/actions/slack-notification/action.yml new file mode 100644 index 0000000000..1427fed3e0 --- /dev/null +++ b/.github/actions/slack-notification/action.yml @@ -0,0 +1,74 @@ +name: 'Slack Notification' +description: 'Generic action to send Slack notifications with optional message updates and automatic status detection' +inputs: + slack-bot-token: + description: 'Slack bot token for authentication' + required: true + payload-file-path: + description: 'Path to JSON file with the Slack message payload' + required: true + update-ts: + description: 'Message timestamp to update (only for updates, leave empty for new messages)' + required: false + default: '' + step-outcome: + description: 'Outcome of a step to determine status (success/failure) - automatically sets STATUS_TEXT and STATUS_COLOR env vars' + required: false + default: '' +outputs: + ts: + description: 'Timestamp of the Slack message' + value: ${{ steps.slack-notification.outputs.ts }} +runs: + using: 'composite' + steps: + - name: Determine status + id: status + shell: bash + run: | + if [[ "${{ inputs.step-outcome }}" == "success" ]]; then + echo "STATUS_TEXT=Completed" >> $GITHUB_ENV + echo "STATUS_COLOR=#6aa84f" >> $GITHUB_ENV + elif [[ "${{ inputs.step-outcome }}" == "failure" ]]; then + echo "STATUS_TEXT=Failed" >> $GITHUB_ENV + echo "STATUS_COLOR=#fc3434" >> $GITHUB_ENV + else + # No outcome provided - pending/in progress state + echo "STATUS_COLOR=#dbab09" >> $GITHUB_ENV + fi + + - name: Send Slack notification (new message) + if: inputs.update-ts == '' + id: slack-notification-post + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_PAYLOAD_FILE_PATH: ${{ inputs.payload-file-path }} + with: + method: chat.postMessage + token: ${{ inputs.slack-bot-token }} + payload-file-path: ${{ inputs.payload-file-path }} + payload-templated: true + errors: true + + - name: Update Slack notification + if: inputs.update-ts != '' + id: slack-notification-update + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_PAYLOAD_FILE_PATH: ${{ inputs.payload-file-path }} + with: + method: chat.update + token: ${{ inputs.slack-bot-token }} + payload-file-path: ${{ inputs.payload-file-path }} + payload-templated: true + errors: true + + - name: Set output + id: slack-notification + shell: bash + run: | + if [[ "${{ inputs.update-ts }}" == "" ]]; then + echo "ts=${{ steps.slack-notification-post.outputs.ts }}" >> $GITHUB_OUTPUT + else + echo "ts=${{ inputs.update-ts }}" >> $GITHUB_OUTPUT + fi diff --git a/.github/actions/trivy-scan/action.yml b/.github/actions/trivy-scan/action.yml new file mode 100644 index 0000000000..e3c2404748 --- /dev/null +++ b/.github/actions/trivy-scan/action.yml @@ -0,0 +1,164 @@ +name: 'Container Security Scan with Trivy' +description: 'Scans container images for vulnerabilities using Trivy and reports results' +author: 'Prowler' + +inputs: + image-name: + description: 'Container image name to scan' + required: true + image-tag: + description: 'Container image tag to scan' + required: true + default: ${{ github.sha }} + severity: + description: 'Severities to scan for (comma-separated)' + required: false + default: 'CRITICAL,HIGH,MEDIUM,LOW' + fail-on-critical: + description: 'Fail the build if critical vulnerabilities are found' + required: false + default: 'false' + upload-sarif: + description: 'Upload results to GitHub Security tab' + required: false + default: 'true' + create-pr-comment: + description: 'Create a comment on the PR with scan results' + required: false + default: 'true' + artifact-retention-days: + description: 'Days to retain the Trivy report artifact' + required: false + default: '2' + +outputs: + critical-count: + description: 'Number of critical vulnerabilities found' + value: ${{ steps.security-check.outputs.critical }} + high-count: + description: 'Number of high vulnerabilities found' + value: ${{ steps.security-check.outputs.high }} + total-count: + description: 'Total number of vulnerabilities found' + value: ${{ steps.security-check.outputs.total }} + +runs: + using: 'composite' + steps: + - name: Cache Trivy vulnerability database + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.cache/trivy + key: trivy-db-${{ runner.os }}-${{ github.run_id }} + restore-keys: | + trivy-db-${{ runner.os }}- + + - name: Run Trivy vulnerability scan (JSON) + uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 + with: + image-ref: ${{ inputs.image-name }}:${{ inputs.image-tag }} + format: 'json' + output: 'trivy-report.json' + severity: ${{ inputs.severity }} + exit-code: '0' + scanners: 'vuln' + timeout: '5m' + + - name: Run Trivy vulnerability scan (SARIF) + if: inputs.upload-sarif == 'true' && github.event_name == 'push' + uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 + with: + image-ref: ${{ inputs.image-name }}:${{ inputs.image-tag }} + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + exit-code: '0' + scanners: 'vuln' + timeout: '5m' + + - name: Upload Trivy results to GitHub Security tab + if: inputs.upload-sarif == 'true' && github.event_name == 'push' + uses: github/codeql-action/upload-sarif@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + with: + sarif_file: 'trivy-results.sarif' + category: 'trivy-container' + + - name: Upload Trivy report artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: trivy-scan-report-${{ inputs.image-name }} + path: trivy-report.json + retention-days: ${{ inputs.artifact-retention-days }} + + - name: Generate security summary + id: security-check + shell: bash + run: | + CRITICAL=$(jq '[.Results[]?.Vulnerabilities[]? | select(.Severity=="CRITICAL")] | length' trivy-report.json) + HIGH=$(jq '[.Results[]?.Vulnerabilities[]? | select(.Severity=="HIGH")] | length' trivy-report.json) + TOTAL=$(jq '[.Results[]?.Vulnerabilities[]?] | length' trivy-report.json) + + echo "critical=$CRITICAL" >> $GITHUB_OUTPUT + echo "high=$HIGH" >> $GITHUB_OUTPUT + echo "total=$TOTAL" >> $GITHUB_OUTPUT + + echo "### 🔒 Container Security Scan" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Image:** \`${{ inputs.image-name }}:${{ inputs.image-tag }}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- 🔴 Critical: $CRITICAL" >> $GITHUB_STEP_SUMMARY + echo "- 🟠 High: $HIGH" >> $GITHUB_STEP_SUMMARY + echo "- **Total**: $TOTAL" >> $GITHUB_STEP_SUMMARY + + - name: Comment scan results on PR + if: inputs.create-pr-comment == 'true' && github.event_name == 'pull_request' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + IMAGE_NAME: ${{ inputs.image-name }} + GITHUB_SHA: ${{ inputs.image-tag }} + SEVERITY: ${{ inputs.severity }} + with: + script: | + const comment = require('./.github/scripts/trivy-pr-comment.js'); + + // Unique identifier to find our comment + const marker = ''; + const body = marker + '\n' + comment; + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const existingComment = comments.find(c => c.body?.includes(marker)); + + if (existingComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body: body + }); + console.log('✅ Updated existing Trivy scan comment'); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + console.log('✅ Created new Trivy scan comment'); + } + + - name: Check for critical vulnerabilities + if: inputs.fail-on-critical == 'true' && steps.security-check.outputs.critical != '0' + shell: bash + run: | + echo "::error::Found ${{ steps.security-check.outputs.critical }} critical vulnerabilities" + echo "::warning::Please update packages or use a different base image" + exit 1 diff --git a/.github/codeql/api-codeql-config.yml b/.github/codeql/api-codeql-config.yml index 9ce26a3651..0925ea4a33 100644 --- a/.github/codeql/api-codeql-config.yml +++ b/.github/codeql/api-codeql-config.yml @@ -1,3 +1,12 @@ -name: "API - CodeQL Config" +name: 'API: CodeQL Config' paths: - - "api/" + - 'api/' + +paths-ignore: + - 'api/tests/**' + - 'api/**/__pycache__/**' + - 'api/**/migrations/**' + - 'api/**/*.md' + +queries: + - uses: security-and-quality diff --git a/.github/codeql/sdk-codeql-config.yml b/.github/codeql/sdk-codeql-config.yml index 7982398f42..4c933ecbda 100644 --- a/.github/codeql/sdk-codeql-config.yml +++ b/.github/codeql/sdk-codeql-config.yml @@ -1,4 +1,18 @@ -name: "SDK - CodeQL Config" +name: 'SDK: CodeQL Config' +paths: + - 'prowler/' + paths-ignore: - - "api/" - - "ui/" + - 'api/' + - 'ui/' + - 'dashboard/' + - 'mcp_server/' + - 'tests/**' + - 'util/**' + - 'contrib/**' + - 'examples/**' + - 'prowler/**/__pycache__/**' + - 'prowler/**/*.md' + +queries: + - uses: security-and-quality diff --git a/.github/codeql/ui-codeql-config.yml b/.github/codeql/ui-codeql-config.yml index fa4f80cae5..2eb4eebe87 100644 --- a/.github/codeql/ui-codeql-config.yml +++ b/.github/codeql/ui-codeql-config.yml @@ -1,3 +1,17 @@ -name: "UI - CodeQL Config" +name: 'UI: CodeQL Config' paths: - - "ui/" + - 'ui/' + +paths-ignore: + - 'ui/node_modules/**' + - 'ui/.next/**' + - 'ui/out/**' + - 'ui/tests/**' + - 'ui/**/*.test.ts' + - 'ui/**/*.test.tsx' + - 'ui/**/*.spec.ts' + - 'ui/**/*.spec.tsx' + - 'ui/**/*.md' + +queries: + - uses: security-and-quality diff --git a/.github/labeler.yml b/.github/labeler.yml index 4985c78f22..9a56691628 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -42,6 +42,11 @@ provider/mongodbatlas: - any-glob-to-any-file: "prowler/providers/mongodbatlas/**" - any-glob-to-any-file: "tests/providers/mongodbatlas/**" +provider/oci: + - changed-files: + - any-glob-to-any-file: "prowler/providers/oraclecloud/**" + - any-glob-to-any-file: "tests/providers/oraclecloud/**" + github_actions: - changed-files: - any-glob-to-any-file: ".github/workflows/*" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 94122d8349..497c4d6214 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -22,6 +22,13 @@ Please add a detailed description of how to review this PR. - [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md) - [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable. +#### UI +- [ ] All issue/task requirements work as expected on the UI +- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px) +- [ ] Screenshots/Video of the functionality flow (if applicable) - Table (640px > X < 1024px) +- [ ] Screenshots/Video of the functionality flow (if applicable) - Desktop (X > 1024px) +- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable. + #### API - [ ] Verify if API specs need to be regenerated. - [ ] Check if version updates are required (e.g., specs, Poetry, etc.). diff --git a/.github/scripts/slack-messages/README.md b/.github/scripts/slack-messages/README.md new file mode 100644 index 0000000000..e7fcf00fdd --- /dev/null +++ b/.github/scripts/slack-messages/README.md @@ -0,0 +1,462 @@ +# Slack Message Templates + +This directory contains reusable message templates for Slack notifications sent from GitHub Actions workflows. + +## Usage + +These JSON templates are used with the `slackapi/slack-github-action` using the Slack API method (`chat.postMessage` and `chat.update`). All templates support rich Block Kit formatting and message updates. + +### Available Templates + +**Container Releases** +- `container-release-started.json`: Simple one-line notification when container push starts +- `container-release-completed.json`: Simple one-line notification when container release completes + +**Deployments** +- `deployment-started.json`: Deployment start notification with Block Kit formatting +- `deployment-completed.json`: Deployment completion notification (updates the start message) + +All templates use the Slack API method and require a Slack Bot Token. + +## Setup Requirements + +1. Create a Slack App (or use existing) +2. Add Bot Token Scopes: `chat:write`, `chat:write.public` +3. Install the app to your workspace +4. Get the Bot Token from OAuth & Permissions page +5. Add secrets: + - `SLACK_BOT_TOKEN`: Your bot token + - `SLACK_CHANNEL_ID`: The channel ID where messages will be posted + +Reference: [Sending data using a Slack API method](https://docs.slack.dev/tools/slack-github-action/sending-techniques/sending-data-slack-api-method/) + +## Environment Variables + +### Required Secrets (GitHub Secrets) +- `SLACK_BOT_TOKEN`: Passed as `token` parameter to the action (not as env variable) +- `SLACK_CHANNEL_ID`: Used in payload as env variable + +### Container Release Variables (configured as env) +- `COMPONENT`: Component name (e.g., "API", "SDK", "UI", "MCP") +- `RELEASE_TAG` / `PROWLER_VERSION`: The release tag or version being deployed +- `GITHUB_SERVER_URL`: Provided by GitHub context +- `GITHUB_REPOSITORY`: Provided by GitHub context +- `GITHUB_RUN_ID`: Provided by GitHub context +- `STATUS_EMOJI`: Status symbol (calculated: `[✓]` for success, `[✗]` for failure) +- `STATUS_TEXT`: Status text (calculated: "completed successfully!" or "failed") + +### Deployment Variables (configured as env) +- `COMPONENT`: Component name (e.g., "API", "SDK", "UI", "MCP") +- `ENVIRONMENT`: Environment name (e.g., "DEVELOPMENT", "PRODUCTION") +- `COMMIT_HASH`: Commit hash being deployed +- `VERSION_DEPLOYED`: Version being deployed +- `GITHUB_ACTOR`: User who triggered the workflow +- `GITHUB_WORKFLOW`: Workflow name +- `GITHUB_SERVER_URL`: Provided by GitHub context +- `GITHUB_REPOSITORY`: Provided by GitHub context +- `GITHUB_RUN_ID`: Provided by GitHub context + +All other variables (MESSAGE_TS, STATUS, STATUS_COLOR, STATUS_EMOJI, etc.) are calculated internally within the workflow and should NOT be configured as environment variables. + +## Example Workflow Usage + +### Using the Generic Slack Notification Action (Recommended) + +**Recommended approach**: Use the generic reusable action `.github/actions/slack-notification` which provides maximum flexibility: + +#### Example 1: Container Release (Start + Completion) + +```yaml +# Send start notification +- name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": "API container release ${{ env.RELEASE_TAG }} push started... <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + } + +# Build and push container +- name: Build and push container + if: github.event_name == 'release' + id: container-push + uses: docker/build-push-action@... + with: + push: true + tags: ... + +# Calculate status +- name: Determine push status + if: github.event_name == 'release' && always() + id: push-status + run: | + if [[ "${{ steps.container-push.outcome }}" == "success" ]]; then + echo "emoji=[✓]" >> $GITHUB_OUTPUT + echo "text=completed successfully!" >> $GITHUB_OUTPUT + else + echo "emoji=[✗]" >> $GITHUB_OUTPUT + echo "text=failed" >> $GITHUB_OUTPUT + fi + +# Send completion notification +- name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": "${{ steps.push-status.outputs.emoji }} API container release ${{ env.RELEASE_TAG }} push ${{ steps.push-status.outputs.text }} <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + } +``` + +#### Example 2: Simple One-Time Message + +```yaml +- name: Send notification + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": "Deployment completed successfully!" + } +``` + +#### Example 3: Deployment with Message Update Pattern + +```yaml +# Send initial deployment message +- name: Notify deployment started + id: slack-start + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": "API deployment to PRODUCTION started", + "attachments": [ + { + "color": "dbab09", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "API | Deployment to PRODUCTION" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Status:*\nIn Progress" + } + ] + } + ] + } + ] + } + +# Run deployment +- name: Deploy + id: deploy + run: terraform apply -auto-approve + +# Calculate status +- name: Determine status + if: always() + id: status + run: | + if [[ "${{ steps.deploy.outcome }}" == "success" ]]; then + echo "color=28a745" >> $GITHUB_OUTPUT + echo "emoji=[✓]" >> $GITHUB_OUTPUT + echo "status=Completed" >> $GITHUB_OUTPUT + else + echo "color=fc3434" >> $GITHUB_OUTPUT + echo "emoji=[✗]" >> $GITHUB_OUTPUT + echo "status=Failed" >> $GITHUB_OUTPUT + fi + +# Update the same message with final status +- name: Update deployment notification + if: always() + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + update-ts: ${{ steps.slack-start.outputs.ts }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "ts": "${{ steps.slack-start.outputs.ts }}", + "text": "${{ steps.status.outputs.emoji }} API deployment to PRODUCTION ${{ steps.status.outputs.status }}", + "attachments": [ + { + "color": "${{ steps.status.outputs.color }}", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "API | Deployment to PRODUCTION" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Status:*\n${{ steps.status.outputs.emoji }} ${{ steps.status.outputs.status }}" + } + ] + } + ] + } + ] + } +``` + +**Benefits of using the generic action:** +- Maximum flexibility: Build any payload you need directly in the workflow +- No template files needed: Everything inline +- Supports all scenarios: one-time messages, start/update patterns, rich Block Kit +- Easy to customize per use case +- Generic: Works for containers, deployments, or any notification type + +For more details, see [Slack Notification Action](../../actions/slack-notification/README.md). + +### Using Message Templates (Alternative Approach) + +Simple one-line notifications for container releases: + +```yaml +# Step 1: Notify when push starts +- name: Notify container push started + if: github.event_name == 'release' + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + +# Step 2: Build and push container +- name: Build and push container + id: container-push + uses: docker/build-push-action@... + with: + push: true + tags: ... + +# Step 3: Determine push status +- name: Determine push status + if: github.event_name == 'release' && always() + id: push-status + run: | + if [[ "${{ steps.container-push.outcome }}" == "success" ]]; then + echo "status-emoji=[✓]" >> $GITHUB_OUTPUT + echo "status-text=completed successfully!" >> $GITHUB_OUTPUT + else + echo "status-emoji=[✗]" >> $GITHUB_OUTPUT + echo "status-text=failed" >> $GITHUB_OUTPUT + fi + +# Step 4: Notify when push completes (success or failure) +- name: Notify container push completed + if: github.event_name == 'release' && always() + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + STATUS_EMOJI: ${{ steps.push-status.outputs.status-emoji }} + STATUS_TEXT: ${{ steps.push-status.outputs.status-text }} + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" +``` + +### Deployment with Update Pattern + +For deployments that start with one message and update it with the final status: + +```yaml +# Step 1: Send deployment start notification +- name: Notify Deployment Start + id: slack-notification-start + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + ENVIRONMENT: PRODUCTION + COMMIT_HASH: ${{ github.sha }} + VERSION_DEPLOYED: latest + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/deployment-started.json" + +# Step 2: Run your deployment steps +- name: Terraform Plan + id: terraform-plan + run: terraform plan + +- name: Terraform Apply + id: terraform-apply + run: terraform apply -auto-approve + +# Step 3: Determine status (calculated internally, not configured) +- name: Determine Status + if: always() + id: determine-status + run: | + if [[ "${{ steps.terraform-apply.outcome }}" == "success" ]]; then + echo "status=Completed" >> $GITHUB_OUTPUT + echo "status-color=28a745" >> $GITHUB_OUTPUT + echo "status-emoji=[✓]" >> $GITHUB_OUTPUT + echo "plan-emoji=[✓]" >> $GITHUB_OUTPUT + echo "apply-emoji=[✓]" >> $GITHUB_OUTPUT + elif [[ "${{ steps.terraform-plan.outcome }}" == "failure" || "${{ steps.terraform-apply.outcome }}" == "failure" ]]; then + echo "status=Failed" >> $GITHUB_OUTPUT + echo "status-color=fc3434" >> $GITHUB_OUTPUT + echo "status-emoji=[✗]" >> $GITHUB_OUTPUT + if [[ "${{ steps.terraform-plan.outcome }}" == "failure" ]]; then + echo "plan-emoji=[✗]" >> $GITHUB_OUTPUT + else + echo "plan-emoji=[✓]" >> $GITHUB_OUTPUT + fi + if [[ "${{ steps.terraform-apply.outcome }}" == "failure" ]]; then + echo "apply-emoji=[✗]" >> $GITHUB_OUTPUT + else + echo "apply-emoji=[✓]" >> $GITHUB_OUTPUT + fi + else + echo "status=Failed" >> $GITHUB_OUTPUT + echo "status-color=fc3434" >> $GITHUB_OUTPUT + echo "status-emoji=[✗]" >> $GITHUB_OUTPUT + echo "plan-emoji=[?]" >> $GITHUB_OUTPUT + echo "apply-emoji=[?]" >> $GITHUB_OUTPUT + fi + +# Step 4: Update the same Slack message (using calculated values) +- name: Notify Deployment Result + if: always() + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + MESSAGE_TS: ${{ steps.slack-notification-start.outputs.ts }} + COMPONENT: API + ENVIRONMENT: PRODUCTION + COMMIT_HASH: ${{ github.sha }} + VERSION_DEPLOYED: latest + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + STATUS: ${{ steps.determine-status.outputs.status }} + STATUS_COLOR: ${{ steps.determine-status.outputs.status-color }} + STATUS_EMOJI: ${{ steps.determine-status.outputs.status-emoji }} + PLAN_EMOJI: ${{ steps.determine-status.outputs.plan-emoji }} + APPLY_EMOJI: ${{ steps.determine-status.outputs.apply-emoji }} + TERRAFORM_PLAN_OUTCOME: ${{ steps.terraform-plan.outcome }} + TERRAFORM_APPLY_OUTCOME: ${{ steps.terraform-apply.outcome }} + with: + method: chat.update + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/deployment-completed.json" +``` + +**Note**: Variables like `STATUS`, `STATUS_COLOR`, `STATUS_EMOJI`, `PLAN_EMOJI`, `APPLY_EMOJI` are calculated by the `determine-status` step based on the outcomes of previous steps. They should NOT be manually configured. + +## Key Features + +### Benefits of Using Slack API Method + +- **Rich Block Kit Formatting**: Full support for Slack's Block Kit including headers, sections, fields, colors, and attachments +- **Message Updates**: Update the same message instead of posting multiple messages (using `chat.update` with `ts`) +- **Consistent Experience**: Same look and feel as Prowler Cloud notifications +- **Flexible**: Easy to customize message appearance by editing JSON templates + +### Differences from Webhook Method + +| Feature | webhook-trigger | Slack API (chat.postMessage) | +|---------|-----------------|------------------------------| +| Setup | Workflow Builder webhook | Slack Bot Token + Channel ID | +| Formatting | Plain text/simple | Full Block Kit support | +| Message Update | No | Yes (with chat.update) | +| Authentication | Webhook URL | Bot Token | +| Scopes Required | None | chat:write, chat:write.public | + +## Message Appearance + +### Container Release (Simple One-Line) + +**Start message:** +``` +API container release 4.5.0 push started... View run +``` + +**Completion message (success):** +``` +[✓] API container release 4.5.0 push completed successfully! View run +``` + +**Completion message (failure):** +``` +[✗] API container release 4.5.0 push failed View run +``` + +All messages are simple one-liners with a clickable "View run" link. The completion message adapts to show success `[✓]` or failure `[✗]` based on the outcome of the container push. + +### Deployment Start +- Header: Component and environment +- Yellow bar (color: `dbab09`) +- Status: In Progress +- Details: Commit, version, actor, workflow +- Link: Direct link to deployment run + +### Deployment Completion +- Header: Component and environment +- Green bar for success (color: `28a745`) / Red bar for failure (color: `fc3434`) +- Status: [✓] Completed or [✗] Failed +- Details: All deployment info plus terraform outcomes +- Link: Direct link to deployment run + +## Adding New Templates + +1. Create a new JSON file with Block Kit structure +2. Use environment variable placeholders (e.g., `$VAR_NAME`) +3. Include `channel` and `text` fields (required) +4. Add `blocks` or `attachments` for rich formatting +5. For update templates, include `ts` field as `$MESSAGE_TS` +6. Document the template in this README +7. Reference it in your workflow using `payload-file-path` + +## Reference + +- [Slack Block Kit Builder](https://app.slack.com/block-kit-builder) +- [Slack API Method Documentation](https://docs.slack.dev/tools/slack-github-action/sending-techniques/sending-data-slack-api-method/) diff --git a/.github/scripts/slack-messages/container-release-completed.json b/.github/scripts/slack-messages/container-release-completed.json new file mode 100644 index 0000000000..c1fc147191 --- /dev/null +++ b/.github/scripts/slack-messages/container-release-completed.json @@ -0,0 +1,17 @@ +{ + "channel": "${{ env.SLACK_CHANNEL_ID }}", + "attachments": [ + { + "color": "${{ env.STATUS_COLOR }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*Status:*\n${{ env.STATUS_TEXT }}\n\n${{ env.COMPONENT }} container release ${{ env.RELEASE_TAG }} push ${{ env.STATUS_TEXT }}\n\n<${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/actions/runs/${{ env.GITHUB_RUN_ID }}|View run>" + } + } + ] + } + ] +} diff --git a/.github/scripts/slack-messages/container-release-started.json b/.github/scripts/slack-messages/container-release-started.json new file mode 100644 index 0000000000..f67129e3ba --- /dev/null +++ b/.github/scripts/slack-messages/container-release-started.json @@ -0,0 +1,17 @@ +{ + "channel": "${{ env.SLACK_CHANNEL_ID }}", + "attachments": [ + { + "color": "${{ env.STATUS_COLOR }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*Status:*\nStarted\n\n${{ env.COMPONENT }} container release ${{ env.RELEASE_TAG }} push started...\n\n<${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/actions/runs/${{ env.GITHUB_RUN_ID }}|View run>" + } + } + ] + } + ] +} diff --git a/.github/scripts/trivy-pr-comment.js b/.github/scripts/trivy-pr-comment.js new file mode 100644 index 0000000000..eb725b1af1 --- /dev/null +++ b/.github/scripts/trivy-pr-comment.js @@ -0,0 +1,102 @@ +const fs = require('fs'); + +// Configuration from environment variables +const REPORT_FILE = process.env.TRIVY_REPORT_FILE || 'trivy-report.json'; +const IMAGE_NAME = process.env.IMAGE_NAME || 'container-image'; +const GITHUB_SHA = process.env.GITHUB_SHA || 'unknown'; +const GITHUB_REPOSITORY = process.env.GITHUB_REPOSITORY || ''; +const GITHUB_RUN_ID = process.env.GITHUB_RUN_ID || ''; +const SEVERITY = process.env.SEVERITY || 'CRITICAL,HIGH,MEDIUM,LOW'; + +// Parse severities to scan +const scannedSeverities = SEVERITY.split(',').map(s => s.trim()); + +// Read and parse the Trivy report +const report = JSON.parse(fs.readFileSync(REPORT_FILE, 'utf-8')); + +let vulnCount = 0; +let vulnsByType = { CRITICAL: 0, HIGH: 0, MEDIUM: 0, LOW: 0 }; +let affectedPackages = new Set(); + +if (report.Results && Array.isArray(report.Results)) { + for (const result of report.Results) { + if (result.Vulnerabilities && Array.isArray(result.Vulnerabilities)) { + for (const vuln of result.Vulnerabilities) { + vulnCount++; + if (vulnsByType[vuln.Severity] !== undefined) { + vulnsByType[vuln.Severity]++; + } + if (vuln.PkgName) { + affectedPackages.add(vuln.PkgName); + } + } + } + } +} + +const shortSha = GITHUB_SHA.substring(0, 7); +const timestamp = new Date().toISOString().replace('T', ' ').substring(0, 19) + ' UTC'; + +// Severity icons and labels +const severityConfig = { + CRITICAL: { icon: '🔴', label: 'Critical' }, + HIGH: { icon: '🟠', label: 'High' }, + MEDIUM: { icon: '🟡', label: 'Medium' }, + LOW: { icon: 'đŸ”ĩ', label: 'Low' } +}; + +let comment = '## 🔒 Container Security Scan\n\n'; +comment += `**Image:** \`${IMAGE_NAME}:${shortSha}\`\n`; +comment += `**Last scan:** ${timestamp}\n\n`; + +if (vulnCount === 0) { + comment += '### ✅ No Vulnerabilities Detected\n\n'; + comment += 'The container image passed all security checks. No known CVEs were found.\n'; +} else { + comment += '### 📊 Vulnerability Summary\n\n'; + comment += '| Severity | Count |\n'; + comment += '|----------|-------|\n'; + + // Only show severities that were scanned + for (const severity of scannedSeverities) { + const config = severityConfig[severity]; + const count = vulnsByType[severity] || 0; + const isBold = (severity === 'CRITICAL' || severity === 'HIGH') && count > 0; + const countDisplay = isBold ? `**${count}**` : count; + comment += `| ${config.icon} ${config.label} | ${countDisplay} |\n`; + } + + comment += `| **Total** | **${vulnCount}** |\n\n`; + + if (affectedPackages.size > 0) { + comment += `**${affectedPackages.size}** package(s) affected\n\n`; + } + + if (vulnsByType.CRITICAL > 0) { + comment += '### ⚠ī¸ Action Required\n\n'; + comment += '**Critical severity vulnerabilities detected.** These should be addressed before merging:\n'; + comment += '- Review the detailed scan results\n'; + comment += '- Update affected packages to patched versions\n'; + comment += '- Consider using a different base image if updates are unavailable\n\n'; + } else if (vulnsByType.HIGH > 0) { + comment += '### ⚠ī¸ Attention Needed\n\n'; + comment += '**High severity vulnerabilities found.** Please review and plan remediation:\n'; + comment += '- Assess the risk and exploitability\n'; + comment += '- Prioritize updates in the next maintenance cycle\n\n'; + } else { + comment += '### ℹī¸ Review Recommended\n\n'; + comment += 'Medium/Low severity vulnerabilities found. Consider addressing during regular maintenance.\n\n'; + } +} + +comment += '---\n'; +comment += '📋 **Resources:**\n'; + +if (GITHUB_REPOSITORY && GITHUB_RUN_ID) { + comment += `- [Download full report](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) (see artifacts)\n`; +} + +comment += '- [View in Security tab](https://github.com/' + (GITHUB_REPOSITORY || 'repository') + '/security/code-scanning)\n'; +comment += '- Scanned with [Trivy](https://github.com/aquasecurity/trivy)\n'; + +module.exports = comment; diff --git a/.github/workflows/api-build-lint-push-containers.yml b/.github/workflows/api-build-lint-push-containers.yml deleted file mode 100644 index e7010cef24..0000000000 --- a/.github/workflows/api-build-lint-push-containers.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: API - Build and Push containers - -on: - push: - branches: - - "master" - paths: - - "api/**" - - "prowler/**" - - ".github/workflows/api-build-lint-push-containers.yml" - - # Uncomment the code below to test this action on PRs - # pull_request: - # branches: - # - "master" - # paths: - # - "api/**" - # - ".github/workflows/api-build-lint-push-containers.yml" - - release: - types: [published] - -env: - # Tags - LATEST_TAG: latest - RELEASE_TAG: ${{ github.event.release.tag_name }} - STABLE_TAG: stable - - WORKING_DIRECTORY: ./api - - # Container Registries - PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud - PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-api - -jobs: - repository-check: - name: Repository check - runs-on: ubuntu-latest - outputs: - is_repo: ${{ steps.repository_check.outputs.is_repo }} - steps: - - name: Repository check - id: repository_check - working-directory: /tmp - run: | - if [[ ${{ github.repository }} == "prowler-cloud/prowler" ]] - then - echo "is_repo=true" >> "${GITHUB_OUTPUT}" - else - echo "This action only runs for prowler-cloud/prowler" - echo "is_repo=false" >> "${GITHUB_OUTPUT}" - fi - - # Build Prowler OSS container - container-build-push: - needs: repository-check - if: needs.repository-check.outputs.is_repo == 'true' - runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ env.WORKING_DIRECTORY }} - - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Set short git commit SHA - id: vars - run: | - shortSha=$(git rev-parse --short ${{ github.sha }}) - echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV - - - name: Login to DockerHub - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Build and push container image (latest) - # Comment the following line for testing - if: github.event_name == 'push' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.WORKING_DIRECTORY }} - # Set push: false for testing - push: true - tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.SHORT_SHA }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push container image (release) - if: github.event_name == 'release' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.WORKING_DIRECTORY }} - push: true - tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Trigger deployment - if: github.event_name == 'push' - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 - with: - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - repository: ${{ secrets.CLOUD_DISPATCH }} - event-type: prowler-api-deploy - client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ env.SHORT_SHA }}"}' diff --git a/.github/workflows/api-code-quality.yml b/.github/workflows/api-code-quality.yml new file mode 100644 index 0000000000..a9dddaa3aa --- /dev/null +++ b/.github/workflows/api-code-quality.yml @@ -0,0 +1,71 @@ +name: 'API: Code Quality' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + API_WORKING_DIR: ./api + +jobs: + api-code-quality: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.12' + defaults: + run: + working-directory: ./api + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + api/** + .github/workflows/api-code-quality.yml + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Setup Python with Poetry + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/setup-python-poetry + with: + python-version: ${{ matrix.python-version }} + working-directory: ./api + + - name: Poetry check + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry check --lock + + - name: Ruff lint + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run ruff check . --exclude contrib + + - name: Ruff format + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run ruff format --check . --exclude contrib + + - name: Pylint + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn src/ diff --git a/.github/workflows/api-codeql.yml b/.github/workflows/api-codeql.yml index d0211c4caa..16ea82538e 100644 --- a/.github/workflows/api-codeql.yml +++ b/.github/workflows/api-codeql.yml @@ -1,36 +1,34 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: API - CodeQL +name: 'API: CodeQL' on: push: branches: - - "master" - - "v5.*" + - 'master' + - 'v5.*' paths: - - "api/**" + - 'api/**' + - '.github/workflows/api-codeql.yml' + - '.github/codeql/api-codeql-config.yml' pull_request: branches: - - "master" - - "v5.*" + - 'master' + - 'v5.*' paths: - - "api/**" + - 'api/**' + - '.github/workflows/api-codeql.yml' + - '.github/codeql/api-codeql-config.yml' schedule: - cron: '00 12 * * *' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - analyze: - name: Analyze + api-analyze: + name: CodeQL Security Analysis runs-on: ubuntu-latest + timeout-minutes: 30 permissions: actions: read contents: read @@ -39,21 +37,20 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'python' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + language: + - 'python' steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 - with: - languages: ${{ matrix.language }} - config-file: ./.github/codeql/api-codeql-config.yml + - name: Initialize CodeQL + uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + with: + languages: ${{ matrix.language }} + config-file: ./.github/codeql/api-codeql-config.yml - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 - with: - category: "/language:${{matrix.language}}" + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + with: + category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/api-container-build-push.yml b/.github/workflows/api-container-build-push.yml new file mode 100644 index 0000000000..354c147a0b --- /dev/null +++ b/.github/workflows/api-container-build-push.yml @@ -0,0 +1,135 @@ +name: 'API: Container Build and Push' + +on: + push: + branches: + - 'master' + paths: + - 'api/**' + - 'prowler/**' + - '.github/workflows/api-build-lint-push-containers.yml' + release: + types: + - 'published' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + # Tags + LATEST_TAG: latest + RELEASE_TAG: ${{ github.event.release.tag_name }} + STABLE_TAG: stable + WORKING_DIRECTORY: ./api + + # Container registries + PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud + PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-api + +jobs: + setup: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + short-sha: ${{ steps.set-short-sha.outputs.short-sha }} + steps: + - name: Calculate short SHA + id: set-short-sha + run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + container-build-push: + needs: setup + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Login to DockerHub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build and push API container (latest) + if: github.event_name == 'push' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.WORKING_DIRECTORY }} + push: true + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + - name: Build and push API container (release) + if: github.event_name == 'release' + id: container-push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.WORKING_DIRECTORY }} + push: true + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} + + trigger-deployment: + if: github.event_name == 'push' + needs: [setup, container-build-push] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Trigger API deployment + uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + with: + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + repository: ${{ secrets.CLOUD_DISPATCH }} + event-type: api-prowler-deployment + client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}' diff --git a/.github/workflows/api-container-checks.yml b/.github/workflows/api-container-checks.yml new file mode 100644 index 0000000000..d069b0d5a6 --- /dev/null +++ b/.github/workflows/api-container-checks.yml @@ -0,0 +1,89 @@ +name: 'API: Container Checks' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + API_WORKING_DIR: ./api + IMAGE_NAME: prowler-api + +jobs: + api-dockerfile-lint: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check if Dockerfile changed + id: dockerfile-changed + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: api/Dockerfile + + - name: Lint Dockerfile with Hadolint + if: steps.dockerfile-changed.outputs.any_changed == 'true' + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: api/Dockerfile + ignore: DL3013 + + api-container-build-and-scan: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: api/** + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Set up Docker Buildx + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build container + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.API_WORKING_DIR }} + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Scan container with Trivy + if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/trivy-scan + with: + image-name: ${{ env.IMAGE_NAME }} + image-tag: ${{ github.sha }} + fail-on-critical: 'false' + severity: 'CRITICAL' diff --git a/.github/workflows/api-pull-request.yml b/.github/workflows/api-pull-request.yml deleted file mode 100644 index e899af8f32..0000000000 --- a/.github/workflows/api-pull-request.yml +++ /dev/null @@ -1,233 +0,0 @@ -name: API - Pull Request - -on: - push: - branches: - - "master" - - "v5.*" - paths: - - ".github/workflows/api-pull-request.yml" - - "api/**" - pull_request: - branches: - - "master" - - "v5.*" - paths: - - ".github/workflows/api-pull-request.yml" - - "api/**" - -env: - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 - POSTGRES_ADMIN_USER: prowler - POSTGRES_ADMIN_PASSWORD: S3cret - POSTGRES_USER: prowler_user - POSTGRES_PASSWORD: prowler - POSTGRES_DB: postgres-db - VALKEY_HOST: localhost - VALKEY_PORT: 6379 - VALKEY_DB: 0 - API_WORKING_DIR: ./api - IMAGE_NAME: prowler-api - IGNORE_FILES: | - api/docs/** - api/README.md - api/CHANGELOG.md - -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.12"] - - # Service containers to run with `test` - services: - # Label used to access the service container - postgres: - image: postgres - env: - POSTGRES_HOST: ${{ env.POSTGRES_HOST }} - POSTGRES_PORT: ${{ env.POSTGRES_PORT }} - POSTGRES_USER: ${{ env.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} - POSTGRES_DB: ${{ env.POSTGRES_DB }} - # Set health checks to wait until postgres has started - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - valkey: - image: valkey/valkey:7-alpine3.19 - env: - VALKEY_HOST: ${{ env.VALKEY_HOST }} - VALKEY_PORT: ${{ env.VALKEY_PORT }} - VALKEY_DB: ${{ env.VALKEY_DB }} - # Set health checks to wait until postgres has started - ports: - - 6379:6379 - options: >- - --health-cmd "valkey-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Test if changes are in not ignored paths - id: are-non-ignored-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - api/** - .github/workflows/api-pull-request.yml - files_ignore: ${{ env.IGNORE_FILES }} - - - name: Replace @master with current branch in pyproject.toml - Only for pull requests to `master` - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' && github.event_name == 'pull_request' && github.base_ref == 'master' - run: | - BRANCH_NAME="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - echo "Using branch: $BRANCH_NAME" - sed -i "s|@master|@$BRANCH_NAME|g" pyproject.toml - - - name: Install poetry - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - python -m pip install --upgrade pip - pipx install poetry==2.1.1 - - - name: Update SDK's poetry.lock resolved_reference to latest commit - Only for push events to `master` - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/master' - run: | - # Get the latest commit hash from the prowler-cloud/prowler repository - LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha') - echo "Latest commit hash: $LATEST_COMMIT" - - # Update the resolved_reference specifically for prowler-cloud/prowler repository - sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / { - s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/ - }' poetry.lock - - # Verify the change was made - echo "Updated resolved_reference:" - grep -A2 -B2 "resolved_reference" poetry.lock - - - name: Update poetry.lock - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry lock - - - name: Set up Python ${{ matrix.python-version }} - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ matrix.python-version }} - cache: "poetry" - - - name: Install dependencies - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry install --no-root - poetry run pip list - VERSION=$(curl --silent "https://api.github.com/repos/hadolint/hadolint/releases/latest" | \ - grep '"tag_name":' | \ - sed -E 's/.*"v([^"]+)".*/\1/' \ - ) && curl -L -o /tmp/hadolint "https://github.com/hadolint/hadolint/releases/download/v${VERSION}/hadolint-Linux-x86_64" \ - && chmod +x /tmp/hadolint - - - name: Poetry check - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry check --lock - - - name: Lint with ruff - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run ruff check . --exclude contrib - - - name: Check Format with ruff - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run ruff format --check . --exclude contrib - - - name: Lint with pylint - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run pylint --disable=W,C,R,E -j 0 -rn -sn src/ - - - name: Bandit - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run bandit -q -lll -x '*_test.py,./contrib/' -r . - - - name: Safety - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - # 76352, 76353, 77323 come from SDK, but they cannot upgrade it yet. It does not affect API - # TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X - run: | - poetry run safety check --ignore 70612,66963,74429,76352,76353,77323,77744,77745 - - - name: Vulture - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run vulture --exclude "contrib,tests,conftest.py" --min-confidence 100 . - - - name: Hadolint - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - /tmp/hadolint Dockerfile --ignore=DL3013 - - - name: Test with pytest - working-directory: ./api - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run pytest --cov=./src/backend --cov-report=xml src/backend - - - name: Upload coverage reports to Codecov - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - flags: api - test-container-build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Test if changes are in not ignored paths - id: are-non-ignored-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: api/** - files_ignore: ${{ env.IGNORE_FILES }} - - name: Set up Docker Buildx - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - name: Build Container - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.API_WORKING_DIR }} - push: false - tags: ${{ env.IMAGE_NAME }}:latest - outputs: type=docker - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/api-security.yml b/.github/workflows/api-security.yml new file mode 100644 index 0000000000..33134ec9c9 --- /dev/null +++ b/.github/workflows/api-security.yml @@ -0,0 +1,69 @@ +name: 'API: Security' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + API_WORKING_DIR: ./api + +jobs: + api-security-scans: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.12' + defaults: + run: + working-directory: ./api + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + api/** + .github/workflows/api-security.yml + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Setup Python with Poetry + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/setup-python-poetry + with: + python-version: ${{ matrix.python-version }} + working-directory: ./api + + - name: Bandit + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run bandit -q -lll -x '*_test.py,./contrib/' -r . + + - name: Safety + if: steps.check-changes.outputs.any_changed == 'true' + # 76352, 76353, 77323 come from SDK, but they cannot upgrade it yet. It does not affect API + # TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X + run: poetry run safety check --ignore 70612,66963,74429,76352,76353,77323,77744,77745 + + - name: Vulture + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run vulture --exclude "contrib,tests,conftest.py" --min-confidence 100 . diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml new file mode 100644 index 0000000000..daeb79abac --- /dev/null +++ b/.github/workflows/api-tests.yml @@ -0,0 +1,107 @@ +name: 'API: Tests' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + POSTGRES_ADMIN_USER: prowler + POSTGRES_ADMIN_PASSWORD: S3cret + POSTGRES_USER: prowler_user + POSTGRES_PASSWORD: prowler + POSTGRES_DB: postgres-db + VALKEY_HOST: localhost + VALKEY_PORT: 6379 + VALKEY_DB: 0 + API_WORKING_DIR: ./api + +jobs: + api-tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.12' + defaults: + run: + working-directory: ./api + + services: + postgres: + image: postgres + env: + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} + POSTGRES_PORT: ${{ env.POSTGRES_PORT }} + POSTGRES_USER: ${{ env.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} + POSTGRES_DB: ${{ env.POSTGRES_DB }} + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + valkey: + image: valkey/valkey:7-alpine3.19 + env: + VALKEY_HOST: ${{ env.VALKEY_HOST }} + VALKEY_PORT: ${{ env.VALKEY_PORT }} + VALKEY_DB: ${{ env.VALKEY_DB }} + ports: + - 6379:6379 + options: >- + --health-cmd "valkey-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + api/** + .github/workflows/api-tests.yml + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Setup Python with Poetry + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/setup-python-poetry + with: + python-version: ${{ matrix.python-version }} + working-directory: ./api + + - name: Run tests with pytest + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run pytest --cov=./src/backend --cov-report=xml src/backend + + - name: Upload coverage reports to Codecov + if: steps.check-changes.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: api diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 2aa0a49c09..974d919fc6 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -1,28 +1,33 @@ -name: Prowler - Automatic Backport +name: 'Tools: Backport' on: pull_request_target: - branches: ['master'] - types: ['labeled', 'closed'] + branches: + - 'master' + types: + - 'labeled' + - 'closed' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: false env: - # The prefix of the label that triggers the backport must not contain the branch name - # so, for example, if the branch is 'master', the label should be 'backport-to-' BACKPORT_LABEL_PREFIX: backport-to- BACKPORT_LABEL_IGNORE: was-backported jobs: backport: - name: Backport PR if: github.event.pull_request.merged == true && !(contains(github.event.pull_request.labels.*.name, 'backport')) && !(contains(github.event.pull_request.labels.*.name, 'was-backported')) runs-on: ubuntu-latest + timeout-minutes: 15 permissions: - id-token: write - pull-requests: write contents: write + pull-requests: write + steps: - name: Check labels - id: preview_label_check + id: label_check uses: agilepathway/label-checker@c3d16ad512e7cea5961df85ff2486bb774caf3c5 # v1.6.65 with: allow_failure: true @@ -31,17 +36,17 @@ jobs: none_of: ${{ env.BACKPORT_LABEL_IGNORE }} repo_token: ${{ secrets.GITHUB_TOKEN }} - - name: Backport Action - if: steps.preview_label_check.outputs.label_check == 'success' - uses: sorenlouv/backport-github-action@ad888e978060bc1b2798690dd9d03c4036560947 # v9.5.1 + - name: Backport PR + if: steps.label_check.outputs.label_check == 'success' + uses: sorenlouv/backport-github-action@516854e7c9f962b9939085c9a92ea28411d1ae90 # v10.2.0 with: github_token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} auto_backport_label_prefix: ${{ env.BACKPORT_LABEL_PREFIX }} - - name: Info log - if: ${{ success() && steps.preview_label_check.outputs.label_check == 'success' }} + - name: Display backport info log + if: success() && steps.label_check.outputs.label_check == 'success' run: cat ~/.backport/backport.info.log - - name: Debug log - if: ${{ failure() && steps.preview_label_check.outputs.label_check == 'success' }} + - name: Display backport debug log + if: failure() && steps.label_check.outputs.label_check == 'success' run: cat ~/.backport/backport.debug.log diff --git a/.github/workflows/build-documentation-on-pr.yml b/.github/workflows/build-documentation-on-pr.yml deleted file mode 100644 index 068197e805..0000000000 --- a/.github/workflows/build-documentation-on-pr.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Prowler - Pull Request Documentation Link - -on: - pull_request: - branches: - - 'master' - - 'v3' - paths: - - 'docs/**' - - '.github/workflows/build-documentation-on-pr.yml' - -env: - PR_NUMBER: ${{ github.event.pull_request.number }} - -jobs: - documentation-link: - name: Documentation Link - runs-on: ubuntu-latest - steps: - - name: Find existing documentation comment - uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 - id: find-comment - with: - issue-number: ${{ env.PR_NUMBER }} - comment-author: 'github-actions[bot]' - body-includes: '' - - - name: Create or update PR comment with the Prowler Documentation URI - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 - with: - comment-id: ${{ steps.find-comment.outputs.comment-id }} - issue-number: ${{ env.PR_NUMBER }} - body: | - - You can check the documentation for this PR here -> [Prowler Documentation](https://prowler-prowler-docs--${{ env.PR_NUMBER }}.com.readthedocs.build/projects/prowler-open-source/en/${{ env.PR_NUMBER }}/) - edit-mode: replace diff --git a/.github/workflows/conventional-commit.yml b/.github/workflows/conventional-commit.yml index ecaf651c34..58e1653b74 100644 --- a/.github/workflows/conventional-commit.yml +++ b/.github/workflows/conventional-commit.yml @@ -1,24 +1,31 @@ -name: Prowler - Conventional Commit +name: 'Tools: Conventional Commit' on: pull_request: - types: - - "opened" - - "edited" - - "synchronize" branches: - - "master" - - "v3" - - "v4.*" - - "v5.*" + - 'master' + - 'v3' + - 'v4.*' + - 'v5.*' + types: + - 'opened' + - 'edited' + - 'synchronize' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: conventional-commit-check: runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: read + steps: - - name: conventional-commit-check - id: conventional-commit-check - uses: agenthunt/conventional-commit-checker-action@9e552d650d0e205553ec7792d447929fc78e012b # v2.0.0 + - name: Check PR title format + uses: agenthunt/conventional-commit-checker-action@f1823f632e95a64547566dcd2c7da920e67117ad # v2.0.1 with: - pr-title-regex: '^(feat|fix|docs|style|refactor|perf|test|chore|build|ci|revert)(\([^)]+\))?!?: .+' - \ No newline at end of file + pr-title-regex: '^(feat|fix|docs|style|refactor|perf|test|chore|build|ci|revert)(\([^)]+\))?!?: .+' diff --git a/.github/workflows/create-backport-label.yml b/.github/workflows/create-backport-label.yml index 3b485ec513..b4308156c7 100644 --- a/.github/workflows/create-backport-label.yml +++ b/.github/workflows/create-backport-label.yml @@ -1,67 +1,70 @@ -name: Prowler - Create Backport Label +name: 'Tools: Backport Label' on: release: - types: [published] + types: + - 'published' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false + +env: + BACKPORT_LABEL_PREFIX: backport-to- + BACKPORT_LABEL_COLOR: B60205 jobs: - create_label: + create-label: runs-on: ubuntu-latest + timeout-minutes: 15 permissions: - contents: write + contents: read issues: write + steps: - - name: Create backport label + - name: Create backport label for minor releases env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - OWNER_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - VERSION_ONLY=${RELEASE_TAG#v} # Remove 'v' prefix if present (e.g., v3.2.0 -> 3.2.0) + RELEASE_TAG="${{ github.event.release.tag_name }}" + + if [ -z "$RELEASE_TAG" ]; then + echo "Error: No release tag provided" + exit 1 + fi + + echo "Processing release tag: $RELEASE_TAG" + + # Remove 'v' prefix if present (e.g., v3.2.0 -> 3.2.0) + VERSION_ONLY="${RELEASE_TAG#v}" # Check if it's a minor version (X.Y.0) - if [[ "$VERSION_ONLY" =~ ^[0-9]+\.[0-9]+\.0$ ]]; then - echo "Release ${RELEASE_TAG} (version ${VERSION_ONLY}) is a minor version. Proceeding to create backport label." + if [[ "$VERSION_ONLY" =~ ^([0-9]+)\.([0-9]+)\.0$ ]]; then + echo "Release $RELEASE_TAG (version $VERSION_ONLY) is a minor version. Proceeding to create backport label." - TWO_DIGIT_VERSION=${VERSION_ONLY%.0} # Extract X.Y from X.Y.0 (e.g., 5.6 from 5.6.0) + # Extract X.Y from X.Y.0 (e.g., 5.6 from 5.6.0) + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + TWO_DIGIT_VERSION="${MAJOR}.${MINOR}" - FINAL_LABEL_NAME="backport-to-v${TWO_DIGIT_VERSION}" - FINAL_DESCRIPTION="Backport PR to the v${TWO_DIGIT_VERSION} branch" + LABEL_NAME="${BACKPORT_LABEL_PREFIX}v${TWO_DIGIT_VERSION}" + LABEL_DESC="Backport PR to the v${TWO_DIGIT_VERSION} branch" + LABEL_COLOR="$BACKPORT_LABEL_COLOR" - echo "Effective label name will be: ${FINAL_LABEL_NAME}" - echo "Effective description will be: ${FINAL_DESCRIPTION}" + echo "Label name: $LABEL_NAME" + echo "Label description: $LABEL_DESC" - # Check if the label already exists - STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token ${GITHUB_TOKEN}" "https://api.github.com/repos/${OWNER_REPO}/labels/${FINAL_LABEL_NAME}") - - if [ "${STATUS_CODE}" -eq 200 ]; then - echo "Label '${FINAL_LABEL_NAME}' already exists." - elif [ "${STATUS_CODE}" -eq 404 ]; then - echo "Label '${FINAL_LABEL_NAME}' does not exist. Creating it..." - # Prepare JSON data payload - JSON_DATA=$(printf '{"name":"%s","description":"%s","color":"B60205"}' "${FINAL_LABEL_NAME}" "${FINAL_DESCRIPTION}") - - CREATE_STATUS_CODE=$(curl -s -o /tmp/curl_create_response.json -w "%{http_code}" -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - --data "${JSON_DATA}" \ - "https://api.github.com/repos/${OWNER_REPO}/labels") - - CREATE_RESPONSE_BODY=$(cat /tmp/curl_create_response.json) - rm -f /tmp/curl_create_response.json - - if [ "$CREATE_STATUS_CODE" -eq 201 ]; then - echo "Label '${FINAL_LABEL_NAME}' created successfully." - else - echo "Error creating label '${FINAL_LABEL_NAME}'. Status: $CREATE_STATUS_CODE" - echo "Response: $CREATE_RESPONSE_BODY" - exit 1 - fi + # Check if label already exists + if gh label list --repo ${{ github.repository }} --limit 1000 | grep -q "^${LABEL_NAME}[[:space:]]"; then + echo "Label '$LABEL_NAME' already exists." else - echo "Error checking for label '${FINAL_LABEL_NAME}'. HTTP Status: ${STATUS_CODE}" - exit 1 + echo "Label '$LABEL_NAME' does not exist. Creating it..." + gh label create "$LABEL_NAME" \ + --description "$LABEL_DESC" \ + --color "$LABEL_COLOR" \ + --repo ${{ github.repository }} + echo "Label '$LABEL_NAME' created successfully." fi else - echo "Release ${RELEASE_TAG} (version ${VERSION_ONLY}) is not a minor version. Skipping backport label creation." - exit 0 + echo "Release $RELEASE_TAG (version $VERSION_ONLY) is not a minor version. Skipping backport label creation." fi diff --git a/.github/workflows/find-secrets.yml b/.github/workflows/find-secrets.yml index e7feaea43a..80b49e3279 100644 --- a/.github/workflows/find-secrets.yml +++ b/.github/workflows/find-secrets.yml @@ -1,19 +1,33 @@ -name: Prowler - Find secrets +name: 'Tools: TruffleHog' -on: pull_request +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - trufflehog: + scan-secrets: runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: - - name: Checkout + - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 - - name: TruffleHog OSS - uses: trufflesecurity/trufflehog@466da5b0bb161144f6afca9afe5d57975828c410 # v3.90.8 + + - name: Scan for secrets with TruffleHog + uses: trufflesecurity/trufflehog@b84c3d14d189e16da175e2c27fa8136603783ffc # v3.90.12 with: - path: ./ - base: ${{ github.event.repository.default_branch }} - head: HEAD - extra_args: --only-verified + extra_args: '--results=verified,unknown' diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 9dfa8993a1..85ccd34cc2 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,17 +1,92 @@ -name: Prowler - PR Labeler +name: 'Tools: PR Labeler' on: - pull_request_target: - branches: - - "master" - - "v3" - - "v4.*" + pull_request_target: + branches: + - 'master' + - 'v5.*' + types: + - 'opened' + - 'reopened' + - 'synchronize' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: labeler: + runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + steps: - - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + - name: Apply labels to PR + uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + with: + sync-labels: true + + label-community: + name: Add 'community' label if the PR is from a community contributor + needs: labeler + if: github.repository == 'prowler-cloud/prowler' && github.event.action == 'opened' + runs-on: ubuntu-latest + permissions: + pull-requests: write + + steps: + - name: Check if author is org member + id: check_membership + env: + AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + # Hardcoded list of prowler-cloud organization members + # This list includes members who have set their organization membership as private + ORG_MEMBERS=( + "AdriiiPRodri" + "Alan-TheGentleman" + "alejandrobailo" + "amitsharm" + "andoniaf" + "cesararroba" + "Chan9390" + "danibarranqueroo" + "HugoPBrito" + "jfagoagas" + "josemazo" + "lydiavilchez" + "mmuller88" + "MrCloudSec" + "pedrooot" + "prowler-bot" + "puchy22" + "rakan-pro" + "RosaRivasProwler" + "StylusFrost" + "toniblyx" + "vicferpoy" + ) + + echo "Checking if $AUTHOR is a member of prowler-cloud organization" + + # Check if author is in the org members list + if printf '%s\n' "${ORG_MEMBERS[@]}" | grep -q "^${AUTHOR}$"; then + echo "is_member=true" >> $GITHUB_OUTPUT + echo "$AUTHOR is an organization member" + else + echo "is_member=false" >> $GITHUB_OUTPUT + echo "$AUTHOR is not an organization member" + fi + + - name: Add community label + if: steps.check_membership.outputs.is_member == 'false' + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + GH_TOKEN: ${{ github.token }} + run: | + echo "Adding 'community' label to PR #$PR_NUMBER" + gh api /repos/${{ github.repository }}/issues/${{ github.event.number }}/labels \ + -X POST \ + -f labels[]='community' diff --git a/.github/workflows/mcp-container-build-push.yml b/.github/workflows/mcp-container-build-push.yml new file mode 100644 index 0000000000..8fb2dfcba6 --- /dev/null +++ b/.github/workflows/mcp-container-build-push.yml @@ -0,0 +1,148 @@ +name: 'MCP: Container Build and Push' + +on: + push: + branches: + - 'master' + paths: + - 'mcp_server/**' + - '.github/workflows/mcp-container-build-push.yml' + release: + types: + - 'published' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + # Tags + LATEST_TAG: latest + RELEASE_TAG: ${{ github.event.release.tag_name }} + STABLE_TAG: stable + WORKING_DIRECTORY: ./mcp_server + + # Container registries + PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud + PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-mcp + +jobs: + setup: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + short-sha: ${{ steps.set-short-sha.outputs.short-sha }} + steps: + - name: Calculate short SHA + id: set-short-sha + run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + container-build-push: + needs: setup + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Login to DockerHub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build and push MCP container (latest) + if: github.event_name == 'push' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.WORKING_DIRECTORY }} + push: true + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} + labels: | + org.opencontainers.image.title=Prowler MCP Server + org.opencontainers.image.description=Model Context Protocol server for Prowler + org.opencontainers.image.vendor=ProwlerPro, Inc. + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.head_commit.timestamp }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: MCP + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + - name: Build and push MCP container (release) + if: github.event_name == 'release' + id: container-push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.WORKING_DIRECTORY }} + push: true + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} + labels: | + org.opencontainers.image.title=Prowler MCP Server + org.opencontainers.image.description=Model Context Protocol server for Prowler + org.opencontainers.image.vendor=ProwlerPro, Inc. + org.opencontainers.image.version=${{ env.RELEASE_TAG }} + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.release.published_at }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: MCP + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} + + trigger-deployment: + if: github.event_name == 'push' + needs: [setup, container-build-push] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Trigger MCP deployment + uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + with: + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + repository: ${{ secrets.CLOUD_DISPATCH }} + event-type: mcp-prowler-deployment + client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}' diff --git a/.github/workflows/mcp-container-checks.yml b/.github/workflows/mcp-container-checks.yml new file mode 100644 index 0000000000..3310cd8c5a --- /dev/null +++ b/.github/workflows/mcp-container-checks.yml @@ -0,0 +1,87 @@ +name: 'MCP: Container Checks' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MCP_WORKING_DIR: ./mcp_server + IMAGE_NAME: prowler-mcp + +jobs: + mcp-dockerfile-lint: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check if Dockerfile changed + id: dockerfile-changed + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: mcp_server/Dockerfile + + - name: Lint Dockerfile with Hadolint + if: steps.dockerfile-changed.outputs.any_changed == 'true' + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: mcp_server/Dockerfile + + mcp-container-build-and-scan: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for MCP changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: mcp_server/** + files_ignore: | + mcp_server/README.md + mcp_server/CHANGELOG.md + + - name: Set up Docker Buildx + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build MCP container + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.MCP_WORKING_DIR }} + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Scan MCP container with Trivy + if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/trivy-scan + with: + image-name: ${{ env.IMAGE_NAME }} + image-tag: ${{ github.sha }} + fail-on-critical: 'false' + severity: 'CRITICAL' diff --git a/.github/workflows/pr-check-changelog.yml b/.github/workflows/pr-check-changelog.yml new file mode 100644 index 0000000000..d1a3220402 --- /dev/null +++ b/.github/workflows/pr-check-changelog.yml @@ -0,0 +1,103 @@ +name: 'Tools: Check Changelog' + +on: + pull_request: + types: + - 'opened' + - 'synchronize' + - 'reopened' + - 'labeled' + - 'unlabeled' + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check-changelog: + if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + env: + MONITORED_FOLDERS: 'api ui prowler mcp_server' + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + api/** + ui/** + prowler/** + mcp_server/** + + - name: Check for folder changes and changelog presence + id: check-folders + run: | + missing_changelogs="" + + # Check api folder + if [[ "${{ steps.changed-files.outputs.any_changed }}" == "true" ]]; then + for folder in $MONITORED_FOLDERS; do + # Get files changed in this folder + changed_in_folder=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "^${folder}/" || true) + + if [ -n "$changed_in_folder" ]; then + echo "Detected changes in ${folder}/" + + # Check if CHANGELOG.md was updated + if ! echo "$changed_in_folder" | grep -q "^${folder}/CHANGELOG.md$"; then + echo "No changelog update found for ${folder}/" + missing_changelogs="${missing_changelogs}- \`${folder}\`"$'\n' + fi + fi + done + fi + + { + echo "missing_changelogs<> $GITHUB_OUTPUT + + - name: Find existing changelog comment + if: github.event.pull_request.head.repo.full_name == github.repository + id: find-comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: '' + + - name: Update PR comment with changelog status + if: github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-id: ${{ steps.find-comment.outputs.comment-id }} + edit-mode: replace + body: | + + ${{ steps.check-folders.outputs.missing_changelogs != '' && format('⚠ī¸ **Changes detected in the following folders without a corresponding update to the `CHANGELOG.md`:** + + {0} + + Please add an entry to the corresponding `CHANGELOG.md` file to maintain a clear history of changes.', steps.check-folders.outputs.missing_changelogs) || '✅ All necessary `CHANGELOG.md` files have been updated.' }} + + - name: Fail if changelog is missing + if: steps.check-folders.outputs.missing_changelogs != '' + run: | + echo "::error::Missing changelog updates in some folders" + exit 1 diff --git a/.github/workflows/pr-conflict-checker.yml b/.github/workflows/pr-conflict-checker.yml index 77280d5136..c46fafc2d0 100644 --- a/.github/workflows/pr-conflict-checker.yml +++ b/.github/workflows/pr-conflict-checker.yml @@ -1,42 +1,40 @@ -name: Prowler - PR Conflict Checker +name: 'Tools: PR Conflict Checker' on: - pull_request: + pull_request_target: types: - - opened - - synchronize - - reopened + - 'opened' + - 'synchronize' + - 'reopened' branches: - - "master" - - "v5.*" - # Leaving this commented until we find a way to run it for forks but in Prowler's context - # pull_request_target: - # types: - # - opened - # - synchronize - # - reopened - # branches: - # - "master" - # - "v5.*" + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: - conflict-checker: + check-conflicts: runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read pull-requests: write issues: write steps: - - name: Checkout repository + - name: Checkout PR head uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 - name: Get changed files id: changed-files uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: - files: | - ** + files: '**' - name: Check for conflict markers id: conflict-check @@ -51,10 +49,10 @@ jobs: if [ -f "$file" ]; then echo "Checking file: $file" - # Look for conflict markers - if grep -l "^<<<<<<<\|^=======\|^>>>>>>>" "$file" 2>/dev/null; then + # Look for conflict markers (more precise regex) + if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then echo "Conflict markers found in: $file" - CONFLICT_FILES="$CONFLICT_FILES$file " + CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n' HAS_CONFLICTS=true fi fi @@ -62,114 +60,64 @@ jobs: if [ "$HAS_CONFLICTS" = true ]; then echo "has_conflicts=true" >> $GITHUB_OUTPUT - echo "conflict_files=$CONFLICT_FILES" >> $GITHUB_OUTPUT - echo "Conflict markers detected in files: $CONFLICT_FILES" + { + echo "conflict_files<> $GITHUB_OUTPUT + echo "Conflict markers detected" else echo "has_conflicts=false" >> $GITHUB_OUTPUT echo "No conflict markers found in changed files" fi - - name: Add conflict label - if: steps.conflict-check.outputs.has_conflicts == 'true' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - with: - github-token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - script: | - const { data: labels } = await github.rest.issues.listLabelsOnIssue({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); + - name: Manage conflict label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }} + run: | + LABEL_NAME="has-conflicts" - const hasConflictLabel = labels.some(label => label.name === 'has-conflicts'); + # Add or remove label based on conflict status + if [ "$HAS_CONFLICTS" = "true" ]; then + echo "Adding conflict label to PR #${PR_NUMBER}..." + gh pr edit "$PR_NUMBER" --add-label "$LABEL_NAME" --repo ${{ github.repository }} || true + else + echo "Removing conflict label from PR #${PR_NUMBER}..." + gh pr edit "$PR_NUMBER" --remove-label "$LABEL_NAME" --repo ${{ github.repository }} || true + fi - if (!hasConflictLabel) { - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - labels: ['has-conflicts'] - }); - console.log('Added has-conflicts label'); - } else { - console.log('has-conflicts label already exists'); - } - - - name: Remove conflict label - if: steps.conflict-check.outputs.has_conflicts == 'false' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - with: - github-token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - script: | - try { - await github.rest.issues.removeLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - name: 'has-conflicts' - }); - console.log('Removed has-conflicts label'); - } catch (error) { - if (error.status === 404) { - console.log('has-conflicts label was not present'); - } else { - throw error; - } - } - - - name: Find existing conflict comment - if: steps.conflict-check.outputs.has_conflicts == 'true' + - name: Find existing comment uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 id: find-comment with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' - body-regex: '(⚠ī¸ \*\*Conflict Markers Detected\*\*|✅ \*\*Conflict Markers Resolved\*\*)' + body-includes: '' - - name: Create or update conflict comment - if: steps.conflict-check.outputs.has_conflicts == 'true' - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 + - name: Create or update comment + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 with: comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} edit-mode: replace body: | - ⚠ī¸ **Conflict Markers Detected** + + ${{ steps.conflict-check.outputs.has_conflicts == 'true' && '⚠ī¸ **Conflict Markers Detected**' || '✅ **Conflict Markers Resolved**' }} - This pull request contains unresolved conflict markers in the following files: - ``` - ${{ steps.conflict-check.outputs.conflict_files }} - ``` + ${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('This pull request contains unresolved conflict markers in the following files: + + {0} Please resolve these conflicts by: 1. Locating the conflict markers: `<<<<<<<`, `=======`, and `>>>>>>>` 2. Manually editing the files to resolve the conflicts 3. Removing all conflict markers - 4. Committing and pushing the changes - - - name: Find existing conflict comment when resolved - if: steps.conflict-check.outputs.has_conflicts == 'false' - uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 - id: find-resolved-comment - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-regex: '(⚠ī¸ \*\*Conflict Markers Detected\*\*|✅ \*\*Conflict Markers Resolved\*\*)' - - - name: Update comment when conflicts resolved - if: steps.conflict-check.outputs.has_conflicts == 'false' && steps.find-resolved-comment.outputs.comment-id != '' - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 - with: - comment-id: ${{ steps.find-resolved-comment.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - edit-mode: replace - body: | - ✅ **Conflict Markers Resolved** - - All conflict markers have been successfully resolved in this pull request. + 4. Committing and pushing the changes', steps.conflict-check.outputs.conflict_files) || 'All conflict markers have been successfully resolved in this pull request.' }} - name: Fail workflow if conflicts detected if: steps.conflict-check.outputs.has_conflicts == 'true' run: | - echo "::error::Workflow failed due to conflict markers in files: ${{ steps.conflict-check.outputs.conflict_files }}" + echo "::error::Workflow failed due to conflict markers detected in the PR" exit 1 diff --git a/.github/workflows/pull-request-merged.yml b/.github/workflows/pr-merged.yml similarity index 55% rename from .github/workflows/pull-request-merged.yml rename to .github/workflows/pr-merged.yml index 4b5a93aabd..d8255026e6 100644 --- a/.github/workflows/pull-request-merged.yml +++ b/.github/workflows/pr-merged.yml @@ -1,27 +1,31 @@ -name: Prowler - Merged Pull Request +name: 'Tools: PR Merged' on: pull_request_target: - branches: ['master'] - types: ['closed'] + branches: + - 'master' + types: + - 'closed' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: false jobs: trigger-cloud-pull-request: - name: Trigger Cloud Pull Request if: github.event.pull_request.merged == true && github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.pull_request.merge_commit_sha }} - - - name: Set short git commit SHA + - name: Calculate short commit SHA id: vars run: | - shortSha=$(git rev-parse --short ${{ github.event.pull_request.merge_commit_sha }}) - echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV + SHORT_SHA="${{ github.event.pull_request.merge_commit_sha }}" + echo "SHORT_SHA=${SHORT_SHA::7}" >> $GITHUB_ENV - - name: Trigger pull request + - name: Trigger Cloud repository pull request uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} @@ -31,8 +35,12 @@ jobs: { "PROWLER_COMMIT_SHA": "${{ github.event.pull_request.merge_commit_sha }}", "PROWLER_COMMIT_SHORT_SHA": "${{ env.SHORT_SHA }}", + "PROWLER_PR_NUMBER": "${{ github.event.pull_request.number }}", "PROWLER_PR_TITLE": ${{ toJson(github.event.pull_request.title) }}, "PROWLER_PR_LABELS": ${{ toJson(github.event.pull_request.labels.*.name) }}, "PROWLER_PR_BODY": ${{ toJson(github.event.pull_request.body) }}, - "PROWLER_PR_URL": ${{ toJson(github.event.pull_request.html_url) }} + "PROWLER_PR_URL": ${{ toJson(github.event.pull_request.html_url) }}, + "PROWLER_PR_MERGED_BY": "${{ github.event.pull_request.merged_by.login }}", + "PROWLER_PR_BASE_BRANCH": "${{ github.event.pull_request.base.ref }}", + "PROWLER_PR_HEAD_BRANCH": "${{ github.event.pull_request.head.ref }}" } diff --git a/.github/workflows/prowler-release-preparation.yml b/.github/workflows/prepare-release.yml similarity index 68% rename from .github/workflows/prowler-release-preparation.yml rename to .github/workflows/prepare-release.yml index 6fc27ea0d1..ae039bd5a6 100644 --- a/.github/workflows/prowler-release-preparation.yml +++ b/.github/workflows/prepare-release.yml @@ -1,6 +1,6 @@ -name: Prowler - Release Preparation +name: 'Tools: Prepare Release' -run-name: Prowler Release Preparation for ${{ inputs.prowler_version }} +run-name: 'Prepare Release for Prowler ${{ inputs.prowler_version }}' on: workflow_dispatch: @@ -10,18 +10,23 @@ on: required: true type: string +concurrency: + group: ${{ github.workflow }}-${{ inputs.prowler_version }} + cancel-in-progress: false + env: - PROWLER_VERSION: ${{ github.event.inputs.prowler_version }} + PROWLER_VERSION: ${{ inputs.prowler_version }} jobs: prepare-release: - if: github.repository == 'prowler-cloud/prowler' + if: github.event_name == 'workflow_dispatch' && github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: write pull-requests: write steps: - - name: Checkout code + - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 @@ -34,13 +39,13 @@ jobs: - name: Install Poetry run: | - python3 -m pip install --user poetry + python3 -m pip install --user poetry==2.1.1 echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Configure Git run: | - git config --global user.name "prowler-bot" - git config --global user.email "179230569+prowler-bot@users.noreply.github.com" + git config --global user.name 'prowler-bot' + git config --global user.email '179230569+prowler-bot@users.noreply.github.com' - name: Parse version and determine branch run: | @@ -59,58 +64,84 @@ jobs: BRANCH_NAME="v${MAJOR_VERSION}.${MINOR_VERSION}" echo "BRANCH_NAME=${BRANCH_NAME}" >> "${GITHUB_ENV}" - # Function to extract the latest version from changelog - extract_latest_version() { - local changelog_file="$1" - if [ -f "$changelog_file" ]; then - # Extract the first version entry (most recent) from changelog - # Format: ## [version] (1.2.3) or ## [vversion] (v1.2.3) - local version=$(grep -m 1 '^## \[' "$changelog_file" | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]') - echo "$version" - else - echo "" - fi - } - - # Read actual versions from changelogs (source of truth) - UI_VERSION=$(extract_latest_version "ui/CHANGELOG.md") - API_VERSION=$(extract_latest_version "api/CHANGELOG.md") - SDK_VERSION=$(extract_latest_version "prowler/CHANGELOG.md") - - echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}" - echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}" - echo "SDK_VERSION=${SDK_VERSION}" >> "${GITHUB_ENV}" - - if [ -n "$UI_VERSION" ]; then - echo "Read UI version from changelog: $UI_VERSION" - else - echo "Warning: No UI version found in ui/CHANGELOG.md" - fi - - if [ -n "$API_VERSION" ]; then - echo "Read API version from changelog: $API_VERSION" - else - echo "Warning: No API version found in api/CHANGELOG.md" - fi - - if [ -n "$SDK_VERSION" ]; then - echo "Read SDK version from changelog: $SDK_VERSION" - else - echo "Warning: No SDK version found in prowler/CHANGELOG.md" - fi - echo "Prowler version: $PROWLER_VERSION" echo "Branch name: $BRANCH_NAME" - echo "UI version: $UI_VERSION" - echo "API version: $API_VERSION" - echo "SDK version: $SDK_VERSION" echo "Is minor release: $([ $PATCH_VERSION -eq 0 ] && echo 'true' || echo 'false')" else echo "Invalid version syntax: '$PROWLER_VERSION' (must be N.N.N)" >&2 exit 1 fi - - name: Extract changelog entries + - name: Checkout release branch + run: | + echo "Checking out branch $BRANCH_NAME for release $PROWLER_VERSION..." + if git show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then + echo "Branch $BRANCH_NAME exists locally, checking out..." + git checkout "$BRANCH_NAME" + elif git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then + echo "Branch $BRANCH_NAME exists remotely, checking out..." + git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME" + else + echo "ERROR: Branch $BRANCH_NAME does not exist. For minor releases (X.Y.0), create it manually first. For patch releases (X.Y.Z), the branch should already exist." + exit 1 + fi + + - name: Read changelog versions from release branch + run: | + # Function to extract the latest version from changelog + extract_latest_version() { + local changelog_file="$1" + if [ -f "$changelog_file" ]; then + # Extract the first version entry (most recent) from changelog + # Format: ## [version] (1.2.3) or ## [vversion] (v1.2.3) + local version=$(grep -m 1 '^## \[' "$changelog_file" | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]') + echo "$version" + else + echo "" + fi + } + + # Read actual versions from changelogs (source of truth) + UI_VERSION=$(extract_latest_version "ui/CHANGELOG.md") + API_VERSION=$(extract_latest_version "api/CHANGELOG.md") + SDK_VERSION=$(extract_latest_version "prowler/CHANGELOG.md") + MCP_VERSION=$(extract_latest_version "mcp_server/CHANGELOG.md") + + echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}" + echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}" + echo "SDK_VERSION=${SDK_VERSION}" >> "${GITHUB_ENV}" + echo "MCP_VERSION=${MCP_VERSION}" >> "${GITHUB_ENV}" + + if [ -n "$UI_VERSION" ]; then + echo "Read UI version from changelog: $UI_VERSION" + else + echo "Warning: No UI version found in ui/CHANGELOG.md" + fi + + if [ -n "$API_VERSION" ]; then + echo "Read API version from changelog: $API_VERSION" + else + echo "Warning: No API version found in api/CHANGELOG.md" + fi + + if [ -n "$SDK_VERSION" ]; then + echo "Read SDK version from changelog: $SDK_VERSION" + else + echo "Warning: No SDK version found in prowler/CHANGELOG.md" + fi + + if [ -n "$MCP_VERSION" ]; then + echo "Read MCP version from changelog: $MCP_VERSION" + else + echo "Warning: No MCP version found in mcp_server/CHANGELOG.md" + fi + + echo "UI version: $UI_VERSION" + echo "API version: $API_VERSION" + echo "SDK version: $SDK_VERSION" + echo "MCP version: $MCP_VERSION" + + - name: Extract and combine changelog entries run: | set -e @@ -136,8 +167,8 @@ jobs: # Remove --- separators sed -i '/^---$/d' "$output_file" - # Remove trailing empty lines - sed -i '/^$/d' "$output_file" + # Remove only trailing empty lines (not all empty lines) + sed -i -e :a -e '/^\s*$/d;N;ba' "$output_file" } # Calculate expected versions for this release @@ -183,7 +214,26 @@ jobs: touch "prowler_changelog.md" fi - # Combine changelogs in order: UI, API, SDK + # MCP has changes if the changelog references this Prowler version + # Check if the changelog contains "(Prowler X.Y.Z)" or "(Prowler UNRELEASED)" + if [ -f "mcp_server/CHANGELOG.md" ]; then + MCP_PROWLER_REF=$(grep -m 1 "^## \[.*\] (Prowler" mcp_server/CHANGELOG.md | sed -E 's/.*\(Prowler ([^)]+)\).*/\1/' | tr -d '[:space:]') + if [ "$MCP_PROWLER_REF" = "$PROWLER_VERSION" ] || [ "$MCP_PROWLER_REF" = "UNRELEASED" ]; then + echo "HAS_MCP_CHANGES=true" >> $GITHUB_ENV + echo "✓ MCP changes detected - Prowler reference: $MCP_PROWLER_REF (version: $MCP_VERSION)" + extract_changelog "mcp_server/CHANGELOG.md" "$MCP_VERSION" "mcp_changelog.md" + else + echo "HAS_MCP_CHANGES=false" >> $GITHUB_ENV + echo "ℹ No MCP changes for this release (Prowler reference: $MCP_PROWLER_REF, input: $PROWLER_VERSION)" + touch "mcp_changelog.md" + fi + else + echo "HAS_MCP_CHANGES=false" >> $GITHUB_ENV + echo "ℹ No MCP changelog found" + touch "mcp_changelog.md" + fi + + # Combine changelogs in order: UI, API, SDK, MCP > combined_changelog.md if [ "$HAS_UI_CHANGES" = "true" ] && [ -s "ui_changelog.md" ]; then @@ -207,25 +257,22 @@ jobs: echo "" >> combined_changelog.md fi + if [ "$HAS_MCP_CHANGES" = "true" ] && [ -s "mcp_changelog.md" ]; then + echo "## MCP" >> combined_changelog.md + echo "" >> combined_changelog.md + cat mcp_changelog.md >> combined_changelog.md + echo "" >> combined_changelog.md + fi + + # Add fallback message if no changelogs were added + if [ ! -s combined_changelog.md ]; then + echo "No component changes detected for this release." >> combined_changelog.md + fi + echo "Combined changelog preview:" cat combined_changelog.md - - name: Checkout existing branch for patch release - if: ${{ env.PATCH_VERSION != '0' }} - run: | - echo "Patch release detected, checking out existing branch $BRANCH_NAME..." - if git show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then - echo "Branch $BRANCH_NAME exists locally, checking out..." - git checkout "$BRANCH_NAME" - elif git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then - echo "Branch $BRANCH_NAME exists remotely, checking out..." - git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME" - else - echo "ERROR: Branch $BRANCH_NAME should exist for patch release $PROWLER_VERSION" - exit 1 - fi - - - name: Verify version in pyproject.toml + - name: Verify SDK version in pyproject.toml run: | CURRENT_VERSION=$(grep '^version = ' pyproject.toml | sed -E 's/version = "([^"]+)"/\1/' | tr -d '[:space:]') PROWLER_VERSION_TRIMMED=$(echo "$PROWLER_VERSION" | tr -d '[:space:]') @@ -235,7 +282,7 @@ jobs: fi echo "✓ pyproject.toml version: $CURRENT_VERSION" - - name: Verify version in prowler/config/config.py + - name: Verify SDK version in prowler/config/config.py run: | CURRENT_VERSION=$(grep '^prowler_version = ' prowler/config/config.py | sed -E 's/prowler_version = "([^"]+)"/\1/' | tr -d '[:space:]') PROWLER_VERSION_TRIMMED=$(echo "$PROWLER_VERSION" | tr -d '[:space:]') @@ -245,7 +292,7 @@ jobs: fi echo "✓ prowler/config/config.py version: $CURRENT_VERSION" - - name: Verify version in api/pyproject.toml + - name: Verify API version in api/pyproject.toml if: ${{ env.HAS_API_CHANGES == 'true' }} run: | CURRENT_API_VERSION=$(grep '^version = ' api/pyproject.toml | sed -E 's/version = "([^"]+)"/\1/' | tr -d '[:space:]') @@ -256,7 +303,7 @@ jobs: fi echo "✓ api/pyproject.toml version: $CURRENT_API_VERSION" - - name: Verify prowler dependency in api/pyproject.toml + - name: Verify API prowler dependency in api/pyproject.toml if: ${{ env.PATCH_VERSION != '0' && env.HAS_API_CHANGES == 'true' }} run: | CURRENT_PROWLER_REF=$(grep 'prowler @ git+https://github.com/prowler-cloud/prowler.git@' api/pyproject.toml | sed -E 's/.*@([^"]+)".*/\1/' | tr -d '[:space:]') @@ -267,7 +314,7 @@ jobs: fi echo "✓ api/pyproject.toml prowler dependency: $CURRENT_PROWLER_REF" - - name: Verify version in api/src/backend/api/v1/views.py + - name: Verify API version in api/src/backend/api/v1/views.py if: ${{ env.HAS_API_CHANGES == 'true' }} run: | CURRENT_API_VERSION=$(grep 'spectacular_settings.VERSION = ' api/src/backend/api/v1/views.py | sed -E 's/.*spectacular_settings.VERSION = "([^"]+)".*/\1/' | tr -d '[:space:]') @@ -278,31 +325,12 @@ jobs: fi echo "✓ api/src/backend/api/v1/views.py version: $CURRENT_API_VERSION" - - name: Checkout existing release branch for minor release - if: ${{ env.PATCH_VERSION == '0' }} - run: | - echo "Minor release detected (patch = 0), checking out existing branch $BRANCH_NAME..." - if git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then - echo "Branch $BRANCH_NAME exists remotely, checking out..." - git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME" - else - echo "ERROR: Branch $BRANCH_NAME should exist for minor release $PROWLER_VERSION. Please create it manually first." - exit 1 - fi - - - name: Prepare prowler dependency update for minor release + - name: Update API prowler dependency for minor release if: ${{ env.PATCH_VERSION == '0' }} run: | CURRENT_PROWLER_REF=$(grep 'prowler @ git+https://github.com/prowler-cloud/prowler.git@' api/pyproject.toml | sed -E 's/.*@([^"]+)".*/\1/' | tr -d '[:space:]') BRANCH_NAME_TRIMMED=$(echo "$BRANCH_NAME" | tr -d '[:space:]') - # Create a temporary branch for the PR from the minor version branch - TEMP_BRANCH="update-api-dependency-$BRANCH_NAME_TRIMMED-$(date +%s)" - echo "TEMP_BRANCH=$TEMP_BRANCH" >> $GITHUB_ENV - - # Create temp branch from the current minor version branch - git checkout -b "$TEMP_BRANCH" - # Minor release: update the dependency to use the release branch echo "Updating prowler dependency from '$CURRENT_PROWLER_REF' to '$BRANCH_NAME_TRIMMED'" sed -i "s|prowler @ git+https://github.com/prowler-cloud/prowler.git@[^\"]*\"|prowler @ git+https://github.com/prowler-cloud/prowler.git@$BRANCH_NAME_TRIMMED\"|" api/pyproject.toml @@ -320,20 +348,19 @@ jobs: poetry lock cd .. - # Commit and push the temporary branch - git add api/pyproject.toml api/poetry.lock - git commit -m "chore(api): update prowler dependency to $BRANCH_NAME_TRIMMED for release $PROWLER_VERSION" - git push origin "$TEMP_BRANCH" - echo "✓ Prepared prowler dependency update to: $UPDATED_PROWLER_REF" - - name: Create Pull Request against release branch + - name: Create PR for API dependency update if: ${{ env.PATCH_VERSION == '0' }} uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - branch: ${{ env.TEMP_BRANCH }} + commit-message: 'chore(api): update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}' + branch: update-api-dependency-${{ env.BRANCH_NAME }}-${{ github.run_number }} base: ${{ env.BRANCH_NAME }} + add-paths: | + api/pyproject.toml + api/poetry.lock title: "chore(api): Update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}" body: | ### Description @@ -355,7 +382,7 @@ jobs: no-changelog - name: Create draft release - uses: softprops/action-gh-release@6cbd405e2c4e67a21c47fa9e383d020e4e28b836 # v2.3.3 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2.4.1 with: tag_name: ${{ env.PROWLER_VERSION }} name: Prowler ${{ env.PROWLER_VERSION }} @@ -366,5 +393,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Clean up temporary files + if: always() run: | - rm -f prowler_changelog.md api_changelog.md ui_changelog.md combined_changelog.md + rm -f prowler_changelog.md api_changelog.md ui_changelog.md mcp_changelog.md combined_changelog.md diff --git a/.github/workflows/pull-request-check-changelog.yml b/.github/workflows/pull-request-check-changelog.yml deleted file mode 100644 index 47dda976d4..0000000000 --- a/.github/workflows/pull-request-check-changelog.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Prowler - Check Changelog - -on: - pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] - -jobs: - check-changelog: - if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - pull-requests: write - env: - MONITORED_FOLDERS: "api ui prowler dashboard" - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - fetch-depth: 0 - - - name: Get list of changed files - id: changed_files - run: | - git fetch origin ${{ github.base_ref }} - git diff --name-only origin/${{ github.base_ref }}...HEAD > changed_files.txt - cat changed_files.txt - - - name: Check for folder changes and changelog presence - id: check_folders - run: | - missing_changelogs="" - - for folder in $MONITORED_FOLDERS; do - if grep -q "^${folder}/" changed_files.txt; then - echo "Detected changes in ${folder}/" - if ! grep -q "^${folder}/CHANGELOG.md$" changed_files.txt; then - echo "No changelog update found for ${folder}/" - missing_changelogs="${missing_changelogs}- \`${folder}\`\n" - fi - fi - done - - echo "missing_changelogs<> $GITHUB_OUTPUT - echo -e "${missing_changelogs}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Find existing changelog comment - if: github.event.pull_request.head.repo.full_name == github.repository - id: find_comment - uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad #v4.0.0 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: '' - - - name: Update PR comment with changelog status - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-id: ${{ steps.find_comment.outputs.comment-id }} - edit-mode: replace - body: | - - ${{ steps.check_folders.outputs.missing_changelogs != '' && format('⚠ī¸ **Changes detected in the following folders without a corresponding update to the `CHANGELOG.md`:** - - {0} - - Please add an entry to the corresponding `CHANGELOG.md` file to maintain a clear history of changes.', steps.check_folders.outputs.missing_changelogs) || '✅ All necessary `CHANGELOG.md` files have been updated. Great job! 🎉' }} - - - name: Fail if changelog is missing - if: steps.check_folders.outputs.missing_changelogs != '' - run: | - echo "ERROR: Missing changelog updates in some folders." - exit 1 diff --git a/.github/workflows/sdk-build-lint-push-containers.yml b/.github/workflows/sdk-build-lint-push-containers.yml deleted file mode 100644 index 31669924b7..0000000000 --- a/.github/workflows/sdk-build-lint-push-containers.yml +++ /dev/null @@ -1,202 +0,0 @@ -name: SDK - Build and Push containers - -on: - push: - branches: - # For `v3-latest` - - "v3" - # For `v4-latest` - - "v4.6" - # For `latest` - - "master" - paths-ignore: - - ".github/**" - - "README.md" - - "docs/**" - - "ui/**" - - "api/**" - - release: - types: [published] - -env: - # AWS Configuration - AWS_REGION_STG: eu-west-1 - AWS_REGION_PLATFORM: eu-west-1 - AWS_REGION: us-east-1 - - # Container's configuration - IMAGE_NAME: prowler - DOCKERFILE_PATH: ./Dockerfile - - # Tags - LATEST_TAG: latest - STABLE_TAG: stable - # The RELEASE_TAG is set during runtime in releases - RELEASE_TAG: "" - # The PROWLER_VERSION and PROWLER_VERSION_MAJOR are set during runtime in releases - PROWLER_VERSION: "" - PROWLER_VERSION_MAJOR: "" - # TEMPORARY_TAG: temporary - - # Python configuration - PYTHON_VERSION: 3.12 - - # Container Registries - PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud - PROWLERCLOUD_DOCKERHUB_IMAGE: prowler - -jobs: - # Build Prowler OSS container - container-build-push: - # needs: dockerfile-linter - runs-on: ubuntu-latest - outputs: - prowler_version_major: ${{ steps.get-prowler-version.outputs.PROWLER_VERSION_MAJOR }} - prowler_version: ${{ steps.get-prowler-version.outputs.PROWLER_VERSION }} - env: - POETRY_VIRTUALENVS_CREATE: "false" - - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install Poetry - run: | - pipx install poetry==2.* - pipx inject poetry poetry-bumpversion - - - name: Get Prowler version - id: get-prowler-version - run: | - PROWLER_VERSION="$(poetry version -s 2>/dev/null)" - echo "PROWLER_VERSION=${PROWLER_VERSION}" >> "${GITHUB_ENV}" - echo "PROWLER_VERSION=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}" - - # Store prowler version major just for the release - PROWLER_VERSION_MAJOR="${PROWLER_VERSION%%.*}" - echo "PROWLER_VERSION_MAJOR=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_ENV}" - echo "PROWLER_VERSION_MAJOR=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_OUTPUT}" - - case ${PROWLER_VERSION_MAJOR} in - 3) - echo "LATEST_TAG=v3-latest" >> "${GITHUB_ENV}" - echo "STABLE_TAG=v3-stable" >> "${GITHUB_ENV}" - ;; - - - 4) - echo "LATEST_TAG=v4-latest" >> "${GITHUB_ENV}" - echo "STABLE_TAG=v4-stable" >> "${GITHUB_ENV}" - ;; - - 5) - echo "LATEST_TAG=latest" >> "${GITHUB_ENV}" - echo "STABLE_TAG=stable" >> "${GITHUB_ENV}" - ;; - - *) - # Fallback if any other version is present - echo "Releasing another Prowler major version, aborting..." - exit 1 - ;; - esac - - - name: Login to DockerHub - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to Public ECR - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - registry: public.ecr.aws - username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }} - password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }} - env: - AWS_REGION: ${{ env.AWS_REGION }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Build and push container image (latest) - if: github.event_name == 'push' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - push: true - tags: | - ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }} - ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} - file: ${{ env.DOCKERFILE_PATH }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push container image (release) - if: github.event_name == 'release' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - # Use local context to get changes - # https://github.com/docker/build-push-action#path-context - context: . - push: true - tags: | - ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.PROWLER_VERSION }} - ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.STABLE_TAG }} - ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.PROWLER_VERSION }} - ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.STABLE_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.PROWLER_VERSION }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} - file: ${{ env.DOCKERFILE_PATH }} - cache-from: type=gha - cache-to: type=gha,mode=max - -# - name: Push README to Docker Hub (toniblyx) -# uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} -# repository: ${{ env.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }} -# readme-filepath: ./README.md -# -# - name: Push README to Docker Hub (prowlercloud) -# uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} -# repository: ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }} -# readme-filepath: ./README.md - - dispatch-action: - needs: container-build-push - runs-on: ubuntu-latest - steps: - - name: Get latest commit info (latest) - if: github.event_name == 'push' - run: | - LATEST_COMMIT_HASH=$(echo ${{ github.event.after }} | cut -b -7) - echo "LATEST_COMMIT_HASH=${LATEST_COMMIT_HASH}" >> $GITHUB_ENV - - - name: Dispatch event (latest) - if: github.event_name == 'push' && needs.container-build-push.outputs.prowler_version_major == '3' - run: | - curl https://api.github.com/repos/${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}/dispatches \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - --data '{"event_type":"dispatch","client_payload":{"version":"v3-latest", "tag": "${{ env.LATEST_COMMIT_HASH }}"}}' - - - name: Dispatch event (release) - if: github.event_name == 'release' && needs.container-build-push.outputs.prowler_version_major == '3' - run: | - curl https://api.github.com/repos/${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}/dispatches \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - --data '{"event_type":"dispatch","client_payload":{"version":"release", "tag":"${{ needs.container-build-push.outputs.prowler_version }}"}}' diff --git a/.github/workflows/sdk-bump-version.yml b/.github/workflows/sdk-bump-version.yml index e3463e1788..0291502ab6 100644 --- a/.github/workflows/sdk-bump-version.yml +++ b/.github/workflows/sdk-bump-version.yml @@ -1,146 +1,218 @@ -name: SDK - Bump Version +name: 'SDK: Bump Version' on: release: - types: [published] + types: + - 'published' +concurrency: + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false env: PROWLER_VERSION: ${{ github.event.release.tag_name }} BASE_BRANCH: master jobs: - bump-version: - name: Bump Version + detect-release-type: runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + is_minor: ${{ steps.detect.outputs.is_minor }} + is_patch: ${{ steps.detect.outputs.is_patch }} + major_version: ${{ steps.detect.outputs.major_version }} + minor_version: ${{ steps.detect.outputs.minor_version }} + patch_version: ${{ steps.detect.outputs.patch_version }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Get Prowler version - shell: bash + - name: Detect release type and parse version + id: detect run: | if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then MAJOR_VERSION=${BASH_REMATCH[1]} MINOR_VERSION=${BASH_REMATCH[2]} - FIX_VERSION=${BASH_REMATCH[3]} + PATCH_VERSION=${BASH_REMATCH[3]} - # Export version components to GitHub environment - echo "MAJOR_VERSION=${MAJOR_VERSION}" >> "${GITHUB_ENV}" - echo "MINOR_VERSION=${MINOR_VERSION}" >> "${GITHUB_ENV}" - echo "FIX_VERSION=${FIX_VERSION}" >> "${GITHUB_ENV}" + echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}" - if (( MAJOR_VERSION == 5 )); then - if (( FIX_VERSION == 0 )); then - echo "Minor Release: $PROWLER_VERSION" + if (( MAJOR_VERSION != 5 )); then + echo "::error::Releasing another Prowler major version, aborting..." + exit 1 + fi - # Set up next minor version for master - BUMP_VERSION_TO=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).${FIX_VERSION} - echo "BUMP_VERSION_TO=${BUMP_VERSION_TO}" >> "${GITHUB_ENV}" - - TARGET_BRANCH=${BASE_BRANCH} - echo "TARGET_BRANCH=${TARGET_BRANCH}" >> "${GITHUB_ENV}" - - # Set up patch version for version branch - PATCH_VERSION_TO=${MAJOR_VERSION}.${MINOR_VERSION}.1 - echo "PATCH_VERSION_TO=${PATCH_VERSION_TO}" >> "${GITHUB_ENV}" - - VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} - echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" - - echo "Bumping to next minor version: ${BUMP_VERSION_TO} in branch ${TARGET_BRANCH}" - echo "Bumping to next patch version: ${PATCH_VERSION_TO} in branch ${VERSION_BRANCH}" - else - echo "Patch Release: $PROWLER_VERSION" - - BUMP_VERSION_TO=${MAJOR_VERSION}.${MINOR_VERSION}.$((FIX_VERSION + 1)) - echo "BUMP_VERSION_TO=${BUMP_VERSION_TO}" >> "${GITHUB_ENV}" - - TARGET_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} - echo "TARGET_BRANCH=${TARGET_BRANCH}" >> "${GITHUB_ENV}" - - echo "Bumping to next patch version: ${BUMP_VERSION_TO} in branch ${TARGET_BRANCH}" - fi + if (( PATCH_VERSION == 0 )); then + echo "is_minor=true" >> "${GITHUB_OUTPUT}" + echo "is_patch=false" >> "${GITHUB_OUTPUT}" + echo "✓ Minor release detected: $PROWLER_VERSION" else - echo "Releasing another Prowler major version, aborting..." - exit 1 + echo "is_minor=false" >> "${GITHUB_OUTPUT}" + echo "is_patch=true" >> "${GITHUB_OUTPUT}" + echo "✓ Patch release detected: $PROWLER_VERSION" fi else - echo "Invalid version syntax: '$PROWLER_VERSION' (must be N.N.N)" >&2 + echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)" exit 1 fi - - name: Bump versions in files + bump-minor-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_minor == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Calculate next minor version run: | - echo "Using PROWLER_VERSION=$PROWLER_VERSION" - echo "Using BUMP_VERSION_TO=$BUMP_VERSION_TO" + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} - set -e + NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0 + echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}" - echo "Bumping version in pyproject.toml ..." - sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${BUMP_VERSION_TO}\"|" pyproject.toml + echo "Current version: $PROWLER_VERSION" + echo "Next minor version: $NEXT_MINOR_VERSION" - echo "Bumping version in prowler/config/config.py ..." - sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${BUMP_VERSION_TO}\"|" prowler/config/config.py + - name: Bump versions in files for master + run: | + set -e - echo "Bumping version in .env ..." - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${BUMP_VERSION_TO}|" .env + sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_MINOR_VERSION}\"|" pyproject.toml + sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_MINOR_VERSION}\"|" prowler/config/config.py + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env - git --no-pager diff + echo "Files modified:" + git --no-pager diff - - name: Create Pull Request + - name: Create PR for next minor version to master uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: - author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - base: ${{ env.TARGET_BRANCH }} - commit-message: "chore(release): Bump version to v${{ env.BUMP_VERSION_TO }}" - branch: "version-bump-to-v${{ env.BUMP_VERSION_TO }}" - title: "chore(release): Bump version to v${{ env.BUMP_VERSION_TO }}" - labels: no-changelog - body: | - ### Description + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: master + commit-message: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}' + branch: version-bump-to-v${{ env.NEXT_MINOR_VERSION }} + title: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}' + labels: no-changelog + body: | + ### Description - Bump Prowler version to v${{ env.BUMP_VERSION_TO }} + Bump Prowler version to v${{ env.NEXT_MINOR_VERSION }} after releasing v${{ env.PROWLER_VERSION }}. - ### License + ### License - By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. - - name: Handle patch version for minor release - if: env.FIX_VERSION == '0' + - name: Checkout version branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }} + + - name: Calculate first patch version run: | - echo "Using PROWLER_VERSION=$PROWLER_VERSION" - echo "Using PATCH_VERSION_TO=$PATCH_VERSION_TO" + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} - set -e + FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1 + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} - echo "Bumping version in pyproject.toml ..." - sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${PATCH_VERSION_TO}\"|" pyproject.toml + echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" - echo "Bumping version in prowler/config/config.py ..." - sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${PATCH_VERSION_TO}\"|" prowler/config/config.py + echo "First patch version: $FIRST_PATCH_VERSION" + echo "Version branch: $VERSION_BRANCH" - echo "Bumping version in .env ..." - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PATCH_VERSION_TO}|" .env + - name: Bump versions in files for version branch + run: | + set -e - git --no-pager diff + sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${FIRST_PATCH_VERSION}\"|" pyproject.toml + sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${FIRST_PATCH_VERSION}\"|" prowler/config/config.py + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env - - name: Create Pull Request for patch version - if: env.FIX_VERSION == '0' + echo "Files modified:" + git --no-pager diff + + - name: Create PR for first patch version to version branch uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: - author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - base: ${{ env.VERSION_BRANCH }} - commit-message: "chore(release): Bump version to v${{ env.PATCH_VERSION_TO }}" - branch: "version-bump-to-v${{ env.PATCH_VERSION_TO }}" - title: "chore(release): Bump version to v${{ env.PATCH_VERSION_TO }}" - labels: no-changelog - body: | - ### Description + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}' + branch: version-bump-to-v${{ env.FIRST_PATCH_VERSION }} + title: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}' + labels: no-changelog + body: | + ### Description - Bump Prowler version to v${{ env.PATCH_VERSION_TO }} + Bump Prowler version to v${{ env.FIRST_PATCH_VERSION }} in version branch after releasing v${{ env.PROWLER_VERSION }}. - ### License + ### License - By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + bump-patch-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_patch == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Calculate next patch version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }} + + NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1)) + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} + + echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" + + echo "Current version: $PROWLER_VERSION" + echo "Next patch version: $NEXT_PATCH_VERSION" + echo "Target branch: $VERSION_BRANCH" + + - name: Bump versions in files for version branch + run: | + set -e + + sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_PATCH_VERSION}\"|" pyproject.toml + sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_PATCH_VERSION}\"|" prowler/config/config.py + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for next patch version to version branch + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}' + branch: version-bump-to-v${{ env.NEXT_PATCH_VERSION }} + title: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}' + labels: no-changelog + body: | + ### Description + + Bump Prowler version to v${{ env.NEXT_PATCH_VERSION }} after releasing v${{ env.PROWLER_VERSION }}. + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. diff --git a/.github/workflows/sdk-code-quality.yml b/.github/workflows/sdk-code-quality.yml new file mode 100644 index 0000000000..b32324d8cb --- /dev/null +++ b/.github/workflows/sdk-code-quality.yml @@ -0,0 +1,90 @@ +name: 'SDK: Code Quality' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + sdk-code-quality: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.9' + - '3.10' + - '3.11' + - '3.12' + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for SDK changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ./** + files_ignore: | + .github/** + prowler/CHANGELOG.md + docs/** + permissions/** + api/** + ui/** + dashboard/** + mcp_server/** + README.md + mkdocs.yml + .backportrc.json + .env + docker-compose* + examples/** + .gitignore + contrib/** + + - name: Install Poetry + if: steps.check-changes.outputs.any_changed == 'true' + run: pipx install poetry==2.1.1 + + - name: Set up Python ${{ matrix.python-version }} + if: steps.check-changes.outputs.any_changed == 'true' + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + + - name: Install dependencies + if: steps.check-changes.outputs.any_changed == 'true' + run: | + poetry install --no-root + poetry run pip list + + - name: Check Poetry lock file + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry check --lock + + - name: Lint with flake8 + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api + + - name: Check format with black + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run black --exclude api ui --check . + + - name: Lint with pylint + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/ diff --git a/.github/workflows/sdk-codeql.yml b/.github/workflows/sdk-codeql.yml index 302f73c0ca..590ba52d34 100644 --- a/.github/workflows/sdk-codeql.yml +++ b/.github/workflows/sdk-codeql.yml @@ -1,44 +1,41 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: SDK - CodeQL +name: 'SDK: CodeQL' on: push: branches: - - "master" - - "v3" - - "v4.*" - - "v5.*" - paths-ignore: - - 'ui/**' - - 'api/**' - - '.github/**' + - 'master' + - 'v5.*' + paths: + - 'prowler/**' + - 'tests/**' + - 'pyproject.toml' + - '.github/workflows/sdk-codeql.yml' + - '.github/codeql/sdk-codeql-config.yml' + - '!prowler/CHANGELOG.md' pull_request: branches: - - "master" - - "v3" - - "v4.*" - - "v5.*" - paths-ignore: - - 'ui/**' - - 'api/**' - - '.github/**' + - 'master' + - 'v5.*' + paths: + - 'prowler/**' + - 'tests/**' + - 'pyproject.toml' + - '.github/workflows/sdk-codeql.yml' + - '.github/codeql/sdk-codeql-config.yml' + - '!prowler/CHANGELOG.md' schedule: - cron: '00 12 * * *' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - analyze: - name: Analyze + sdk-analyze: + if: github.repository == 'prowler-cloud/prowler' + name: CodeQL Security Analysis runs-on: ubuntu-latest + timeout-minutes: 30 permissions: actions: read contents: read @@ -47,21 +44,20 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'python' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + language: + - 'python' steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 - with: - languages: ${{ matrix.language }} - config-file: ./.github/codeql/sdk-codeql-config.yml + - name: Initialize CodeQL + uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + with: + languages: ${{ matrix.language }} + config-file: ./.github/codeql/sdk-codeql-config.yml - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 - with: - category: "/language:${{matrix.language}}" + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 + with: + category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/sdk-container-build-push.yml b/.github/workflows/sdk-container-build-push.yml new file mode 100644 index 0000000000..1f7343d0cf --- /dev/null +++ b/.github/workflows/sdk-container-build-push.yml @@ -0,0 +1,217 @@ +name: 'SDK: Container Build and Push' + +on: + push: + branches: + - 'v3' # For v3-latest + - 'v4.6' # For v4-latest + - 'master' # For latest + paths-ignore: + - '.github/**' + - '!.github/workflows/sdk-container-build-push.yml' + - 'README.md' + - 'docs/**' + - 'ui/**' + - 'api/**' + release: + types: + - 'published' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + # Container configuration + IMAGE_NAME: prowler + DOCKERFILE_PATH: ./Dockerfile + + # Python configuration + PYTHON_VERSION: '3.12' + + # Tags (dynamically set based on version) + LATEST_TAG: latest + STABLE_TAG: stable + + # Container registries + PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud + PROWLERCLOUD_DOCKERHUB_IMAGE: prowler + + # AWS configuration (for ECR) + AWS_REGION: us-east-1 + +jobs: + container-build-push: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + packages: write + outputs: + prowler_version: ${{ steps.get-prowler-version.outputs.prowler_version }} + prowler_version_major: ${{ steps.get-prowler-version.outputs.prowler_version_major }} + env: + POETRY_VIRTUALENVS_CREATE: 'false' + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Poetry + run: | + pipx install poetry==2.1.1 + pipx inject poetry poetry-bumpversion + + - name: Get Prowler version and set tags + id: get-prowler-version + run: | + PROWLER_VERSION="$(poetry version -s 2>/dev/null)" + echo "prowler_version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}" + echo "PROWLER_VERSION=${PROWLER_VERSION}" >> "${GITHUB_ENV}" + + # Extract major version + PROWLER_VERSION_MAJOR="${PROWLER_VERSION%%.*}" + echo "prowler_version_major=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_OUTPUT}" + echo "PROWLER_VERSION_MAJOR=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_ENV}" + + # Set version-specific tags + case ${PROWLER_VERSION_MAJOR} in + 3) + echo "LATEST_TAG=v3-latest" >> "${GITHUB_ENV}" + echo "STABLE_TAG=v3-stable" >> "${GITHUB_ENV}" + echo "✓ Prowler v3 detected - tags: v3-latest, v3-stable" + ;; + 4) + echo "LATEST_TAG=v4-latest" >> "${GITHUB_ENV}" + echo "STABLE_TAG=v4-stable" >> "${GITHUB_ENV}" + echo "✓ Prowler v4 detected - tags: v4-latest, v4-stable" + ;; + 5) + echo "LATEST_TAG=latest" >> "${GITHUB_ENV}" + echo "STABLE_TAG=stable" >> "${GITHUB_ENV}" + echo "✓ Prowler v5 detected - tags: latest, stable" + ;; + *) + echo "::error::Unsupported Prowler major version: ${PROWLER_VERSION_MAJOR}" + exit 1 + ;; + esac + + - name: Login to DockerHub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Public ECR + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: public.ecr.aws + username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }} + password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }} + env: + AWS_REGION: ${{ env.AWS_REGION }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build and push SDK container (latest) + if: github.event_name == 'push' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + file: ${{ env.DOCKERFILE_PATH }} + push: true + tags: | + ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }} + ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: SDK + RELEASE_TAG: ${{ env.PROWLER_VERSION }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + - name: Build and push SDK container (release) + if: github.event_name == 'release' + id: container-push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + file: ${{ env.DOCKERFILE_PATH }} + push: true + tags: | + ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.PROWLER_VERSION }} + ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.STABLE_TAG }} + ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.PROWLER_VERSION }} + ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.STABLE_TAG }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.PROWLER_VERSION }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: SDK + RELEASE_TAG: ${{ env.PROWLER_VERSION }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} + + dispatch-v3-deployment: + if: needs.container-build-push.outputs.prowler_version_major == '3' + needs: container-build-push + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Calculate short SHA + id: short-sha + run: echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + - name: Dispatch v3 deployment (latest) + if: github.event_name == 'push' + uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + with: + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }} + event-type: dispatch + client-payload: '{"version":"v3-latest","tag":"${{ steps.short-sha.outputs.short_sha }}"}' + + - name: Dispatch v3 deployment (release) + if: github.event_name == 'release' + uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + with: + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }} + event-type: dispatch + client-payload: '{"version":"release","tag":"${{ needs.container-build-push.outputs.prowler_version }}"}' diff --git a/.github/workflows/sdk-container-checks.yml b/.github/workflows/sdk-container-checks.yml new file mode 100644 index 0000000000..4e8a7cf2ad --- /dev/null +++ b/.github/workflows/sdk-container-checks.yml @@ -0,0 +1,103 @@ +name: 'SDK: Container Checks' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + IMAGE_NAME: prowler + +jobs: + sdk-dockerfile-lint: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check if Dockerfile changed + id: dockerfile-changed + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: Dockerfile + + - name: Lint Dockerfile with Hadolint + if: steps.dockerfile-changed.outputs.any_changed == 'true' + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: Dockerfile + ignore: DL3013 + + sdk-container-build-and-scan: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for SDK changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ./** + files_ignore: | + .github/** + prowler/CHANGELOG.md + docs/** + permissions/** + api/** + ui/** + dashboard/** + mcp_server/** + README.md + mkdocs.yml + .backportrc.json + .env + docker-compose* + examples/** + .gitignore + contrib/** + + - name: Set up Docker Buildx + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build SDK container + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Scan SDK container with Trivy + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/trivy-scan + with: + image-name: ${{ env.IMAGE_NAME }} + image-tag: ${{ github.sha }} + fail-on-critical: 'false' + severity: 'CRITICAL' diff --git a/.github/workflows/sdk-pull-request.yml b/.github/workflows/sdk-pull-request.yml deleted file mode 100644 index bfb8017001..0000000000 --- a/.github/workflows/sdk-pull-request.yml +++ /dev/null @@ -1,271 +0,0 @@ -name: SDK - Pull Request - -on: - push: - branches: - - "master" - - "v3" - - "v4.*" - - "v5.*" - pull_request: - branches: - - "master" - - "v3" - - "v4.*" - - "v5.*" -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Test if changes are in not ignored paths - id: are-non-ignored-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: ./** - files_ignore: | - .github/** - docs/** - permissions/** - api/** - ui/** - prowler/CHANGELOG.md - README.md - mkdocs.yml - .backportrc.json - .env - docker-compose* - examples/** - .gitignore - - - name: Install poetry - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - python -m pip install --upgrade pip - pipx install poetry==2.1.1 - - - name: Set up Python ${{ matrix.python-version }} - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ matrix.python-version }} - cache: "poetry" - - - name: Install dependencies - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry install --no-root - poetry run pip list - VERSION=$(curl --silent "https://api.github.com/repos/hadolint/hadolint/releases/latest" | \ - grep '"tag_name":' | \ - sed -E 's/.*"v([^"]+)".*/\1/' \ - ) && curl -L -o /tmp/hadolint "https://github.com/hadolint/hadolint/releases/download/v${VERSION}/hadolint-Linux-x86_64" \ - && chmod +x /tmp/hadolint - - - name: Poetry check - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry check --lock - - - name: Lint with flake8 - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api - - - name: Checking format with black - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run black --exclude api ui --check . - - - name: Lint with pylint - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/ - - - name: Bandit - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run bandit -q -lll -x '*_test.py,./contrib/,./api/,./ui' -r . - - - name: Safety - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run safety check --ignore 70612 -r pyproject.toml - - - name: Vulture - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run vulture --exclude "contrib,api,ui" --min-confidence 100 . - - - name: Dockerfile - Check if Dockerfile has changed - id: dockerfile-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - Dockerfile - - - name: Hadolint - if: steps.dockerfile-changed-files.outputs.any_changed == 'true' - run: | - /tmp/hadolint Dockerfile --ignore=DL3013 - - # Test AWS - - name: AWS - Check if any file has changed - id: aws-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/aws/** - ./tests/providers/aws/** - ./poetry.lock - - - name: AWS - Test - if: steps.aws-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws - - # Test Azure - - name: Azure - Check if any file has changed - id: azure-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/azure/** - ./tests/providers/azure/** - ./poetry.lock - - - name: Azure - Test - if: steps.azure-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/azure --cov-report=xml:azure_coverage.xml tests/providers/azure - - # Test GCP - - name: GCP - Check if any file has changed - id: gcp-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/gcp/** - ./tests/providers/gcp/** - ./poetry.lock - - - name: GCP - Test - if: steps.gcp-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/gcp --cov-report=xml:gcp_coverage.xml tests/providers/gcp - - # Test Kubernetes - - name: Kubernetes - Check if any file has changed - id: kubernetes-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/kubernetes/** - ./tests/providers/kubernetes/** - ./poetry.lock - - - name: Kubernetes - Test - if: steps.kubernetes-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/kubernetes --cov-report=xml:kubernetes_coverage.xml tests/providers/kubernetes - - # Test GitHub - - name: GitHub - Check if any file has changed - id: github-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/github/** - ./tests/providers/github/** - ./poetry.lock - - - name: GitHub - Test - if: steps.github-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/github --cov-report=xml:github_coverage.xml tests/providers/github - - # Test NHN - - name: NHN - Check if any file has changed - id: nhn-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/nhn/** - ./tests/providers/nhn/** - ./poetry.lock - - - name: NHN - Test - if: steps.nhn-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/nhn --cov-report=xml:nhn_coverage.xml tests/providers/nhn - - # Test M365 - - name: M365 - Check if any file has changed - id: m365-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/m365/** - ./tests/providers/m365/** - ./poetry.lock - - - name: M365 - Test - if: steps.m365-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/m365 --cov-report=xml:m365_coverage.xml tests/providers/m365 - - # Test IaC - - name: IaC - Check if any file has changed - id: iac-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/iac/** - ./tests/providers/iac/** - ./poetry.lock - - - name: IaC - Test - if: steps.iac-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/iac --cov-report=xml:iac_coverage.xml tests/providers/iac - - # Test MongoDB Atlas - - name: MongoDB Atlas - Check if any file has changed - id: mongodb-atlas-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/mongodbatlas/** - ./tests/providers/mongodbatlas/** - .poetry.lock - - - name: MongoDB Atlas - Test - if: steps.mongodb-atlas-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/mongodbatlas --cov-report=xml:mongodb_atlas_coverage.xml tests/providers/mongodbatlas - - # Common Tests - - name: Lib - Test - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/lib --cov-report=xml:lib_coverage.xml tests/lib - - - name: Config - Test - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/config --cov-report=xml:config_coverage.xml tests/config - - # Codecov - - name: Upload coverage reports to Codecov - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - flags: prowler - files: ./aws_coverage.xml,./azure_coverage.xml,./gcp_coverage.xml,./kubernetes_coverage.xml,./github_coverage.xml,./nhn_coverage.xml,./m365_coverage.xml,./lib_coverage.xml,./config_coverage.xml diff --git a/.github/workflows/sdk-pypi-release.yml b/.github/workflows/sdk-pypi-release.yml index 59cc948340..0f74ba054c 100644 --- a/.github/workflows/sdk-pypi-release.yml +++ b/.github/workflows/sdk-pypi-release.yml @@ -1,98 +1,119 @@ -name: SDK - PyPI release +name: 'SDK: PyPI Release' on: release: - types: [published] + types: + - 'published' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false env: RELEASE_TAG: ${{ github.event.release.tag_name }} - PYTHON_VERSION: 3.11 - # CACHE: "poetry" + PYTHON_VERSION: '3.12' jobs: - repository-check: - name: Repository check + validate-release: + if: github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read outputs: - is_repo: ${{ steps.repository_check.outputs.is_repo }} - steps: - - name: Repository check - id: repository_check - working-directory: /tmp - run: | - if [[ ${{ github.repository }} == "prowler-cloud/prowler" ]] - then - echo "is_repo=true" >> "${GITHUB_OUTPUT}" - else - echo "This action only runs for prowler-cloud/prowler" - echo "is_repo=false" >> "${GITHUB_OUTPUT}" - fi + prowler_version: ${{ steps.parse-version.outputs.version }} + major_version: ${{ steps.parse-version.outputs.major }} - release-prowler-job: - runs-on: ubuntu-latest - needs: repository-check - if: needs.repository-check.outputs.is_repo == 'true' - env: - POETRY_VIRTUALENVS_CREATE: "false" - name: Release Prowler to PyPI steps: - - name: Repository check - working-directory: /tmp - run: | - if [[ "${{ github.repository }}" != "prowler-cloud/prowler" ]]; then - echo "This action only runs for prowler-cloud/prowler" - exit 1 - fi - - - name: Get Prowler version + - name: Parse and validate version + id: parse-version run: | PROWLER_VERSION="${{ env.RELEASE_TAG }}" + echo "version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}" - case ${PROWLER_VERSION%%.*} in - 3) - echo "Releasing Prowler v3 with tag ${PROWLER_VERSION}" + # Extract major version + MAJOR_VERSION="${PROWLER_VERSION%%.*}" + echo "major=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}" + + # Validate major version + case ${MAJOR_VERSION} in + 3|4|5) + echo "✓ Releasing Prowler v${MAJOR_VERSION} with tag ${PROWLER_VERSION}" ;; - 4) - echo "Releasing Prowler v4 with tag ${PROWLER_VERSION}" - ;; - 5) - echo "Releasing Prowler v5 with tag ${PROWLER_VERSION}" - ;; - *) - echo "Releasing another Prowler major version, aborting..." + *) + echo "::error::Unsupported Prowler major version: ${MAJOR_VERSION}" exit 1 ;; esac - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + publish-prowler: + needs: validate-release + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + environment: + name: pypi-prowler + url: https://pypi.org/project/prowler/${{ needs.validate-release.outputs.prowler_version }}/ - - name: Install dependencies - run: | - pipx install poetry==2.1.1 + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Setup Python + - name: Install Poetry + run: pipx install poetry==2.1.1 + + - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - # cache: ${{ env.CACHE }} + cache: 'poetry' - name: Build Prowler package - run: | - poetry build + run: poetry build - name: Publish Prowler package to PyPI - run: | - poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }} - poetry publish + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + print-hash: true - - name: Replicate PyPI package + publish-prowler-cloud: + needs: validate-release + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + environment: + name: pypi-prowler-cloud + url: https://pypi.org/project/prowler-cloud/${{ needs.validate-release.outputs.prowler_version }}/ + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Install Poetry + run: pipx install poetry==2.1.1 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'poetry' + + - name: Install toml package + run: pip install toml + + - name: Replicate PyPI package for prowler-cloud run: | - rm -rf ./dist && rm -rf ./build && rm -rf prowler.egg-info - pip install toml + rm -rf ./dist ./build prowler.egg-info python util/replicate_pypi_package.py - poetry build + + - name: Build prowler-cloud package + run: poetry build - name: Publish prowler-cloud package to PyPI - run: | - poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }} - poetry publish + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + print-hash: true diff --git a/.github/workflows/sdk-refresh-aws-services-regions.yml b/.github/workflows/sdk-refresh-aws-services-regions.yml index ded41cdc02..884b677294 100644 --- a/.github/workflows/sdk-refresh-aws-services-regions.yml +++ b/.github/workflows/sdk-refresh-aws-services-regions.yml @@ -1,68 +1,90 @@ -# This is a basic workflow to help you get started with Actions - -name: SDK - Refresh AWS services' regions +name: 'SDK: Refresh AWS Regions' on: schedule: - - cron: "0 9 * * 1" # runs at 09:00 UTC every Monday + - cron: '0 9 * * 1' # Every Monday at 09:00 UTC + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false env: - GITHUB_BRANCH: "master" - AWS_REGION_DEV: us-east-1 + PYTHON_VERSION: '3.12' + AWS_REGION: 'us-east-1' -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on + refresh-aws-regions: + if: github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest + timeout-minutes: 15 permissions: id-token: write pull-requests: write contents: write - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ env.GITHUB_BRANCH }} - - name: setup python + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: 'master' + + - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - python-version: 3.9 #install the python needed + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install boto3 + run: pip install boto3 - - name: Configure AWS Credentials -- DEV - uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0 with: - aws-region: ${{ env.AWS_REGION_DEV }} + aws-region: ${{ env.AWS_REGION }} role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }} - role-session-name: refresh-AWS-regions-dev + role-session-name: prowler-refresh-aws-regions - # Runs a single command using the runners shell - - name: Run a one-line script - run: python3 util/update_aws_services_regions.py + - name: Update AWS services regions + run: python util/update_aws_services_regions.py - # Create pull request - - name: Create Pull Request + - name: Create pull request + id: create-pr uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: - author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - commit-message: "feat(regions_update): Update regions for AWS services" - branch: "aws-services-regions-updated-${{ github.sha }}" - labels: "status/waiting-for-revision, severity/low, provider/aws, no-changelog" - title: "chore(regions_update): Changes in regions for AWS services" + author: 'prowler-bot <179230569+prowler-bot@users.noreply.github.com>' + committer: 'github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>' + commit-message: 'feat(aws): update regions for AWS services' + branch: 'aws-regions-update-${{ github.run_number }}' + title: 'feat(aws): Update regions for AWS services' + labels: | + status/waiting-for-revision + severity/low + provider/aws + no-changelog body: | ### Description - This PR updates the regions for AWS services. + Automated update of AWS service regions from the official AWS IP ranges. + + **Trigger:** ${{ github.event_name == 'schedule' && 'Scheduled (weekly)' || github.event_name == 'workflow_dispatch' && 'Manual' || 'Workflow update' }} + **Run:** [#${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + + ### Checklist + + - [x] This is an automated update from AWS official sources + - [x] No manual review of region data required ### License By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + - name: PR creation result + run: | + if [[ "${{ steps.create-pr.outputs.pull-request-number }}" ]]; then + echo "✓ Pull request #${{ steps.create-pr.outputs.pull-request-number }} created successfully" + echo "URL: ${{ steps.create-pr.outputs.pull-request-url }}" + else + echo "✓ No changes detected - AWS regions are up to date" + fi diff --git a/.github/workflows/sdk-security.yml b/.github/workflows/sdk-security.yml new file mode 100644 index 0000000000..a15bd2539f --- /dev/null +++ b/.github/workflows/sdk-security.yml @@ -0,0 +1,77 @@ +name: 'SDK: Security' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + sdk-security-scans: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for SDK changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ./** + files_ignore: | + .github/** + prowler/CHANGELOG.md + docs/** + permissions/** + api/** + ui/** + dashboard/** + mcp_server/** + README.md + mkdocs.yml + .backportrc.json + .env + docker-compose* + examples/** + .gitignore + contrib/** + + - name: Install Poetry + if: steps.check-changes.outputs.any_changed == 'true' + run: pipx install poetry==2.1.1 + + - name: Set up Python 3.12 + if: steps.check-changes.outputs.any_changed == 'true' + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.12' + cache: 'poetry' + + - name: Install dependencies + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry install --no-root + + - name: Security scan with Bandit + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run bandit -q -lll -x '*_test.py,./contrib/,./api/,./ui' -r . + + - name: Security scan with Safety + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run safety check --ignore 70612 -r pyproject.toml + + - name: Dead code detection with Vulture + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run vulture --exclude "contrib,api,ui" --min-confidence 100 . diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml new file mode 100644 index 0000000000..b01c5e7778 --- /dev/null +++ b/.github/workflows/sdk-tests.yml @@ -0,0 +1,360 @@ +name: 'SDK: Tests' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + sdk-tests: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.9' + - '3.10' + - '3.11' + - '3.12' + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for SDK changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ./** + files_ignore: | + .github/** + prowler/CHANGELOG.md + docs/** + permissions/** + api/** + ui/** + dashboard/** + mcp_server/** + README.md + mkdocs.yml + .backportrc.json + .env + docker-compose* + examples/** + .gitignore + contrib/** + + - name: Install Poetry + if: steps.check-changes.outputs.any_changed == 'true' + run: pipx install poetry==2.1.1 + + - name: Set up Python ${{ matrix.python-version }} + if: steps.check-changes.outputs.any_changed == 'true' + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + + - name: Install dependencies + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry install --no-root + + # AWS Provider + - name: Check if AWS files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-aws + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/aws/** + ./tests/**/aws/** + ./poetry.lock + + - name: Run AWS tests + if: steps.changed-aws.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws + + - name: Upload AWS coverage to Codecov + if: steps.changed-aws.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-aws + files: ./aws_coverage.xml + + # Azure Provider + - name: Check if Azure files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-azure + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/azure/** + ./tests/**/azure/** + ./poetry.lock + + - name: Run Azure tests + if: steps.changed-azure.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/azure --cov-report=xml:azure_coverage.xml tests/providers/azure + + - name: Upload Azure coverage to Codecov + if: steps.changed-azure.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-azure + files: ./azure_coverage.xml + + # GCP Provider + - name: Check if GCP files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-gcp + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/gcp/** + ./tests/**/gcp/** + ./poetry.lock + + - name: Run GCP tests + if: steps.changed-gcp.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/gcp --cov-report=xml:gcp_coverage.xml tests/providers/gcp + + - name: Upload GCP coverage to Codecov + if: steps.changed-gcp.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-gcp + files: ./gcp_coverage.xml + + # Kubernetes Provider + - name: Check if Kubernetes files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-kubernetes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/kubernetes/** + ./tests/**/kubernetes/** + ./poetry.lock + + - name: Run Kubernetes tests + if: steps.changed-kubernetes.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/kubernetes --cov-report=xml:kubernetes_coverage.xml tests/providers/kubernetes + + - name: Upload Kubernetes coverage to Codecov + if: steps.changed-kubernetes.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-kubernetes + files: ./kubernetes_coverage.xml + + # GitHub Provider + - name: Check if GitHub files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-github + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/github/** + ./tests/**/github/** + ./poetry.lock + + - name: Run GitHub tests + if: steps.changed-github.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/github --cov-report=xml:github_coverage.xml tests/providers/github + + - name: Upload GitHub coverage to Codecov + if: steps.changed-github.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-github + files: ./github_coverage.xml + + # NHN Provider + - name: Check if NHN files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-nhn + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/nhn/** + ./tests/**/nhn/** + ./poetry.lock + + - name: Run NHN tests + if: steps.changed-nhn.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/nhn --cov-report=xml:nhn_coverage.xml tests/providers/nhn + + - name: Upload NHN coverage to Codecov + if: steps.changed-nhn.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-nhn + files: ./nhn_coverage.xml + + # M365 Provider + - name: Check if M365 files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-m365 + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/m365/** + ./tests/**/m365/** + ./poetry.lock + + - name: Run M365 tests + if: steps.changed-m365.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/m365 --cov-report=xml:m365_coverage.xml tests/providers/m365 + + - name: Upload M365 coverage to Codecov + if: steps.changed-m365.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-m365 + files: ./m365_coverage.xml + + # IaC Provider + - name: Check if IaC files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-iac + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/iac/** + ./tests/**/iac/** + ./poetry.lock + + - name: Run IaC tests + if: steps.changed-iac.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/iac --cov-report=xml:iac_coverage.xml tests/providers/iac + + - name: Upload IaC coverage to Codecov + if: steps.changed-iac.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-iac + files: ./iac_coverage.xml + + # MongoDB Atlas Provider + - name: Check if MongoDB Atlas files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-mongodbatlas + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/mongodbatlas/** + ./tests/**/mongodbatlas/** + ./poetry.lock + + - name: Run MongoDB Atlas tests + if: steps.changed-mongodbatlas.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/mongodbatlas --cov-report=xml:mongodbatlas_coverage.xml tests/providers/mongodbatlas + + - name: Upload MongoDB Atlas coverage to Codecov + if: steps.changed-mongodbatlas.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-mongodbatlas + files: ./mongodbatlas_coverage.xml + + # OCI Provider + - name: Check if OCI files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-oraclecloud + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/oraclecloud/** + ./tests/**/oraclecloud/** + ./poetry.lock + + - name: Run OCI tests + if: steps.changed-oraclecloud.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/oraclecloud --cov-report=xml:oraclecloud_coverage.xml tests/providers/oraclecloud + + - name: Upload OCI coverage to Codecov + if: steps.changed-oraclecloud.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-oraclecloud + files: ./oraclecloud_coverage.xml + + # Lib + - name: Check if Lib files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-lib + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/lib/** + ./tests/lib/** + ./poetry.lock + + - name: Run Lib tests + if: steps.changed-lib.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/lib --cov-report=xml:lib_coverage.xml tests/lib + + - name: Upload Lib coverage to Codecov + if: steps.changed-lib.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-lib + files: ./lib_coverage.xml + + # Config + - name: Check if Config files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-config + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/config/** + ./tests/config/** + ./poetry.lock + + - name: Run Config tests + if: steps.changed-config.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/config --cov-report=xml:config_coverage.xml tests/config + + - name: Upload Config coverage to Codecov + if: steps.changed-config.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-config + files: ./config_coverage.xml diff --git a/.github/workflows/ui-build-lint-push-containers.yml b/.github/workflows/ui-build-lint-push-containers.yml deleted file mode 100644 index 16d9d80fbb..0000000000 --- a/.github/workflows/ui-build-lint-push-containers.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: UI - Build and Push containers - -on: - push: - branches: - - "master" - paths: - - "ui/**" - - ".github/workflows/ui-build-lint-push-containers.yml" - - # Uncomment the below code to test this action on PRs - # pull_request: - # branches: - # - "master" - # paths: - # - "ui/**" - # - ".github/workflows/ui-build-lint-push-containers.yml" - - release: - types: [published] - -env: - # Tags - LATEST_TAG: latest - RELEASE_TAG: ${{ github.event.release.tag_name }} - STABLE_TAG: stable - - WORKING_DIRECTORY: ./ui - - # Container Registries - PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud - PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-ui - NEXT_PUBLIC_API_BASE_URL: http://prowler-api:8080/api/v1 - -jobs: - repository-check: - name: Repository check - runs-on: ubuntu-latest - outputs: - is_repo: ${{ steps.repository_check.outputs.is_repo }} - steps: - - name: Repository check - id: repository_check - working-directory: /tmp - run: | - if [[ ${{ github.repository }} == "prowler-cloud/prowler" ]] - then - echo "is_repo=true" >> "${GITHUB_OUTPUT}" - else - echo "This action only runs for prowler-cloud/prowler" - echo "is_repo=false" >> "${GITHUB_OUTPUT}" - fi - - # Build Prowler OSS container - container-build-push: - needs: repository-check - if: needs.repository-check.outputs.is_repo == 'true' - runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ env.WORKING_DIRECTORY }} - - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Set short git commit SHA - id: vars - run: | - shortSha=$(git rev-parse --short ${{ github.sha }}) - echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV - - - name: Login to DockerHub - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Build and push container image (latest) - # Comment the following line for testing - if: github.event_name == 'push' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.WORKING_DIRECTORY }} - build-args: | - NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ env.SHORT_SHA }} - NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }} - # Set push: false for testing - push: true - tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.SHORT_SHA }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push container image (release) - if: github.event_name == 'release' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.WORKING_DIRECTORY }} - build-args: | - NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${{ env.RELEASE_TAG }} - NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }} - push: true - tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Trigger deployment - if: github.event_name == 'push' - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 - with: - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - repository: ${{ secrets.CLOUD_DISPATCH }} - event-type: prowler-ui-deploy - client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ env.SHORT_SHA }}"}' diff --git a/.github/workflows/ui-codeql.yml b/.github/workflows/ui-codeql.yml index 05ab6e6123..2b55cf673c 100644 --- a/.github/workflows/ui-codeql.yml +++ b/.github/workflows/ui-codeql.yml @@ -1,36 +1,37 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: UI - CodeQL +name: 'UI: CodeQL' on: push: branches: - - "master" - - "v5.*" + - 'master' + - 'v5.*' paths: - - "ui/**" + - 'ui/**' + - '.github/workflows/ui-codeql.yml' + - '.github/codeql/ui-codeql-config.yml' + - '!ui/CHANGELOG.md' pull_request: branches: - - "master" - - "v5.*" + - 'master' + - 'v5.*' paths: - - "ui/**" + - 'ui/**' + - '.github/workflows/ui-codeql.yml' + - '.github/codeql/ui-codeql-config.yml' + - '!ui/CHANGELOG.md' schedule: - - cron: "00 12 * * *" + - cron: '00 12 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - analyze: - name: Analyze + ui-analyze: + if: github.repository == 'prowler-cloud/prowler' + name: CodeQL Security Analysis runs-on: ubuntu-latest + timeout-minutes: 30 permissions: actions: read contents: read @@ -39,21 +40,20 @@ jobs: strategy: fail-fast: false matrix: - language: ["javascript"] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + language: + - 'javascript-typescript' steps: - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/ui-codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: - category: "/language:${{matrix.language}}" + category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/ui-container-build-push.yml b/.github/workflows/ui-container-build-push.yml new file mode 100644 index 0000000000..6641903953 --- /dev/null +++ b/.github/workflows/ui-container-build-push.yml @@ -0,0 +1,143 @@ +name: 'UI: Container Build and Push' + +on: + push: + branches: + - 'master' + paths: + - 'ui/**' + - '.github/workflows/ui-container-build-push.yml' + release: + types: + - 'published' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + # Tags + LATEST_TAG: latest + RELEASE_TAG: ${{ github.event.release.tag_name }} + STABLE_TAG: stable + WORKING_DIRECTORY: ./ui + + # Container registries + PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud + PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-ui + + # Build args + NEXT_PUBLIC_API_BASE_URL: http://prowler-api:8080/api/v1 + +jobs: + setup: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + short-sha: ${{ steps.set-short-sha.outputs.short-sha }} + steps: + - name: Calculate short SHA + id: set-short-sha + run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + container-build-push: + needs: setup + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Login to DockerHub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build and push UI container (latest) + if: github.event_name == 'push' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.WORKING_DIRECTORY }} + build-args: | + NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ needs.setup.outputs.short-sha }} + NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }} + push: true + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: UI + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + - name: Build and push UI container (release) + if: github.event_name == 'release' + id: container-push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.WORKING_DIRECTORY }} + build-args: | + NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${{ env.RELEASE_TAG }} + NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }} + push: true + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: UI + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} + + trigger-deployment: + if: github.event_name == 'push' + needs: [setup, container-build-push] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Trigger UI deployment + uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + with: + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + repository: ${{ secrets.CLOUD_DISPATCH }} + event-type: ui-prowler-deployment + client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}' diff --git a/.github/workflows/ui-container-checks.yml b/.github/workflows/ui-container-checks.yml new file mode 100644 index 0000000000..e436c3974a --- /dev/null +++ b/.github/workflows/ui-container-checks.yml @@ -0,0 +1,91 @@ +name: 'UI: Container Checks' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + UI_WORKING_DIR: ./ui + IMAGE_NAME: prowler-ui + +jobs: + ui-dockerfile-lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check if Dockerfile changed + id: dockerfile-changed + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ui/Dockerfile + + - name: Lint Dockerfile with Hadolint + if: steps.dockerfile-changed.outputs.any_changed == 'true' + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: ui/Dockerfile + ignore: DL3018 + + ui-container-build-and-scan: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for UI changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ui/** + files_ignore: | + ui/CHANGELOG.md + ui/README.md + + - name: Set up Docker Buildx + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build UI container + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.UI_WORKING_DIR }} + target: prod + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX + + - name: Scan UI container with Trivy + if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/trivy-scan + with: + image-name: ${{ env.IMAGE_NAME }} + image-tag: ${{ github.sha }} + fail-on-critical: 'false' + severity: 'CRITICAL' diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index dea6f1f3e2..ed3b692819 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -18,6 +18,22 @@ jobs: AUTH_TRUST_HOST: true NEXTAUTH_URL: 'http://localhost:3000' NEXT_PUBLIC_API_BASE_URL: 'http://localhost:8080/api/v1' + E2E_ADMIN_USER: ${{ secrets.E2E_ADMIN_USER }} + E2E_ADMIN_PASSWORD: ${{ secrets.E2E_ADMIN_PASSWORD }} + E2E_AWS_PROVIDER_ACCOUNT_ID: ${{ secrets.E2E_AWS_PROVIDER_ACCOUNT_ID }} + E2E_AWS_PROVIDER_ACCESS_KEY: ${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }} + E2E_AWS_PROVIDER_SECRET_KEY: ${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }} + E2E_AWS_PROVIDER_ROLE_ARN: ${{ secrets.E2E_AWS_PROVIDER_ROLE_ARN }} + E2E_AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_AZURE_SUBSCRIPTION_ID }} + E2E_AZURE_CLIENT_ID: ${{ secrets.E2E_AZURE_CLIENT_ID }} + E2E_AZURE_SECRET_ID: ${{ secrets.E2E_AZURE_SECRET_ID }} + E2E_AZURE_TENANT_ID: ${{ secrets.E2E_AZURE_TENANT_ID }} + E2E_M365_DOMAIN_ID: ${{ secrets.E2E_M365_DOMAIN_ID }} + E2E_M365_CLIENT_ID: ${{ secrets.E2E_M365_CLIENT_ID }} + E2E_M365_SECRET_ID: ${{ secrets.E2E_M365_SECRET_ID }} + E2E_M365_TENANT_ID: ${{ secrets.E2E_M365_TENANT_ID }} + E2E_M365_CERTIFICATE_CONTENT: ${{ secrets.E2E_M365_CERTIFICATE_CONTENT }} + E2E_NEW_PASSWORD: ${{ secrets.E2E_NEW_PASSWORD }} steps: - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -59,7 +75,7 @@ jobs: echo "All database fixtures loaded successfully!" ' - name: Setup Node.js environment - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 with: node-version: '20.x' cache: 'npm' diff --git a/.github/workflows/ui-pull-request.yml b/.github/workflows/ui-pull-request.yml deleted file mode 100644 index 22af5073a7..0000000000 --- a/.github/workflows/ui-pull-request.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: UI - Pull Request - -on: - push: - branches: - - "master" - - "v5.*" - paths: - - ".github/workflows/ui-pull-request.yml" - - "ui/**" - pull_request: - branches: - - master - - "v5.*" - paths: - - 'ui/**' -env: - UI_WORKING_DIR: ./ui - IMAGE_NAME: prowler-ui - -jobs: - test-and-coverage: - runs-on: ubuntu-latest - strategy: - matrix: - os: [ubuntu-latest] - node-version: [20.x] - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - - name: Setup Node.js ${{ matrix.node-version }} - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 - with: - node-version: ${{ matrix.node-version }} - cache: 'npm' - cache-dependency-path: './ui/package-lock.json' - - name: Install dependencies - working-directory: ./ui - run: npm ci - - name: Run Healthcheck - working-directory: ./ui - run: npm run healthcheck - - name: Build the application - working-directory: ./ui - run: npm run build - - test-container-build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - name: Build Container - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.UI_WORKING_DIR }} - # Always build using `prod` target - target: prod - push: false - tags: ${{ env.IMAGE_NAME }}:latest - outputs: type=docker - build-args: | - NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml new file mode 100644 index 0000000000..a5a5b5fa17 --- /dev/null +++ b/.github/workflows/ui-tests.yml @@ -0,0 +1,64 @@ +name: 'UI: Tests' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + UI_WORKING_DIR: ./ui + NODE_VERSION: '20.x' + +jobs: + ui-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + defaults: + run: + working-directory: ./ui + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for UI changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ui/** + .github/workflows/ui-tests.yml + files_ignore: | + ui/CHANGELOG.md + ui/README.md + + - name: Setup Node.js ${{ env.NODE_VERSION }} + if: steps.check-changes.outputs.any_changed == 'true' + uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: './ui/package-lock.json' + + - name: Install dependencies + if: steps.check-changes.outputs.any_changed == 'true' + run: npm ci + + - name: Run healthcheck + if: steps.check-changes.outputs.any_changed == 'true' + run: npm run healthcheck + + - name: Build application + if: steps.check-changes.outputs.any_changed == 'true' + run: npm run build diff --git a/.gitignore b/.gitignore index 4b39b18b83..7792a031b4 100644 --- a/.gitignore +++ b/.gitignore @@ -39,21 +39,92 @@ secrets-*/ # JUnit Reports junit-reports/ -# VSCode files -.vscode/ +# Test and coverage artifacts +*_coverage.xml +pytest_*.xml +.coverage +htmlcov/ -# Cursor files +# VSCode files and settings +.vscode/ +*.code-workspace +.vscode-test/ + +# VSCode extension settings and workspaces +.history/ +.ionide/ + +# MCP Server Settings (various locations) +**/cline_mcp_settings.json +**/mcp_settings.json +**/mcp-config.json +**/mcpServers.json +.mcp/ + +# AI Coding Assistants - Cursor .cursorignore .cursor/ +.cursorrules -# RooCode files +# AI Coding Assistants - RooCode .roo/ .rooignore .roomodes -# Cline files +# AI Coding Assistants - Cline (formerly Claude Dev) .cline/ .clineignore +.clinerules + +# AI Coding Assistants - Continue +.continue/ +continue.json +.continuerc +.continuerc.json + +# AI Coding Assistants - GitHub Copilot +.copilot/ +.github/copilot/ + +# AI Coding Assistants - Amazon Q Developer (formerly CodeWhisperer) +.aws/ +.codewhisperer/ +.amazonq/ +.aws-toolkit/ + +# AI Coding Assistants - Tabnine +.tabnine/ +tabnine_config.json + +# AI Coding Assistants - Kiro +.kiro/ +.kiroignore +kiro.config.json + +# AI Coding Assistants - Aider +.aider/ +.aider.chat.history.md +.aider.input.history +.aider.tags.cache.v3/ + +# AI Coding Assistants - Windsurf +.windsurf/ +.windsurfignore + +# AI Coding Assistants - Replit Agent +.replit +.replitignore + +# AI Coding Assistants - Supermaven +.supermaven/ + +# AI Coding Assistants - Sourcegraph Cody +.cody/ + +# AI Coding Assistants - General +.ai/ +.aiconfig +ai-config.json # Terraform .terraform* @@ -64,7 +135,6 @@ junit-reports/ ui/.env* api/.env* mcp_server/.env* -.env.local # Coverage .coverage* @@ -83,3 +153,6 @@ CLAUDE.md # MCP Server mcp_server/prowler_mcp_server/prowler_app/server.py mcp_server/prowler_mcp_server/prowler_app/utils/schema.yaml + +# Compliance report +*.pdf diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2dc954e9d9..f530feb99a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,4 +10,4 @@ Want some swag as appreciation for your contribution? # Prowler Developer Guide -https://docs.prowler.com/projects/prowler-open-source/en/latest/developer-guide/introduction/ +https://goto.prowler.com/devguide diff --git a/Makefile b/Makefile index 368bb885bd..861c9cf7fe 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,14 @@ help: ## Show this help. @echo "Prowler Makefile" @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +##@ Build no cache +build-no-cache-dev: + docker compose -f docker-compose-dev.yml build --no-cache api-dev worker-dev worker-beat + ##@ Development Environment run-api-dev: ## Start development environment with API, PostgreSQL, Valkey, and workers - docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat --build + docker compose -f docker-compose-dev.yml up api-dev postgres valkey worker-dev worker-beat + +##@ Development Environment +build-and-run-api-dev: build-no-cache-dev run-api-dev + diff --git a/README.md b/README.md index 4e4babea04..25cc8f6abe 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Prowler includes hundreds of built-in controls to ensure compliance with standar Prowler App is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments. -![Prowler App](docs/products/img/overview.png) +![Prowler App](docs/images/products/overview.png) >For more details, refer to the [Prowler App Documentation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-app-installation) @@ -73,25 +73,26 @@ prowler ```console prowler dashboard ``` -![Prowler Dashboard](docs/products/img/dashboard.png) +![Prowler Dashboard](docs/images/products/dashboard.png) # Prowler at a Glance > [!Tip] > For the most accurate and up-to-date information about checks, services, frameworks, and categories, visit [**Prowler Hub**](https://hub.prowler.com). -| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Stage | Interface | -|---|---|---|---|---|---|---|---| -| AWS | 576 | 82 | 36 | 10 | Official | Stable | UI, API, CLI | -| GCP | 79 | 13 | 10 | 3 | Official | Stable | UI, API, CLI | -| Azure | 162 | 19 | 11 | 4 | Official | Stable | UI, API, CLI | -| Kubernetes | 83 | 7 | 5 | 7 | Official | Stable | UI, API, CLI | +| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface | +|---|---|---|---|---|---|---| +| AWS | 576 | 82 | 39 | 10 | Official | UI, API, CLI | +| GCP | 79 | 13 | 13 | 3 | Official | UI, API, CLI | +| Azure | 162 | 19 | 13 | 4 | Official | UI, API, CLI | +| Kubernetes | 83 | 7 | 5 | 7 | Official | UI, API, CLI | | GitHub | 17 | 2 | 1 | 0 | Official | Stable | UI, API, CLI | -| M365 | 70 | 7 | 3 | 2 | Official | Stable | UI, API, CLI | -| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | Beta | CLI | -| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | Beta | CLI | -| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | Beta | CLI | -| NHN | 6 | 2 | 1 | 0 | Unofficial | Beta | CLI | +| M365 | 70 | 7 | 3 | 2 | Official | UI, API, CLI | +| OCI | 51 | 13 | 1 | 10 | Official | UI, API, CLI | +| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI | +| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | CLI, API | +| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI | +| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI | > [!Note] > The numbers in the table are updated periodically. diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index d6b692c110..58e18b5caa 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,13 +2,55 @@ All notable changes to the **Prowler API** are documented in this file. -## [1.14.0] (Prowler UNRELEASED) +## [1.15.0] (Prowler UNRELEASED) + +### Added +- IaC (Infrastructure as Code) provider support for remote repositories [(#8751)](https://github.com/prowler-cloud/prowler/pull/8751) +- Extend `GET /api/v1/providers` with provider-type filters and optional pagination disable to support the new Overview filters [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975) +- New endpoint to retrieve the number of providers grouped by provider type [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975) +- Support for configuring multiple LLM providers [(#8772)](https://github.com/prowler-cloud/prowler/pull/8772) +- Support C5 compliance framework for Azure provider [(#9081)](https://github.com/prowler-cloud/prowler/pull/9081) +- Support for Oracle Cloud Infrastructure (OCI) provider [(#8927)](https://github.com/prowler-cloud/prowler/pull/8927) +- Support muting findings based on simple rules with custom reason [(#9051)](https://github.com/prowler-cloud/prowler/pull/9051) +- Support C5 compliance framework for the GCP provider [(#9097)](https://github.com/prowler-cloud/prowler/pull/9097) +- Support for Amazon Bedrock and OpenAI compatible providers in Lighthouse AI [(#8957)](https://github.com/prowler-cloud/prowler/pull/8957) +- Tenant-wide ThreatScore overview aggregation and snapshot persistence with backfill support [(#9148)](https://github.com/prowler-cloud/prowler/pull/9148) +- Support for MongoDB Atlas provider [(#9167)](https://github.com/prowler-cloud/prowler/pull/9167) + +### Security +- Django updated to the latest 5.1 security release, 5.1.14, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/113) and [denial-of-service vulnerability](https://github.com/prowler-cloud/prowler/security/dependabot/114) [(#9176)](https://github.com/prowler-cloud/prowler/pull/9176) + +--- + +## [1.14.2] (Prowler UNRELEASED) + +### Fixed +- Update unique constraint for `Provider` model to exclude soft-deleted entries, resolving duplicate errors when re-deleting providers.[(#9054)](https://github.com/prowler-cloud/prowler/pull/9054) +- Remove compliance generation for providers without compliance frameworks [(#9208)](https://github.com/prowler-cloud/prowler/pull/9208) + +## [1.14.1] (Prowler 5.13.1) + +### Fixed +- `/api/v1/overviews/providers` collapses data by provider type so the UI receives a single aggregated record per cloud family even when multiple accounts exist [(#9053)](https://github.com/prowler-cloud/prowler/pull/9053) +- Added retry logic to database transactions to handle Aurora read replica connection failures during scale-down events [(#9064)](https://github.com/prowler-cloud/prowler/pull/9064) +- Security Hub integrations stop failing when they read relationships via the replica by allowing replica relations and saving updates through the primary [(#9080)](https://github.com/prowler-cloud/prowler/pull/9080) + +--- + +## [1.14.0] (Prowler 5.13.0) ### Added - Default JWT keys are generated and stored if they are missing from configuration [(#8655)](https://github.com/prowler-cloud/prowler/pull/8655) - `compliance_name` for each compliance [(#7920)](https://github.com/prowler-cloud/prowler/pull/7920) +- Support C5 compliance framework for the AWS provider [(#8830)](https://github.com/prowler-cloud/prowler/pull/8830) +- Support for M365 Certificate authentication [(#8538)](https://github.com/prowler-cloud/prowler/pull/8538) - API Key support [(#8805)](https://github.com/prowler-cloud/prowler/pull/8805) +- SAML role mapping protection for single-admin tenants to prevent accidental lockout [(#8882)](https://github.com/prowler-cloud/prowler/pull/8882) - Support for `passed_findings` and `total_findings` fields in compliance requirement overview for accurate Prowler ThreatScore calculation [(#8582)](https://github.com/prowler-cloud/prowler/pull/8582) +- PDF reporting for Prowler ThreatScore [(#8867)](https://github.com/prowler-cloud/prowler/pull/8867) +- Database read replica support [(#8869)](https://github.com/prowler-cloud/prowler/pull/8869) +- Support Common Cloud Controls for AWS, Azure and GCP [(#8000)](https://github.com/prowler-cloud/prowler/pull/8000) +- Add `provider_id__in` filter support to findings and findings severity overview endpoints [(#8951)](https://github.com/prowler-cloud/prowler/pull/8951) ### Changed - Now the MANAGE_ACCOUNT permission is required to modify or read user permissions instead of MANAGE_USERS [(#8281)](https://github.com/prowler-cloud/prowler/pull/8281) diff --git a/api/Dockerfile b/api/Dockerfile index a3cfb21782..2d7883a957 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,6 +5,9 @@ LABEL maintainer="https://github.com/prowler-cloud/api" ARG POWERSHELL_VERSION=7.5.0 ENV POWERSHELL_VERSION=${POWERSHELL_VERSION} +ARG TRIVY_VERSION=0.66.0 +ENV TRIVY_VERSION=${TRIVY_VERSION} + # hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ wget \ @@ -36,6 +39,24 @@ RUN ARCH=$(uname -m) && \ ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh && \ rm /tmp/powershell.tar.gz +# Install Trivy for IaC scanning +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + TRIVY_ARCH="Linux-64bit" ; \ + elif [ "$ARCH" = "aarch64" ]; then \ + TRIVY_ARCH="Linux-ARM64" ; \ + else \ + echo "Unsupported architecture for Trivy: $ARCH" && exit 1 ; \ + fi && \ + wget --progress=dot:giga "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_${TRIVY_ARCH}.tar.gz" -O /tmp/trivy.tar.gz && \ + tar zxf /tmp/trivy.tar.gz -C /tmp && \ + mv /tmp/trivy /usr/local/bin/trivy && \ + chmod +x /usr/local/bin/trivy && \ + rm /tmp/trivy.tar.gz && \ + # Create trivy cache directory with proper permissions + mkdir -p /tmp/.cache/trivy && \ + chmod 777 /tmp/.cache/trivy + # Add prowler user RUN addgroup --gid 1000 prowler && \ adduser --uid 1000 --gid 1000 --disabled-password --gecos "" prowler diff --git a/api/poetry.lock b/api/poetry.lock index c9bd9c6be3..332d4d9f73 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -273,14 +273,14 @@ tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" a [[package]] name = "authlib" -version = "1.6.4" +version = "1.6.5" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "authlib-1.6.4-py2.py3-none-any.whl", hash = "sha256:39313d2a2caac3ecf6d8f95fbebdfd30ae6ea6ae6a6db794d976405fdd9aa796"}, - {file = "authlib-1.6.4.tar.gz", hash = "sha256:104b0442a43061dc8bc23b133d1d06a2b0a9c2e3e33f34c4338929e816287649"}, + {file = "authlib-1.6.5-py2.py3-none-any.whl", hash = "sha256:3e0e0507807f842b02175507bdee8957a1d5707fd4afb17c32fb43fee90b6e3a"}, + {file = "authlib-1.6.5.tar.gz", hash = "sha256:6aaf9c79b7cc96c900f0b284061691c5d4e61221640a948fe690b556a6d6d10b"}, ] [package.dependencies] @@ -1256,6 +1256,98 @@ files = [ {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, ] +[[package]] +name = "contourpy" +version = "1.3.3" +description = "Python library for calculating contours of 2D quadrilateral grids" +optional = false +python-versions = ">=3.11" +groups = ["main"] +files = [ + {file = "contourpy-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1"}, + {file = "contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db"}, + {file = "contourpy-1.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620"}, + {file = "contourpy-1.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f"}, + {file = "contourpy-1.3.3-cp311-cp311-win32.whl", hash = "sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff"}, + {file = "contourpy-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42"}, + {file = "contourpy-1.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470"}, + {file = "contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb"}, + {file = "contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1"}, + {file = "contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7"}, + {file = "contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411"}, + {file = "contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69"}, + {file = "contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b"}, + {file = "contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc"}, + {file = "contourpy-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5"}, + {file = "contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9"}, + {file = "contourpy-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659"}, + {file = "contourpy-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7"}, + {file = "contourpy-1.3.3-cp313-cp313-win32.whl", hash = "sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d"}, + {file = "contourpy-1.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263"}, + {file = "contourpy-1.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9"}, + {file = "contourpy-1.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d"}, + {file = "contourpy-1.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b"}, + {file = "contourpy-1.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a"}, + {file = "contourpy-1.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e"}, + {file = "contourpy-1.3.3-cp313-cp313t-win32.whl", hash = "sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3"}, + {file = "contourpy-1.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8"}, + {file = "contourpy-1.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301"}, + {file = "contourpy-1.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a"}, + {file = "contourpy-1.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3"}, + {file = "contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b"}, + {file = "contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36"}, + {file = "contourpy-1.3.3-cp314-cp314-win32.whl", hash = "sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d"}, + {file = "contourpy-1.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd"}, + {file = "contourpy-1.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339"}, + {file = "contourpy-1.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772"}, + {file = "contourpy-1.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0"}, + {file = "contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4"}, + {file = "contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f"}, + {file = "contourpy-1.3.3-cp314-cp314t-win32.whl", hash = "sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae"}, + {file = "contourpy-1.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc"}, + {file = "contourpy-1.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77"}, + {file = "contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880"}, +] + +[package.dependencies] +numpy = ">=1.25" + +[package.extras] +bokeh = ["bokeh", "selenium"] +docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] +mypy = ["bokeh", "contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.17.0)", "types-Pillow"] +test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] +test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] + [[package]] name = "coverage" version = "7.5.4" @@ -1390,6 +1482,22 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["certifi (>=2024)", "cryptography-vectors (==44.0.1)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] +[[package]] +name = "cycler" +version = "0.12.1" +description = "Composable style cycles" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"}, + {file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"}, +] + +[package.extras] +docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] +tests = ["pytest", "pytest-cov", "pytest-xdist"] + [[package]] name = "dash" version = "3.1.1" @@ -1563,14 +1671,14 @@ with-social = ["django-allauth[socialaccount] (>=64.0.0)"] [[package]] name = "django" -version = "5.1.13" +version = "5.1.14" description = "A high-level Python web framework that encourages rapid development and clean, pragmatic design." optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "django-5.1.13-py3-none-any.whl", hash = "sha256:06f257f79dc4c17f3f9e23b106a4c5ed1335abecbe731e83c598c941d14fbeed"}, - {file = "django-5.1.13.tar.gz", hash = "sha256:543ff21679f15e80edfc01fe7ea35f8291b6d4ea589433882913626a7c1cf929"}, + {file = "django-5.1.14-py3-none-any.whl", hash = "sha256:2a4b9c20404fd1bf50aaaa5542a19d860594cba1354f688f642feb271b91df27"}, + {file = "django-5.1.14.tar.gz", hash = "sha256:b98409fb31fdd6e8c3a6ba2eef3415cc5c0020057b43b21ba7af6eff5f014831"}, ] [package.dependencies] @@ -2120,6 +2228,87 @@ werkzeug = ">=3.1.0" async = ["asgiref (>=3.2)"] dotenv = ["python-dotenv"] +[[package]] +name = "fonttools" +version = "4.60.1" +description = "Tools to manipulate font files" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "fonttools-4.60.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9a52f254ce051e196b8fe2af4634c2d2f02c981756c6464dc192f1b6050b4e28"}, + {file = "fonttools-4.60.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c7420a2696a44650120cdd269a5d2e56a477e2bfa9d95e86229059beb1c19e15"}, + {file = "fonttools-4.60.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee0c0b3b35b34f782afc673d503167157094a16f442ace7c6c5e0ca80b08f50c"}, + {file = "fonttools-4.60.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:282dafa55f9659e8999110bd8ed422ebe1c8aecd0dc396550b038e6c9a08b8ea"}, + {file = "fonttools-4.60.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4ba4bd646e86de16160f0fb72e31c3b9b7d0721c3e5b26b9fa2fc931dfdb2652"}, + {file = "fonttools-4.60.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0b0835ed15dd5b40d726bb61c846a688f5b4ce2208ec68779bc81860adb5851a"}, + {file = "fonttools-4.60.1-cp310-cp310-win32.whl", hash = "sha256:1525796c3ffe27bb6268ed2a1bb0dcf214d561dfaf04728abf01489eb5339dce"}, + {file = "fonttools-4.60.1-cp310-cp310-win_amd64.whl", hash = "sha256:268ecda8ca6cb5c4f044b1fb9b3b376e8cd1b361cef275082429dc4174907038"}, + {file = "fonttools-4.60.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7b4c32e232a71f63a5d00259ca3d88345ce2a43295bb049d21061f338124246f"}, + {file = "fonttools-4.60.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3630e86c484263eaac71d117085d509cbcf7b18f677906824e4bace598fb70d2"}, + {file = "fonttools-4.60.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5c1015318e4fec75dd4943ad5f6a206d9727adf97410d58b7e32ab644a807914"}, + {file = "fonttools-4.60.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e6c58beb17380f7c2ea181ea11e7db8c0ceb474c9dd45f48e71e2cb577d146a1"}, + {file = "fonttools-4.60.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ec3681a0cb34c255d76dd9d865a55f260164adb9fa02628415cdc2d43ee2c05d"}, + {file = "fonttools-4.60.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f4b5c37a5f40e4d733d3bbaaef082149bee5a5ea3156a785ff64d949bd1353fa"}, + {file = "fonttools-4.60.1-cp311-cp311-win32.whl", hash = "sha256:398447f3d8c0c786cbf1209711e79080a40761eb44b27cdafffb48f52bcec258"}, + {file = "fonttools-4.60.1-cp311-cp311-win_amd64.whl", hash = "sha256:d066ea419f719ed87bc2c99a4a4bfd77c2e5949cb724588b9dd58f3fd90b92bf"}, + {file = "fonttools-4.60.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7b0c6d57ab00dae9529f3faf187f2254ea0aa1e04215cf2f1a8ec277c96661bc"}, + {file = "fonttools-4.60.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:839565cbf14645952d933853e8ade66a463684ed6ed6c9345d0faf1f0e868877"}, + {file = "fonttools-4.60.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8177ec9676ea6e1793c8a084a90b65a9f778771998eb919d05db6d4b1c0b114c"}, + {file = "fonttools-4.60.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:996a4d1834524adbb423385d5a629b868ef9d774670856c63c9a0408a3063401"}, + {file = "fonttools-4.60.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a46b2f450bc79e06ef3b6394f0c68660529ed51692606ad7f953fc2e448bc903"}, + {file = "fonttools-4.60.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6ec722ee589e89a89f5b7574f5c45604030aa6ae24cb2c751e2707193b466fed"}, + {file = "fonttools-4.60.1-cp312-cp312-win32.whl", hash = "sha256:b2cf105cee600d2de04ca3cfa1f74f1127f8455b71dbad02b9da6ec266e116d6"}, + {file = "fonttools-4.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:992775c9fbe2cf794786fa0ffca7f09f564ba3499b8fe9f2f80bd7197db60383"}, + {file = "fonttools-4.60.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6f68576bb4bbf6060c7ab047b1574a1ebe5c50a17de62830079967b211059ebb"}, + {file = "fonttools-4.60.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:eedacb5c5d22b7097482fa834bda0dafa3d914a4e829ec83cdea2a01f8c813c4"}, + {file = "fonttools-4.60.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b33a7884fabd72bdf5f910d0cf46be50dce86a0362a65cfc746a4168c67eb96c"}, + {file = "fonttools-4.60.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2409d5fb7b55fd70f715e6d34e7a6e4f7511b8ad29a49d6df225ee76da76dd77"}, + {file = "fonttools-4.60.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c8651e0d4b3bdeda6602b85fdc2abbefc1b41e573ecb37b6779c4ca50753a199"}, + {file = "fonttools-4.60.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:145daa14bf24824b677b9357c5e44fd8895c2a8f53596e1b9ea3496081dc692c"}, + {file = "fonttools-4.60.1-cp313-cp313-win32.whl", hash = "sha256:2299df884c11162617a66b7c316957d74a18e3758c0274762d2cc87df7bc0272"}, + {file = "fonttools-4.60.1-cp313-cp313-win_amd64.whl", hash = "sha256:a3db56f153bd4c5c2b619ab02c5db5192e222150ce5a1bc10f16164714bc39ac"}, + {file = "fonttools-4.60.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a884aef09d45ba1206712c7dbda5829562d3fea7726935d3289d343232ecb0d3"}, + {file = "fonttools-4.60.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8a44788d9d91df72d1a5eac49b31aeb887a5f4aab761b4cffc4196c74907ea85"}, + {file = "fonttools-4.60.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e852d9dda9f93ad3651ae1e3bb770eac544ec93c3807888798eccddf84596537"}, + {file = "fonttools-4.60.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:154cb6ee417e417bf5f7c42fe25858c9140c26f647c7347c06f0cc2d47eff003"}, + {file = "fonttools-4.60.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5664fd1a9ea7f244487ac8f10340c4e37664675e8667d6fee420766e0fb3cf08"}, + {file = "fonttools-4.60.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:583b7f8e3c49486e4d489ad1deacfb8d5be54a8ef34d6df824f6a171f8511d99"}, + {file = "fonttools-4.60.1-cp314-cp314-win32.whl", hash = "sha256:66929e2ea2810c6533a5184f938502cfdaea4bc3efb7130d8cc02e1c1b4108d6"}, + {file = "fonttools-4.60.1-cp314-cp314-win_amd64.whl", hash = "sha256:f3d5be054c461d6a2268831f04091dc82753176f6ea06dc6047a5e168265a987"}, + {file = "fonttools-4.60.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:b6379e7546ba4ae4b18f8ae2b9bc5960936007a1c0e30b342f662577e8bc3299"}, + {file = "fonttools-4.60.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9d0ced62b59e0430b3690dbc5373df1c2aa7585e9a8ce38eff87f0fd993c5b01"}, + {file = "fonttools-4.60.1-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:875cb7764708b3132637f6c5fb385b16eeba0f7ac9fa45a69d35e09b47045801"}, + {file = "fonttools-4.60.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a184b2ea57b13680ab6d5fbde99ccef152c95c06746cb7718c583abd8f945ccc"}, + {file = "fonttools-4.60.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:026290e4ec76583881763fac284aca67365e0be9f13a7fb137257096114cb3bc"}, + {file = "fonttools-4.60.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f0e8817c7d1a0c2eedebf57ef9a9896f3ea23324769a9a2061a80fe8852705ed"}, + {file = "fonttools-4.60.1-cp314-cp314t-win32.whl", hash = "sha256:1410155d0e764a4615774e5c2c6fc516259fe3eca5882f034eb9bfdbee056259"}, + {file = "fonttools-4.60.1-cp314-cp314t-win_amd64.whl", hash = "sha256:022beaea4b73a70295b688f817ddc24ed3e3418b5036ffcd5658141184ef0d0c"}, + {file = "fonttools-4.60.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:122e1a8ada290423c493491d002f622b1992b1ab0b488c68e31c413390dc7eb2"}, + {file = "fonttools-4.60.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a140761c4ff63d0cb9256ac752f230460ee225ccef4ad8f68affc723c88e2036"}, + {file = "fonttools-4.60.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0eae96373e4b7c9e45d099d7a523444e3554360927225c1cdae221a58a45b856"}, + {file = "fonttools-4.60.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:596ecaca36367027d525b3b426d8a8208169d09edcf8c7506aceb3a38bfb55c7"}, + {file = "fonttools-4.60.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2ee06fc57512144d8b0445194c2da9f190f61ad51e230f14836286470c99f854"}, + {file = "fonttools-4.60.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b42d86938e8dda1cd9a1a87a6d82f1818eaf933348429653559a458d027446da"}, + {file = "fonttools-4.60.1-cp39-cp39-win32.whl", hash = "sha256:8b4eb332f9501cb1cd3d4d099374a1e1306783ff95489a1026bde9eb02ccc34a"}, + {file = "fonttools-4.60.1-cp39-cp39-win_amd64.whl", hash = "sha256:7473a8ed9ed09aeaa191301244a5a9dbe46fe0bf54f9d6cd21d83044c3321217"}, + {file = "fonttools-4.60.1-py3-none-any.whl", hash = "sha256:906306ac7afe2156fcf0042173d6ebbb05416af70f6b370967b47f8f00103bbb"}, + {file = "fonttools-4.60.1.tar.gz", hash = "sha256:ef00af0439ebfee806b25f24c8f92109157ff3fac5731dc7867957812e87b8d9"}, +] + +[package.extras] +all = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\"", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0) ; python_version <= \"3.12\"", "xattr ; sys_platform == \"darwin\"", "zopfli (>=0.1.4)"] +graphite = ["lz4 (>=1.7.4.2)"] +interpolatable = ["munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\""] +lxml = ["lxml (>=4.0)"] +pathops = ["skia-pathops (>=0.5.0)"] +plot = ["matplotlib"] +repacker = ["uharfbuzz (>=0.23.0)"] +symfont = ["sympy"] +type1 = ["xattr ; sys_platform == \"darwin\""] +unicode = ["unicodedata2 (>=15.1.0) ; python_version <= \"3.12\""] +woff = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "zopfli (>=0.1.4)"] + [[package]] name = "freezegun" version = "1.5.1" @@ -2787,6 +2976,117 @@ files = [ [package.dependencies] referencing = ">=0.31.0" +[[package]] +name = "kiwisolver" +version = "1.4.9" +description = "A fast implementation of the Cassowary constraint solver" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "kiwisolver-1.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b4b4d74bda2b8ebf4da5bd42af11d02d04428b2c32846e4c2c93219df8a7987b"}, + {file = "kiwisolver-1.4.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fb3b8132019ea572f4611d770991000d7f58127560c4889729248eb5852a102f"}, + {file = "kiwisolver-1.4.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:84fd60810829c27ae375114cd379da1fa65e6918e1da405f356a775d49a62bcf"}, + {file = "kiwisolver-1.4.9-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b78efa4c6e804ecdf727e580dbb9cba85624d2e1c6b5cb059c66290063bd99a9"}, + {file = "kiwisolver-1.4.9-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4efec7bcf21671db6a3294ff301d2fc861c31faa3c8740d1a94689234d1b415"}, + {file = "kiwisolver-1.4.9-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90f47e70293fc3688b71271100a1a5453aa9944a81d27ff779c108372cf5567b"}, + {file = "kiwisolver-1.4.9-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fdca1def57a2e88ef339de1737a1449d6dbf5fab184c54a1fca01d541317154"}, + {file = "kiwisolver-1.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:9cf554f21be770f5111a1690d42313e140355e687e05cf82cb23d0a721a64a48"}, + {file = "kiwisolver-1.4.9-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fc1795ac5cd0510207482c3d1d3ed781143383b8cfd36f5c645f3897ce066220"}, + {file = "kiwisolver-1.4.9-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:ccd09f20ccdbbd341b21a67ab50a119b64a403b09288c27481575105283c1586"}, + {file = "kiwisolver-1.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:540c7c72324d864406a009d72f5d6856f49693db95d1fbb46cf86febef873634"}, + {file = "kiwisolver-1.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:ede8c6d533bc6601a47ad4046080d36b8fc99f81e6f1c17b0ac3c2dc91ac7611"}, + {file = "kiwisolver-1.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:7b4da0d01ac866a57dd61ac258c5607b4cd677f63abaec7b148354d2b2cdd536"}, + {file = "kiwisolver-1.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eb14a5da6dc7642b0f3a18f13654847cd8b7a2550e2645a5bda677862b03ba16"}, + {file = "kiwisolver-1.4.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39a219e1c81ae3b103643d2aedb90f1ef22650deb266ff12a19e7773f3e5f089"}, + {file = "kiwisolver-1.4.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2405a7d98604b87f3fc28b1716783534b1b4b8510d8142adca34ee0bc3c87543"}, + {file = "kiwisolver-1.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dc1ae486f9abcef254b5618dfb4113dd49f94c68e3e027d03cf0143f3f772b61"}, + {file = "kiwisolver-1.4.9-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a1f570ce4d62d718dce3f179ee78dac3b545ac16c0c04bb363b7607a949c0d1"}, + {file = "kiwisolver-1.4.9-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb27e7b78d716c591e88e0a09a2139c6577865d7f2e152488c2cc6257f460872"}, + {file = "kiwisolver-1.4.9-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:15163165efc2f627eb9687ea5f3a28137217d217ac4024893d753f46bce9de26"}, + {file = "kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bdee92c56a71d2b24c33a7d4c2856bd6419d017e08caa7802d2963870e315028"}, + {file = "kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:412f287c55a6f54b0650bd9b6dce5aceddb95864a1a90c87af16979d37c89771"}, + {file = "kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2c93f00dcba2eea70af2be5f11a830a742fe6b579a1d4e00f47760ef13be247a"}, + {file = "kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f117e1a089d9411663a3207ba874f31be9ac8eaa5b533787024dc07aeb74f464"}, + {file = "kiwisolver-1.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:be6a04e6c79819c9a8c2373317d19a96048e5a3f90bec587787e86a1153883c2"}, + {file = "kiwisolver-1.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:0ae37737256ba2de764ddc12aed4956460277f00c4996d51a197e72f62f5eec7"}, + {file = "kiwisolver-1.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ac5a486ac389dddcc5bef4f365b6ae3ffff2c433324fb38dd35e3fab7c957999"}, + {file = "kiwisolver-1.4.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2ba92255faa7309d06fe44c3a4a97efe1c8d640c2a79a5ef728b685762a6fd2"}, + {file = "kiwisolver-1.4.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a2899935e724dd1074cb568ce7ac0dce28b2cd6ab539c8e001a8578eb106d14"}, + {file = "kiwisolver-1.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f6008a4919fdbc0b0097089f67a1eb55d950ed7e90ce2cc3e640abadd2757a04"}, + {file = "kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67bb8b474b4181770f926f7b7d2f8c0248cbcb78b660fdd41a47054b28d2a752"}, + {file = "kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2327a4a30d3ee07d2fbe2e7933e8a37c591663b96ce42a00bc67461a87d7df77"}, + {file = "kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a08b491ec91b1d5053ac177afe5290adacf1f0f6307d771ccac5de30592d198"}, + {file = "kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d8fc5c867c22b828001b6a38d2eaeb88160bf5783c6cb4a5e440efc981ce286d"}, + {file = "kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3b3115b2581ea35bb6d1f24a4c90af37e5d9b49dcff267eeed14c3893c5b86ab"}, + {file = "kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858e4c22fb075920b96a291928cb7dea5644e94c0ee4fcd5af7e865655e4ccf2"}, + {file = "kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ed0fecd28cc62c54b262e3736f8bb2512d8dcfdc2bcf08be5f47f96bf405b145"}, + {file = "kiwisolver-1.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:f68208a520c3d86ea51acf688a3e3002615a7f0238002cccc17affecc86a8a54"}, + {file = "kiwisolver-1.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:2c1a4f57df73965f3f14df20b80ee29e6a7930a57d2d9e8491a25f676e197c60"}, + {file = "kiwisolver-1.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5d0432ccf1c7ab14f9949eec60c5d1f924f17c037e9f8b33352fa05799359b8"}, + {file = "kiwisolver-1.4.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efb3a45b35622bb6c16dbfab491a8f5a391fe0e9d45ef32f4df85658232ca0e2"}, + {file = "kiwisolver-1.4.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a12cf6398e8a0a001a059747a1cbf24705e18fe413bc22de7b3d15c67cffe3f"}, + {file = "kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b67e6efbf68e077dd71d1a6b37e43e1a99d0bff1a3d51867d45ee8908b931098"}, + {file = "kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5656aa670507437af0207645273ccdfee4f14bacd7f7c67a4306d0dcaeaf6eed"}, + {file = "kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bfc08add558155345129c7803b3671cf195e6a56e7a12f3dde7c57d9b417f525"}, + {file = "kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:40092754720b174e6ccf9e845d0d8c7d8e12c3d71e7fc35f55f3813e96376f78"}, + {file = "kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:497d05f29a1300d14e02e6441cf0f5ee81c1ff5a304b0d9fb77423974684e08b"}, + {file = "kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bdd1a81a1860476eb41ac4bc1e07b3f07259e6d55bbf739b79c8aaedcf512799"}, + {file = "kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:e6b93f13371d341afee3be9f7c5964e3fe61d5fa30f6a30eb49856935dfe4fc3"}, + {file = "kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d75aa530ccfaa593da12834b86a0724f58bff12706659baa9227c2ccaa06264c"}, + {file = "kiwisolver-1.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:dd0a578400839256df88c16abddf9ba14813ec5f21362e1fe65022e00c883d4d"}, + {file = "kiwisolver-1.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:d4188e73af84ca82468f09cadc5ac4db578109e52acb4518d8154698d3a87ca2"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:5a0f2724dfd4e3b3ac5a82436a8e6fd16baa7d507117e4279b660fe8ca38a3a1"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1b11d6a633e4ed84fc0ddafd4ebfd8ea49b3f25082c04ad12b8315c11d504dc1"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61874cdb0a36016354853593cffc38e56fc9ca5aa97d2c05d3dcf6922cd55a11"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:60c439763a969a6af93b4881db0eed8fadf93ee98e18cbc35bc8da868d0c4f0c"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92a2f997387a1b79a75e7803aa7ded2cfbe2823852ccf1ba3bcf613b62ae3197"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a31d512c812daea6d8b3be3b2bfcbeb091dbb09177706569bcfc6240dcf8b41c"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:52a15b0f35dad39862d376df10c5230155243a2c1a436e39eb55623ccbd68185"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a30fd6fdef1430fd9e1ba7b3398b5ee4e2887783917a687d86ba69985fb08748"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cc9617b46837c6468197b5945e196ee9ca43057bb7d9d1ae688101e4e1dddf64"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:0ab74e19f6a2b027ea4f845a78827969af45ce790e6cb3e1ebab71bdf9f215ff"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dba5ee5d3981160c28d5490f0d1b7ed730c22470ff7f6cc26cfcfaacb9896a07"}, + {file = "kiwisolver-1.4.9-cp313-cp313t-win_arm64.whl", hash = "sha256:0749fd8f4218ad2e851e11cc4dc05c7cbc0cbc4267bdfdb31782e65aace4ee9c"}, + {file = "kiwisolver-1.4.9-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:9928fe1eb816d11ae170885a74d074f57af3a0d65777ca47e9aeb854a1fba386"}, + {file = "kiwisolver-1.4.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d0005b053977e7b43388ddec89fa567f43d4f6d5c2c0affe57de5ebf290dc552"}, + {file = "kiwisolver-1.4.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2635d352d67458b66fd0667c14cb1d4145e9560d503219034a18a87e971ce4f3"}, + {file = "kiwisolver-1.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:767c23ad1c58c9e827b649a9ab7809fd5fd9db266a9cf02b0e926ddc2c680d58"}, + {file = "kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72d0eb9fba308b8311685c2268cf7d0a0639a6cd027d8128659f72bdd8a024b4"}, + {file = "kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f68e4f3eeca8fb22cc3d731f9715a13b652795ef657a13df1ad0c7dc0e9731df"}, + {file = "kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d84cd4061ae292d8ac367b2c3fa3aad11cb8625a95d135fe93f286f914f3f5a6"}, + {file = "kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a60ea74330b91bd22a29638940d115df9dc00af5035a9a2a6ad9399ffb4ceca5"}, + {file = "kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ce6a3a4e106cf35c2d9c4fa17c05ce0b180db622736845d4315519397a77beaf"}, + {file = "kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:77937e5e2a38a7b48eef0585114fe7930346993a88060d0bf886086d2aa49ef5"}, + {file = "kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:24c175051354f4a28c5d6a31c93906dc653e2bf234e8a4bbfb964892078898ce"}, + {file = "kiwisolver-1.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:0763515d4df10edf6d06a3c19734e2566368980d21ebec439f33f9eb936c07b7"}, + {file = "kiwisolver-1.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:0e4e2bf29574a6a7b7f6cb5fa69293b9f96c928949ac4a53ba3f525dffb87f9c"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d976bbb382b202f71c67f77b0ac11244021cfa3f7dfd9e562eefcea2df711548"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2489e4e5d7ef9a1c300a5e0196e43d9c739f066ef23270607d45aba368b91f2d"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e2ea9f7ab7fbf18fffb1b5434ce7c69a07582f7acc7717720f1d69f3e806f90c"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b34e51affded8faee0dfdb705416153819d8ea9250bbbf7ea1b249bdeb5f1122"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8aacd3d4b33b772542b2e01beb50187536967b514b00003bdda7589722d2a64"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7cf974dd4e35fa315563ac99d6287a1024e4dc2077b8a7d7cd3d2fb65d283134"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:85bd218b5ecfbee8c8a82e121802dcb519a86044c9c3b2e4aef02fa05c6da370"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0856e241c2d3df4efef7c04a1e46b1936b6120c9bcf36dd216e3acd84bc4fb21"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9af39d6551f97d31a4deebeac6f45b156f9755ddc59c07b402c148f5dbb6482a"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:bb4ae2b57fc1d8cbd1cf7b1d9913803681ffa903e7488012be5b76dedf49297f"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:aedff62918805fb62d43a4aa2ecd4482c380dc76cd31bd7c8878588a61bd0369"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:1fa333e8b2ce4d9660f2cda9c0e1b6bafcfb2457a9d259faa82289e73ec24891"}, + {file = "kiwisolver-1.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:4a48a2ce79d65d363597ef7b567ce3d14d68783d2b2263d98db3d9477805ba32"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4d1d9e582ad4d63062d34077a9a1e9f3c34088a2ec5135b1f7190c07cf366527"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:deed0c7258ceb4c44ad5ec7d9918f9f14fd05b2be86378d86cf50e63d1e7b771"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a590506f303f512dff6b7f75fd2fd18e16943efee932008fe7140e5fa91d80e"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e09c2279a4d01f099f52d5c4b3d9e208e91edcbd1a175c9662a8b16e000fece9"}, + {file = "kiwisolver-1.4.9-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c9e7cdf45d594ee04d5be1b24dd9d49f3d1590959b2271fb30b5ca2b262c00fb"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:720e05574713db64c356e86732c0f3c5252818d05f9df320f0ad8380641acea5"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:17680d737d5335b552994a2008fab4c851bcd7de33094a82067ef3a576ff02fa"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:85b5352f94e490c028926ea567fc569c52ec79ce131dadb968d3853e809518c2"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:464415881e4801295659462c49461a24fb107c140de781d55518c4b80cb6790f"}, + {file = "kiwisolver-1.4.9-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:fb940820c63a9590d31d88b815e7a3aa5915cad3ce735ab45f0c730b39547de1"}, + {file = "kiwisolver-1.4.9.tar.gz", hash = "sha256:c3b22c26c6fd6811b0ae8363b95ca8ce4ea3c202d3d0975b2914310ceb1bcc4d"}, +] + [[package]] name = "kombu" version = "5.5.4" @@ -3137,6 +3437,85 @@ dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"] docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"] tests = ["pytest", "simplejson"] +[[package]] +name = "matplotlib" +version = "3.10.6" +description = "Python plotting package" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "matplotlib-3.10.6-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:bc7316c306d97463a9866b89d5cc217824e799fa0de346c8f68f4f3d27c8693d"}, + {file = "matplotlib-3.10.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d00932b0d160ef03f59f9c0e16d1e3ac89646f7785165ce6ad40c842db16cc2e"}, + {file = "matplotlib-3.10.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8fa4c43d6bfdbfec09c733bca8667de11bfa4970e8324c471f3a3632a0301c15"}, + {file = "matplotlib-3.10.6-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ea117a9c1627acaa04dbf36265691921b999cbf515a015298e54e1a12c3af837"}, + {file = "matplotlib-3.10.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:08fc803293b4e1694ee325896030de97f74c141ccff0be886bb5915269247676"}, + {file = "matplotlib-3.10.6-cp310-cp310-win_amd64.whl", hash = "sha256:2adf92d9b7527fbfb8818e050260f0ebaa460f79d61546374ce73506c9421d09"}, + {file = "matplotlib-3.10.6-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:905b60d1cb0ee604ce65b297b61cf8be9f4e6cfecf95a3fe1c388b5266bc8f4f"}, + {file = "matplotlib-3.10.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7bac38d816637343e53d7185d0c66677ff30ffb131044a81898b5792c956ba76"}, + {file = "matplotlib-3.10.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:942a8de2b5bfff1de31d95722f702e2966b8a7e31f4e68f7cd963c7cd8861cf6"}, + {file = "matplotlib-3.10.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3276c85370bc0dfca051ec65c5817d1e0f8f5ce1b7787528ec8ed2d524bbc2f"}, + {file = "matplotlib-3.10.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9df5851b219225731f564e4b9e7f2ac1e13c9e6481f941b5631a0f8e2d9387ce"}, + {file = "matplotlib-3.10.6-cp311-cp311-win_amd64.whl", hash = "sha256:abb5d9478625dd9c9eb51a06d39aae71eda749ae9b3138afb23eb38824026c7e"}, + {file = "matplotlib-3.10.6-cp311-cp311-win_arm64.whl", hash = "sha256:886f989ccfae63659183173bb3fced7fd65e9eb793c3cc21c273add368536951"}, + {file = "matplotlib-3.10.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:31ca662df6a80bd426f871105fdd69db7543e28e73a9f2afe80de7e531eb2347"}, + {file = "matplotlib-3.10.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1678bb61d897bb4ac4757b5ecfb02bfb3fddf7f808000fb81e09c510712fda75"}, + {file = "matplotlib-3.10.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:56cd2d20842f58c03d2d6e6c1f1cf5548ad6f66b91e1e48f814e4fb5abd1cb95"}, + {file = "matplotlib-3.10.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:662df55604a2f9a45435566d6e2660e41efe83cd94f4288dfbf1e6d1eae4b0bb"}, + {file = "matplotlib-3.10.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:08f141d55148cd1fc870c3387d70ca4df16dee10e909b3b038782bd4bda6ea07"}, + {file = "matplotlib-3.10.6-cp312-cp312-win_amd64.whl", hash = "sha256:590f5925c2d650b5c9d813c5b3b5fc53f2929c3f8ef463e4ecfa7e052044fb2b"}, + {file = "matplotlib-3.10.6-cp312-cp312-win_arm64.whl", hash = "sha256:f44c8d264a71609c79a78d50349e724f5d5fc3684ead7c2a473665ee63d868aa"}, + {file = "matplotlib-3.10.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:819e409653c1106c8deaf62e6de6b8611449c2cd9939acb0d7d4e57a3d95cc7a"}, + {file = "matplotlib-3.10.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:59c8ac8382fefb9cb71308dde16a7c487432f5255d8f1fd32473523abecfecdf"}, + {file = "matplotlib-3.10.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:84e82d9e0fd70c70bc55739defbd8055c54300750cbacf4740c9673a24d6933a"}, + {file = "matplotlib-3.10.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25f7a3eb42d6c1c56e89eacd495661fc815ffc08d9da750bca766771c0fd9110"}, + {file = "matplotlib-3.10.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f9c862d91ec0b7842920a4cfdaaec29662195301914ea54c33e01f1a28d014b2"}, + {file = "matplotlib-3.10.6-cp313-cp313-win_amd64.whl", hash = "sha256:1b53bd6337eba483e2e7d29c5ab10eee644bc3a2491ec67cc55f7b44583ffb18"}, + {file = "matplotlib-3.10.6-cp313-cp313-win_arm64.whl", hash = "sha256:cbd5eb50b7058b2892ce45c2f4e92557f395c9991f5c886d1bb74a1582e70fd6"}, + {file = "matplotlib-3.10.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:acc86dd6e0e695c095001a7fccff158c49e45e0758fdf5dcdbb0103318b59c9f"}, + {file = "matplotlib-3.10.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e228cd2ffb8f88b7d0b29e37f68ca9aaf83e33821f24a5ccc4f082dd8396bc27"}, + {file = "matplotlib-3.10.6-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:658bc91894adeab669cf4bb4a186d049948262987e80f0857216387d7435d833"}, + {file = "matplotlib-3.10.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8913b7474f6dd83ac444c9459c91f7f0f2859e839f41d642691b104e0af056aa"}, + {file = "matplotlib-3.10.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:091cea22e059b89f6d7d1a18e2c33a7376c26eee60e401d92a4d6726c4e12706"}, + {file = "matplotlib-3.10.6-cp313-cp313t-win_amd64.whl", hash = "sha256:491e25e02a23d7207629d942c666924a6b61e007a48177fdd231a0097b7f507e"}, + {file = "matplotlib-3.10.6-cp313-cp313t-win_arm64.whl", hash = "sha256:3d80d60d4e54cda462e2cd9a086d85cd9f20943ead92f575ce86885a43a565d5"}, + {file = "matplotlib-3.10.6-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:70aaf890ce1d0efd482df969b28a5b30ea0b891224bb315810a3940f67182899"}, + {file = "matplotlib-3.10.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1565aae810ab79cb72e402b22facfa6501365e73ebab70a0fdfb98488d2c3c0c"}, + {file = "matplotlib-3.10.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3b23315a01981689aa4e1a179dbf6ef9fbd17143c3eea77548c2ecfb0499438"}, + {file = "matplotlib-3.10.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30fdd37edf41a4e6785f9b37969de57aea770696cb637d9946eb37470c94a453"}, + {file = "matplotlib-3.10.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bc31e693da1c08012c764b053e702c1855378e04102238e6a5ee6a7117c53a47"}, + {file = "matplotlib-3.10.6-cp314-cp314-win_amd64.whl", hash = "sha256:05be9bdaa8b242bc6ff96330d18c52f1fc59c6fb3a4dd411d953d67e7e1baf98"}, + {file = "matplotlib-3.10.6-cp314-cp314-win_arm64.whl", hash = "sha256:f56a0d1ab05d34c628592435781d185cd99630bdfd76822cd686fb5a0aecd43a"}, + {file = "matplotlib-3.10.6-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:94f0b4cacb23763b64b5dace50d5b7bfe98710fed5f0cef5c08135a03399d98b"}, + {file = "matplotlib-3.10.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cc332891306b9fb39462673d8225d1b824c89783fee82840a709f96714f17a5c"}, + {file = "matplotlib-3.10.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee1d607b3fb1590deb04b69f02ea1d53ed0b0bf75b2b1a5745f269afcbd3cdd3"}, + {file = "matplotlib-3.10.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:376a624a218116461696b27b2bbf7a8945053e6d799f6502fc03226d077807bf"}, + {file = "matplotlib-3.10.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:83847b47f6524c34b4f2d3ce726bb0541c48c8e7692729865c3df75bfa0f495a"}, + {file = "matplotlib-3.10.6-cp314-cp314t-win_amd64.whl", hash = "sha256:c7e0518e0d223683532a07f4b512e2e0729b62674f1b3a1a69869f98e6b1c7e3"}, + {file = "matplotlib-3.10.6-cp314-cp314t-win_arm64.whl", hash = "sha256:4dd83e029f5b4801eeb87c64efd80e732452781c16a9cf7415b7b63ec8f374d7"}, + {file = "matplotlib-3.10.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:13fcd07ccf17e354398358e0307a1f53f5325dca22982556ddb9c52837b5af41"}, + {file = "matplotlib-3.10.6-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:470fc846d59d1406e34fa4c32ba371039cd12c2fe86801159a965956f2575bd1"}, + {file = "matplotlib-3.10.6-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7173f8551b88f4ef810a94adae3128c2530e0d07529f7141be7f8d8c365f051"}, + {file = "matplotlib-3.10.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f2d684c3204fa62421bbf770ddfebc6b50130f9cad65531eeba19236d73bb488"}, + {file = "matplotlib-3.10.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:6f4a69196e663a41d12a728fab8751177215357906436804217d6d9cf0d4d6cf"}, + {file = "matplotlib-3.10.6-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d6ca6ef03dfd269f4ead566ec6f3fb9becf8dab146fb999022ed85ee9f6b3eb"}, + {file = "matplotlib-3.10.6.tar.gz", hash = "sha256:ec01b645840dd1996df21ee37f208cd8ba57644779fa20464010638013d3203c"}, +] + +[package.dependencies] +contourpy = ">=1.0.1" +cycler = ">=0.10" +fonttools = ">=4.22.0" +kiwisolver = ">=1.3.1" +numpy = ">=1.23" +packaging = ">=20.0" +pillow = ">=8" +pyparsing = ">=2.3.1" +python-dateutil = ">=2.7" + +[package.extras] +dev = ["meson-python (>=0.13.1,<0.17.0)", "pybind11 (>=2.13.2,!=2.13.3)", "setuptools (>=64)", "setuptools_scm (>=7)"] + [[package]] name = "mccabe" version = "0.7.0" @@ -3857,6 +4236,131 @@ files = [ [package.dependencies] setuptools = "*" +[[package]] +name = "pillow" +version = "11.3.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, + {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae"}, + {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9"}, + {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:040a5b691b0713e1f6cbe222e0f4f74cd233421e105850ae3b3c0ceda520f42e"}, + {file = "pillow-11.3.0-cp310-cp310-win32.whl", hash = "sha256:89bd777bc6624fe4115e9fac3352c79ed60f3bb18651420635f26e643e3dd1f6"}, + {file = "pillow-11.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:19d2ff547c75b8e3ff46f4d9ef969a06c30ab2d4263a9e287733aa8b2429ce8f"}, + {file = "pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f"}, + {file = "pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722"}, + {file = "pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f"}, + {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e"}, + {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:932c754c2d51ad2b2271fd01c3d121daaa35e27efae2a616f77bf164bc0b3e94"}, + {file = "pillow-11.3.0-cp311-cp311-win32.whl", hash = "sha256:b4b8f3efc8d530a1544e5962bd6b403d5f7fe8b9e08227c6b255f98ad82b4ba0"}, + {file = "pillow-11.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:1a992e86b0dd7aeb1f053cd506508c0999d710a8f07b4c791c63843fc6a807ac"}, + {file = "pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd"}, + {file = "pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4"}, + {file = "pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024"}, + {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809"}, + {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6be31e3fc9a621e071bc17bb7de63b85cbe0bfae91bb0363c893cbe67247780d"}, + {file = "pillow-11.3.0-cp312-cp312-win32.whl", hash = "sha256:7b161756381f0918e05e7cb8a371fff367e807770f8fe92ecb20d905d0e1c149"}, + {file = "pillow-11.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6444696fce635783440b7f7a9fc24b3ad10a9ea3f0ab66c5905be1c19ccf17d"}, + {file = "pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1c627742b539bba4309df89171356fcb3cc5a9178355b2727d1b74a6cf155fbd"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:30b7c02f3899d10f13d7a48163c8969e4e653f8b43416d23d13d1bbfdc93b9f8"}, + {file = "pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f"}, + {file = "pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c"}, + {file = "pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8"}, + {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2"}, + {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:45dfc51ac5975b938e9809451c51734124e73b04d0f0ac621649821a63852e7b"}, + {file = "pillow-11.3.0-cp313-cp313-win32.whl", hash = "sha256:a4d336baed65d50d37b88ca5b60c0fa9d81e3a87d4a7930d3880d1624d5b31f3"}, + {file = "pillow-11.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bce5c4fd0921f99d2e858dc4d4d64193407e1b99478bc5cacecba2311abde51"}, + {file = "pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580"}, + {file = "pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e"}, + {file = "pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59"}, + {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe"}, + {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83e1b0161c9d148125083a35c1c5a89db5b7054834fd4387499e06552035236c"}, + {file = "pillow-11.3.0-cp313-cp313t-win32.whl", hash = "sha256:2a3117c06b8fb646639dce83694f2f9eac405472713fcb1ae887469c0d4f6788"}, + {file = "pillow-11.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:857844335c95bea93fb39e0fa2726b4d9d758850b34075a7e3ff4f4fa3aa3b31"}, + {file = "pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e"}, + {file = "pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12"}, + {file = "pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77"}, + {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874"}, + {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:98a9afa7b9007c67ed84c57c9e0ad86a6000da96eaa638e4f8abe5b65ff83f0a"}, + {file = "pillow-11.3.0-cp314-cp314-win32.whl", hash = "sha256:02a723e6bf909e7cea0dac1b0e0310be9d7650cd66222a5f1c571455c0a45214"}, + {file = "pillow-11.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a418486160228f64dd9e9efcd132679b7a02a5f22c982c78b6fc7dab3fefb635"}, + {file = "pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6"}, + {file = "pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae"}, + {file = "pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477"}, + {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50"}, + {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a1bc6ba083b145187f648b667e05a2534ecc4b9f2784c2cbe3089e44868f2b9b"}, + {file = "pillow-11.3.0-cp314-cp314t-win32.whl", hash = "sha256:118ca10c0d60b06d006be10a501fd6bbdfef559251ed31b794668ed569c87e12"}, + {file = "pillow-11.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8924748b688aa210d79883357d102cd64690e56b923a186f35a82cbc10f997db"}, + {file = "pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa"}, + {file = "pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f"}, + {file = "pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:23cff760a9049c502721bdb743a7cb3e03365fafcdfc2ef9784610714166e5a4"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6359a3bc43f57d5b375d1ad54a0074318a0844d11b76abccf478c37c986d3cfc"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a"}, + {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978"}, + {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:97afb3a00b65cc0804d1c7abddbf090a81eaac02768af58cbdcaaa0a931e0b6d"}, + {file = "pillow-11.3.0-cp39-cp39-win32.whl", hash = "sha256:ea944117a7974ae78059fcc1800e5d3295172bb97035c0c1d9345fca1419da71"}, + {file = "pillow-11.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:e5c5858ad8ec655450a7c7df532e9842cf8df7cc349df7225c60d5d348c8aada"}, + {file = "pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8"}, + {file = "pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +test-arrow = ["pyarrow"] +tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "trove-classifiers (>=2024.10.12)"] +typing = ["typing-extensions ; python_version < \"3.10\""] +xmp = ["defusedxml"] + [[package]] name = "platformdirs" version = "4.3.8" @@ -5016,6 +5520,29 @@ attrs = ">=22.2.0" rpds-py = ">=0.7.0" typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} +[[package]] +name = "reportlab" +version = "4.4.4" +description = "The Reportlab Toolkit" +optional = false +python-versions = "<4,>=3.9" +groups = ["main"] +files = [ + {file = "reportlab-4.4.4-py3-none-any.whl", hash = "sha256:299b3b0534e7202bb94ed2ddcd7179b818dcda7de9d8518a57c85a58a1ebaadb"}, + {file = "reportlab-4.4.4.tar.gz", hash = "sha256:cb2f658b7f4a15be2cc68f7203aa67faef67213edd4f2d4bdd3eb20dab75a80d"}, +] + +[package.dependencies] +charset-normalizer = "*" +pillow = ">=9.0.0" + +[package.extras] +accel = ["rl_accel (>=0.9.0,<1.1)"] +bidi = ["rlbidi"] +pycairo = ["freetype-py (>=2.3.0,<2.4)", "rlPyCairo (>=0.2.0,<1)"] +renderpm = ["rl_renderPM (>=4.0.3,<4.1)"] +shaping = ["uharfbuzz"] + [[package]] name = "requests" version = "2.32.5" @@ -6259,4 +6786,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "03442fd4673006c5a74374f90f53621fd1c9d117279fe6cc0355ef833eb7f9bb" +content-hash = "943e2cd6b87229704550d4e140b36509fb9f58896ebb5834b9fbabe28a9ee92f" diff --git a/api/pyproject.toml b/api/pyproject.toml index d7ef074e36..ffc8413489 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -7,7 +7,7 @@ authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}] dependencies = [ "celery[pytest] (>=5.4.0,<6.0.0)", "dj-rest-auth[with_social,jwt] (==7.0.1)", - "django (==5.1.13)", + "django (==5.1.14)", "django-allauth[saml] (>=65.8.0,<66.0.0)", "django-celery-beat (>=2.7.0,<3.0.0)", "django-celery-results (>=2.5.1,<3.0.0)", @@ -33,7 +33,9 @@ dependencies = [ "xmlsec==1.3.14", "h2 (==4.3.0)", "markdown (>=3.9,<4.0)", - "drf-simple-apikey (==2.2.1)" + "drf-simple-apikey (==2.2.1)", + "matplotlib (>=3.10.6,<4.0.0)", + "reportlab (>=4.4.4,<5.0.0)" ] description = "Prowler's API (Django/DRF)" license = "Apache-2.0" @@ -41,7 +43,7 @@ name = "prowler-api" package-mode = false # Needed for the SDK compatibility requires-python = ">=3.11,<3.13" -version = "1.14.0" +version = "1.15.0" [project.scripts] celery = "src.backend.config.settings.celery" diff --git a/api/src/backend/api/admin.py b/api/src/backend/api/admin.py deleted file mode 100644 index 4185d360e9..0000000000 --- a/api/src/backend/api/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -# from django.contrib import admin - -# Register your models here. diff --git a/api/src/backend/api/base_views.py b/api/src/backend/api/base_views.py index 36455c56b7..7cdf8fdd7f 100644 --- a/api/src/backend/api/base_views.py +++ b/api/src/backend/api/base_views.py @@ -1,13 +1,15 @@ +from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from rest_framework import permissions from rest_framework.exceptions import NotAuthenticated from rest_framework.filters import SearchFilter +from rest_framework.permissions import SAFE_METHODS from rest_framework_json_api import filters from rest_framework_json_api.views import ModelViewSet from api.authentication import CombinedJWTOrAPIKeyAuthentication -from api.db_router import MainRouter +from api.db_router import MainRouter, reset_read_db_alias, set_read_db_alias from api.db_utils import POSTGRES_USER_VAR, rls_transaction from api.filters import CustomDjangoFilterBackend from api.models import Role, Tenant @@ -31,6 +33,20 @@ class BaseViewSet(ModelViewSet): ordering_fields = "__all__" ordering = ["id"] + def _get_request_db_alias(self, request): + if request is None: + return MainRouter.default_db + + read_alias = ( + MainRouter.replica_db + if request.method in SAFE_METHODS + and MainRouter.replica_db in settings.DATABASES + else None + ) + if read_alias: + return read_alias + return MainRouter.default_db + def initial(self, request, *args, **kwargs): """ Sets required_permissions before permissions are checked. @@ -48,8 +64,21 @@ class BaseViewSet(ModelViewSet): class BaseRLSViewSet(BaseViewSet): def dispatch(self, request, *args, **kwargs): - with transaction.atomic(): - return super().dispatch(request, *args, **kwargs) + self.db_alias = self._get_request_db_alias(request) + alias_token = None + try: + if self.db_alias != MainRouter.default_db: + alias_token = set_read_db_alias(self.db_alias) + + if request is not None: + request.db_alias = self.db_alias + + with transaction.atomic(using=self.db_alias): + return super().dispatch(request, *args, **kwargs) + finally: + if alias_token is not None: + reset_read_db_alias(alias_token) + self.db_alias = MainRouter.default_db def initial(self, request, *args, **kwargs): # Ideally, this logic would be in the `.setup()` method but DRF view sets don't call it @@ -61,7 +90,9 @@ class BaseRLSViewSet(BaseViewSet): if tenant_id is None: raise NotAuthenticated("Tenant ID is not present in token") - with rls_transaction(tenant_id): + with rls_transaction( + tenant_id, using=getattr(self, "db_alias", MainRouter.default_db) + ): self.request.tenant_id = tenant_id return super().initial(request, *args, **kwargs) @@ -73,18 +104,33 @@ class BaseRLSViewSet(BaseViewSet): class BaseTenantViewset(BaseViewSet): def dispatch(self, request, *args, **kwargs): - with transaction.atomic(): - tenant = super().dispatch(request, *args, **kwargs) - + self.db_alias = self._get_request_db_alias(request) + alias_token = None try: - # If the request is a POST, create the admin role - if request.method == "POST": - isinstance(tenant, dict) and self._create_admin_role(tenant.data["id"]) - except Exception as e: - self._handle_creation_error(e, tenant) - raise + if self.db_alias != MainRouter.default_db: + alias_token = set_read_db_alias(self.db_alias) - return tenant + if request is not None: + request.db_alias = self.db_alias + + with transaction.atomic(using=self.db_alias): + tenant = super().dispatch(request, *args, **kwargs) + + try: + # If the request is a POST, create the admin role + if request.method == "POST": + isinstance(tenant, dict) and self._create_admin_role( + tenant.data["id"] + ) + except Exception as e: + self._handle_creation_error(e, tenant) + raise + + return tenant + finally: + if alias_token is not None: + reset_read_db_alias(alias_token) + self.db_alias = MainRouter.default_db def _create_admin_role(self, tenant_id): Role.objects.using(MainRouter.admin_db).create( @@ -117,14 +163,31 @@ class BaseTenantViewset(BaseViewSet): raise NotAuthenticated("Tenant ID is not present in token") user_id = str(request.user.id) - with rls_transaction(value=user_id, parameter=POSTGRES_USER_VAR): + with rls_transaction( + value=user_id, + parameter=POSTGRES_USER_VAR, + using=getattr(self, "db_alias", MainRouter.default_db), + ): return super().initial(request, *args, **kwargs) class BaseUserViewset(BaseViewSet): def dispatch(self, request, *args, **kwargs): - with transaction.atomic(): - return super().dispatch(request, *args, **kwargs) + self.db_alias = self._get_request_db_alias(request) + alias_token = None + try: + if self.db_alias != MainRouter.default_db: + alias_token = set_read_db_alias(self.db_alias) + + if request is not None: + request.db_alias = self.db_alias + + with transaction.atomic(using=self.db_alias): + return super().dispatch(request, *args, **kwargs) + finally: + if alias_token is not None: + reset_read_db_alias(alias_token) + self.db_alias = MainRouter.default_db def initial(self, request, *args, **kwargs): # TODO refactor after improving RLS on users @@ -137,6 +200,8 @@ class BaseUserViewset(BaseViewSet): if tenant_id is None: raise NotAuthenticated("Tenant ID is not present in token") - with rls_transaction(tenant_id): + with rls_transaction( + tenant_id, using=getattr(self, "db_alias", MainRouter.default_db) + ): self.request.tenant_id = tenant_id return super().initial(request, *args, **kwargs) diff --git a/api/src/backend/api/compliance.py b/api/src/backend/api/compliance.py index 832b1745eb..da39fc23bb 100644 --- a/api/src/backend/api/compliance.py +++ b/api/src/backend/api/compliance.py @@ -144,18 +144,23 @@ def generate_scan_compliance( Returns: None: This function modifies the compliance_overview in place. """ + for compliance_id in PROWLER_CHECKS[provider_type][check_id]: for requirement in compliance_overview[compliance_id]["requirements"].values(): if check_id in requirement["checks"]: requirement["checks"][check_id] = status requirement["checks_status"][status.lower()] += 1 - if requirement["status"] != "FAIL" and any( - value == "FAIL" for value in requirement["checks"].values() - ): - requirement["status"] = "FAIL" - compliance_overview[compliance_id]["requirements_status"]["passed"] -= 1 - compliance_overview[compliance_id]["requirements_status"]["failed"] += 1 + if requirement["status"] != "FAIL" and any( + value == "FAIL" for value in requirement["checks"].values() + ): + requirement["status"] = "FAIL" + compliance_overview[compliance_id]["requirements_status"][ + "passed" + ] -= 1 + compliance_overview[compliance_id]["requirements_status"][ + "failed" + ] += 1 def generate_compliance_overview_template(prowler_compliance: dict): diff --git a/api/src/backend/api/db_router.py b/api/src/backend/api/db_router.py index dc34c1191a..2a9085746e 100644 --- a/api/src/backend/api/db_router.py +++ b/api/src/backend/api/db_router.py @@ -1,9 +1,31 @@ +from contextvars import ContextVar + +from django.conf import settings + ALLOWED_APPS = ("django", "socialaccount", "account", "authtoken", "silk") +_read_db_alias = ContextVar("read_db_alias", default=None) + + +def set_read_db_alias(alias: str | None): + if not alias: + return None + return _read_db_alias.set(alias) + + +def get_read_db_alias() -> str | None: + return _read_db_alias.get() + + +def reset_read_db_alias(token) -> None: + if token is not None: + _read_db_alias.reset(token) + class MainRouter: default_db = "default" admin_db = "admin" + replica_db = "replica" def db_for_read(self, model, **hints): # noqa: F841 model_table_name = model._meta.db_table @@ -11,6 +33,9 @@ class MainRouter: model_table_name.startswith(f"{app}_") for app in ALLOWED_APPS ): return self.admin_db + read_alias = get_read_db_alias() + if read_alias: + return read_alias return None def db_for_write(self, model, **hints): # noqa: F841 @@ -23,7 +48,13 @@ class MainRouter: return db == self.admin_db def allow_relation(self, obj1, obj2, **hints): # noqa: F841 - # Allow relations if both objects are in either "default" or "admin" db connectors - if {obj1._state.db, obj2._state.db} <= {self.default_db, self.admin_db}: + # Allow relations when both objects originate from allowed connectors + allowed_dbs = {self.default_db, self.admin_db, self.replica_db} + if {obj1._state.db, obj2._state.db} <= allowed_dbs: return True return None + + +READ_REPLICA_ALIAS = ( + MainRouter.replica_db if MainRouter.replica_db in settings.DATABASES else None +) diff --git a/api/src/backend/api/db_utils.py b/api/src/backend/api/db_utils.py index 337f8444fe..c6fcaeb43a 100644 --- a/api/src/backend/api/db_utils.py +++ b/api/src/backend/api/db_utils.py @@ -1,17 +1,36 @@ import re import secrets +import time import uuid from contextlib import contextmanager from datetime import datetime, timedelta, timezone +from celery.utils.log import get_task_logger +from config.env import env from django.conf import settings from django.contrib.auth.models import BaseUserManager -from django.db import connection, models, transaction +from django.db import ( + DEFAULT_DB_ALIAS, + OperationalError, + connection, + connections, + models, + transaction, +) from django_celery_beat.models import PeriodicTask from psycopg2 import connect as psycopg2_connect from psycopg2.extensions import AsIs, new_type, register_adapter, register_type from rest_framework_json_api.serializers import ValidationError +from api.db_router import ( + READ_REPLICA_ALIAS, + get_read_db_alias, + reset_read_db_alias, + set_read_db_alias, +) + +logger = get_task_logger(__name__) + DB_USER = settings.DATABASES["default"]["USER"] if not settings.TESTING else "test" DB_PASSWORD = ( settings.DATABASES["default"]["PASSWORD"] if not settings.TESTING else "test" @@ -26,6 +45,9 @@ TASK_RUNNER_DB_TABLE = "django_celery_results_taskresult" POSTGRES_TENANT_VAR = "api.tenant_id" POSTGRES_USER_VAR = "api.user_id" +REPLICA_MAX_ATTEMPTS = env.int("POSTGRES_REPLICA_MAX_ATTEMPTS", default=3) +REPLICA_RETRY_BASE_DELAY = env.float("POSTGRES_REPLICA_RETRY_BASE_DELAY", default=0.5) + SET_CONFIG_QUERY = "SELECT set_config(%s, %s::text, TRUE);" @@ -49,7 +71,11 @@ def psycopg_connection(database_alias: str): @contextmanager -def rls_transaction(value: str, parameter: str = POSTGRES_TENANT_VAR): +def rls_transaction( + value: str, + parameter: str = POSTGRES_TENANT_VAR, + using: str | None = None, +): """ Creates a new database transaction setting the given configuration value for Postgres RLS. It validates the if the value is a valid UUID. @@ -57,16 +83,59 @@ def rls_transaction(value: str, parameter: str = POSTGRES_TENANT_VAR): Args: value (str): Database configuration parameter value. parameter (str): Database configuration parameter name, by default is 'api.tenant_id'. + using (str | None): Optional database alias to run the transaction against. Defaults to the + active read alias (if any) or Django's default connection. """ - with transaction.atomic(): - with connection.cursor() as cursor: - try: - # just in case the value is a UUID object - uuid.UUID(str(value)) - except ValueError: - raise ValidationError("Must be a valid UUID") - cursor.execute(SET_CONFIG_QUERY, [parameter, value]) - yield cursor + requested_alias = using or get_read_db_alias() + db_alias = requested_alias or DEFAULT_DB_ALIAS + if db_alias not in connections: + db_alias = DEFAULT_DB_ALIAS + + alias = db_alias + is_replica = READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS + max_attempts = REPLICA_MAX_ATTEMPTS if is_replica else 1 + + for attempt in range(1, max_attempts + 1): + router_token = None + + # On final attempt, fallback to primary + if attempt == max_attempts and is_replica: + logger.warning( + f"RLS transaction failed after {attempt - 1} attempts on replica, " + f"falling back to primary DB" + ) + alias = DEFAULT_DB_ALIAS + + conn = connections[alias] + try: + if alias != DEFAULT_DB_ALIAS: + router_token = set_read_db_alias(alias) + + with transaction.atomic(using=alias): + with conn.cursor() as cursor: + try: + # just in case the value is a UUID object + uuid.UUID(str(value)) + except ValueError: + raise ValidationError("Must be a valid UUID") + cursor.execute(SET_CONFIG_QUERY, [parameter, value]) + yield cursor + return + except OperationalError as e: + # If on primary or max attempts reached, raise + if not is_replica or attempt == max_attempts: + raise + + # Retry with exponential backoff + delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1)) + logger.info( + f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), " + f"retrying in {delay}s. Error: {e}" + ) + time.sleep(delay) + finally: + if router_token is not None: + reset_read_db_alias(router_token) class CustomUserManager(BaseUserManager): diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index ed706006e9..1d863267f2 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -27,7 +27,10 @@ from api.models import ( Finding, Integration, Invitation, + LighthouseProviderConfiguration, + LighthouseProviderModels, Membership, + MuteRule, OverviewStatusChoices, PermissionChoices, Processor, @@ -44,6 +47,7 @@ from api.models import ( StatusChoices, Task, TenantAPIKey, + ThreatScoreSnapshot, User, ) from api.rls import Tenant @@ -245,6 +249,14 @@ class ProviderFilter(FilterSet): choices=Provider.ProviderChoices.choices, lookup_expr="in", ) + provider_type = ChoiceFilter( + choices=Provider.ProviderChoices.choices, field_name="provider" + ) + provider_type__in = ChoiceInFilter( + field_name="provider", + choices=Provider.ProviderChoices.choices, + lookup_expr="in", + ) class Meta: model = Provider @@ -765,6 +777,7 @@ class ComplianceOverviewFilter(FilterSet): class ScanSummaryFilter(FilterSet): inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") + provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in") provider_type = ChoiceFilter( field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices ) @@ -927,3 +940,95 @@ class TenantApiKeyFilter(FilterSet): "revoked": ["exact"], "name": ["exact", "icontains"], } + + +class LighthouseProviderConfigFilter(FilterSet): + provider_type = ChoiceFilter( + choices=LighthouseProviderConfiguration.LLMProviderChoices.choices + ) + provider_type__in = ChoiceInFilter( + choices=LighthouseProviderConfiguration.LLMProviderChoices.choices, + field_name="provider_type", + lookup_expr="in", + ) + is_active = BooleanFilter() + + class Meta: + model = LighthouseProviderConfiguration + fields = { + "provider_type": ["exact", "in"], + "is_active": ["exact"], + } + + +class LighthouseProviderModelsFilter(FilterSet): + provider_type = ChoiceFilter( + choices=LighthouseProviderConfiguration.LLMProviderChoices.choices, + field_name="provider_configuration__provider_type", + ) + provider_type__in = ChoiceInFilter( + choices=LighthouseProviderConfiguration.LLMProviderChoices.choices, + field_name="provider_configuration__provider_type", + lookup_expr="in", + ) + + # Allow filtering by model id + model_id = CharFilter(field_name="model_id", lookup_expr="exact") + model_id__icontains = CharFilter(field_name="model_id", lookup_expr="icontains") + model_id__in = CharInFilter(field_name="model_id", lookup_expr="in") + + class Meta: + model = LighthouseProviderModels + fields = { + "model_id": ["exact", "icontains", "in"], + } + + +class MuteRuleFilter(FilterSet): + inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") + updated_at = DateFilter(field_name="updated_at", lookup_expr="date") + created_by = UUIDFilter(field_name="created_by__id", lookup_expr="exact") + + class Meta: + model = MuteRule + fields = { + "id": ["exact", "in"], + "name": ["exact", "icontains"], + "reason": ["icontains"], + "enabled": ["exact"], + "inserted_at": ["gte", "lte"], + "updated_at": ["gte", "lte"], + } + + +class ThreatScoreSnapshotFilter(FilterSet): + """ + Filter for ThreatScore snapshots. + Allows filtering by scan, provider, compliance_id, and date ranges. + """ + + inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") + scan_id = UUIDFilter(field_name="scan__id", lookup_expr="exact") + scan_id__in = UUIDInFilter(field_name="scan__id", lookup_expr="in") + provider_id = UUIDFilter(field_name="provider__id", lookup_expr="exact") + provider_id__in = UUIDInFilter(field_name="provider__id", lookup_expr="in") + provider_type = ChoiceFilter( + field_name="provider__provider", choices=Provider.ProviderChoices.choices + ) + provider_type__in = ChoiceInFilter( + field_name="provider__provider", + choices=Provider.ProviderChoices.choices, + lookup_expr="in", + ) + compliance_id = CharFilter(field_name="compliance_id", lookup_expr="exact") + compliance_id__in = CharInFilter(field_name="compliance_id", lookup_expr="in") + + class Meta: + model = ThreatScoreSnapshot + fields = { + "scan": ["exact", "in"], + "provider": ["exact", "in"], + "compliance_id": ["exact", "in"], + "inserted_at": ["date", "gte", "lte"], + "overall_score": ["exact", "gte", "lte"], + } diff --git a/api/src/backend/api/migrations/0048_api_key.py b/api/src/backend/api/migrations/0048_api_key.py index 9bc68af8e6..c3142ecda1 100644 --- a/api/src/backend/api/migrations/0048_api_key.py +++ b/api/src/backend/api/migrations/0048_api_key.py @@ -2,6 +2,7 @@ import uuid +import django.core.validators import django.db.models.deletion import drf_simple_apikey.models from django.conf import settings @@ -20,7 +21,13 @@ class Migration(migrations.Migration): migrations.CreateModel( name="TenantAPIKey", fields=[ - ("name", models.CharField(blank=True, max_length=255, null=True)), + ( + "name", + models.CharField( + max_length=100, + validators=[django.core.validators.MinLengthValidator(3)], + ), + ), ( "expiry_date", models.DateTimeField( @@ -37,7 +44,7 @@ class Migration(migrations.Migration): help_text="If the API key is revoked, entities cannot use it anymore. (This cannot be undone.)", ), ), - ("created", models.DateTimeField(auto_now=True)), + ("created", models.DateTimeField(auto_now_add=True, editable=False)), ( "whitelisted_ips", models.JSONField( @@ -110,7 +117,11 @@ class Migration(migrations.Migration): "constraints": [ models.UniqueConstraint( fields=("tenant_id", "prefix"), name="unique_api_key_prefixes" - ) + ), + models.UniqueConstraint( + fields=("tenant_id", "name"), + name="unique_api_key_name_per_tenant", + ), ], }, ), diff --git a/api/src/backend/api/migrations/0050_lighthouse_multi_llm.py b/api/src/backend/api/migrations/0050_lighthouse_multi_llm.py new file mode 100644 index 0000000000..99a9353327 --- /dev/null +++ b/api/src/backend/api/migrations/0050_lighthouse_multi_llm.py @@ -0,0 +1,266 @@ +# Generated by Django 5.1.12 on 2025-10-09 07:50 + +import json +import logging +import uuid + +import django.db.models.deletion +from config.custom_logging import BackendLogger +from cryptography.fernet import Fernet +from django.conf import settings +from django.db import migrations, models + +import api.rls +from api.db_router import MainRouter + +logger = logging.getLogger(BackendLogger.API) + + +def migrate_lighthouse_configs_forward(apps, schema_editor): + """ + Migrate data from old LighthouseConfiguration to new multi-provider models. + Old system: one LighthouseConfiguration per tenant (always OpenAI). + """ + LighthouseConfiguration = apps.get_model("api", "LighthouseConfiguration") + LighthouseProviderConfiguration = apps.get_model( + "api", "LighthouseProviderConfiguration" + ) + LighthouseTenantConfiguration = apps.get_model( + "api", "LighthouseTenantConfiguration" + ) + LighthouseProviderModels = apps.get_model("api", "LighthouseProviderModels") + + fernet = Fernet(settings.SECRETS_ENCRYPTION_KEY.encode()) + + # Migrate only tenants that actually have a LighthouseConfiguration + for old_config in ( + LighthouseConfiguration.objects.using(MainRouter.admin_db) + .select_related("tenant") + .all() + ): + tenant = old_config.tenant + tenant_id = str(tenant.id) + + try: + # Create OpenAI provider configuration for this tenant + api_key_decrypted = fernet.decrypt(bytes(old_config.api_key)).decode() + credentials_encrypted = fernet.encrypt( + json.dumps({"api_key": api_key_decrypted}).encode() + ) + provider_config = LighthouseProviderConfiguration.objects.using( + MainRouter.admin_db + ).create( + tenant=tenant, + provider_type="openai", + credentials=credentials_encrypted, + is_active=old_config.is_active, + ) + + # Create tenant configuration from old values + LighthouseTenantConfiguration.objects.using(MainRouter.admin_db).create( + tenant=tenant, + business_context=old_config.business_context or "", + default_provider="openai", + default_models={"openai": old_config.model}, + ) + + # Create initial provider model record + LighthouseProviderModels.objects.using(MainRouter.admin_db).create( + tenant=tenant, + provider_configuration=provider_config, + model_id=old_config.model, + model_name=old_config.model, + default_parameters={}, + ) + + except Exception: + logger.exception( + "Failed to migrate lighthouse config for tenant %s", tenant_id + ) + continue + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0049_compliancerequirementoverview_passed_failed_findings"), + ] + + operations = [ + migrations.CreateModel( + name="LighthouseProviderConfiguration", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "provider_type", + models.CharField( + choices=[("openai", "OpenAI")], + help_text="LLM provider name", + max_length=50, + ), + ), + ("base_url", models.URLField(blank=True, null=True)), + ( + "credentials", + models.BinaryField( + help_text="Encrypted JSON credentials for the provider" + ), + ), + ("is_active", models.BooleanField(default=True)), + ], + options={ + "db_table": "lighthouse_provider_configurations", + "abstract": False, + }, + ), + migrations.CreateModel( + name="LighthouseProviderModels", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("model_id", models.CharField(max_length=100)), + ("model_name", models.CharField(max_length=100)), + ("default_parameters", models.JSONField(blank=True, default=dict)), + ], + options={ + "db_table": "lighthouse_provider_models", + "abstract": False, + }, + ), + migrations.CreateModel( + name="LighthouseTenantConfiguration", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("business_context", models.TextField(blank=True, default="")), + ("default_provider", models.CharField(blank=True, max_length=50)), + ("default_models", models.JSONField(blank=True, default=dict)), + ], + options={ + "db_table": "lighthouse_tenant_config", + "abstract": False, + }, + ), + migrations.AddField( + model_name="lighthouseproviderconfiguration", + name="tenant", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + migrations.AddField( + model_name="lighthouseprovidermodels", + name="provider_configuration", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="available_models", + to="api.lighthouseproviderconfiguration", + ), + ), + migrations.AddField( + model_name="lighthouseprovidermodels", + name="tenant", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + migrations.AddField( + model_name="lighthousetenantconfiguration", + name="tenant", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + migrations.AddIndex( + model_name="lighthouseproviderconfiguration", + index=models.Index( + fields=["tenant_id", "provider_type"], name="lh_pc_tenant_type_idx" + ), + ), + migrations.AddConstraint( + model_name="lighthouseproviderconfiguration", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_lighthouseproviderconfiguration", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + migrations.AddConstraint( + model_name="lighthouseproviderconfiguration", + constraint=models.UniqueConstraint( + fields=("tenant_id", "provider_type"), + name="unique_provider_config_per_tenant", + ), + ), + migrations.AddIndex( + model_name="lighthouseprovidermodels", + index=models.Index( + fields=["tenant_id", "provider_configuration"], + name="lh_prov_models_cfg_idx", + ), + ), + migrations.AddConstraint( + model_name="lighthouseprovidermodels", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_lighthouseprovidermodels", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + migrations.AddConstraint( + model_name="lighthouseprovidermodels", + constraint=models.UniqueConstraint( + fields=("tenant_id", "provider_configuration", "model_id"), + name="unique_provider_model_per_configuration", + ), + ), + migrations.AddConstraint( + model_name="lighthousetenantconfiguration", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_lighthousetenantconfiguration", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + migrations.AddConstraint( + model_name="lighthousetenantconfiguration", + constraint=models.UniqueConstraint( + fields=("tenant_id",), name="unique_tenant_lighthouse_config" + ), + ), + # Migrate data from old LighthouseConfiguration to new tables + # This runs after all tables, indexes, and constraints are created + # The old Lighthouse configuration table is not removed, so reverse_code is noop + # During rollbacks, the old Lighthouse configuration remains intact while the new tables are removed + migrations.RunPython( + migrate_lighthouse_configs_forward, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/api/src/backend/api/migrations/0051_oraclecloud_provider.py b/api/src/backend/api/migrations/0051_oraclecloud_provider.py new file mode 100644 index 0000000000..022c022ea6 --- /dev/null +++ b/api/src/backend/api/migrations/0051_oraclecloud_provider.py @@ -0,0 +1,34 @@ +# Generated by Django 5.1.7 on 2025-10-14 00:00 + +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0050_lighthouse_multi_llm"), + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("oraclecloud", "Oracle Cloud Infrastructure"), + ], + default="aws", + ), + ), + migrations.RunSQL( + "ALTER TYPE provider ADD VALUE IF NOT EXISTS 'oraclecloud';", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/api/src/backend/api/migrations/0052_mute_rules.py b/api/src/backend/api/migrations/0052_mute_rules.py new file mode 100644 index 0000000000..56a3ff516f --- /dev/null +++ b/api/src/backend/api/migrations/0052_mute_rules.py @@ -0,0 +1,117 @@ +# Generated by Django 5.1.13 on 2025-10-22 11:56 + +import uuid + +import django.contrib.postgres.fields +import django.core.validators +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0051_oraclecloud_provider"), + ] + + operations = [ + migrations.CreateModel( + name="MuteRule", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "name", + models.CharField( + help_text="Human-readable name for this rule", + max_length=100, + validators=[django.core.validators.MinLengthValidator(3)], + ), + ), + ( + "reason", + models.TextField( + help_text="Reason for muting", + max_length=500, + validators=[django.core.validators.MinLengthValidator(3)], + ), + ), + ( + "enabled", + models.BooleanField( + default=True, help_text="Whether this rule is currently enabled" + ), + ), + ( + "finding_uids", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=255), + help_text="List of finding UIDs to mute", + size=None, + ), + ), + ], + options={ + "db_table": "mute_rules", + "abstract": False, + }, + ), + migrations.AddField( + model_name="finding", + name="muted_at", + field=models.DateTimeField( + blank=True, help_text="Timestamp when this finding was muted", null=True + ), + ), + migrations.AlterField( + model_name="tenantapikey", + name="name", + field=models.CharField( + max_length=100, + validators=[django.core.validators.MinLengthValidator(3)], + ), + ), + migrations.AddField( + model_name="muterule", + name="created_by", + field=models.ForeignKey( + help_text="User who created this rule", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="created_mute_rules", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddField( + model_name="muterule", + name="tenant", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + migrations.AddConstraint( + model_name="muterule", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_muterule", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + migrations.AddConstraint( + model_name="muterule", + constraint=models.UniqueConstraint( + fields=("tenant_id", "name"), name="unique_mute_rule_name_per_tenant" + ), + ), + ] diff --git a/api/src/backend/api/migrations/0053_lighthouse_bedrock_openai_compatible.py b/api/src/backend/api/migrations/0053_lighthouse_bedrock_openai_compatible.py new file mode 100644 index 0000000000..d7054d6443 --- /dev/null +++ b/api/src/backend/api/migrations/0053_lighthouse_bedrock_openai_compatible.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1.12 on 2025-10-14 11:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0052_mute_rules"), + ] + + operations = [ + migrations.AlterField( + model_name="lighthouseproviderconfiguration", + name="provider_type", + field=models.CharField( + choices=[ + ("openai", "OpenAI"), + ("bedrock", "AWS Bedrock"), + ("openai_compatible", "OpenAI Compatible"), + ], + help_text="LLM provider name", + max_length=50, + ), + ) + ] diff --git a/api/src/backend/api/migrations/0054_iac_provider.py b/api/src/backend/api/migrations/0054_iac_provider.py new file mode 100644 index 0000000000..3ba456b0cf --- /dev/null +++ b/api/src/backend/api/migrations/0054_iac_provider.py @@ -0,0 +1,35 @@ +# Generated by Django 5.1.10 on 2025-09-09 09:25 + +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0053_lighthouse_bedrock_openai_compatible"), + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("oci", "Oracle Cloud Infrastructure"), + ("iac", "IaC"), + ], + default="aws", + ), + ), + migrations.RunSQL( + "ALTER TYPE provider ADD VALUE IF NOT EXISTS 'iac';", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/api/src/backend/api/migrations/0055_mongodbatlas_provider.py b/api/src/backend/api/migrations/0055_mongodbatlas_provider.py new file mode 100644 index 0000000000..beb7a3946a --- /dev/null +++ b/api/src/backend/api/migrations/0055_mongodbatlas_provider.py @@ -0,0 +1,32 @@ +# Generated by Django 5.1.13 on 2025-11-05 08:37 + +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0054_iac_provider"), + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("mongodbatlas", "MongoDB Atlas"), + ("iac", "IaC"), + ("oraclecloud", "Oracle Cloud Infrastructure"), + ], + default="aws", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0056_remove_provider_unique_provider_uids_and_more.py b/api/src/backend/api/migrations/0056_remove_provider_unique_provider_uids_and_more.py new file mode 100644 index 0000000000..14cbc4fa47 --- /dev/null +++ b/api/src/backend/api/migrations/0056_remove_provider_unique_provider_uids_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.13 on 2025-11-06 09:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0055_mongodbatlas_provider"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="provider", + name="unique_provider_uids", + ), + migrations.AddConstraint( + model_name="provider", + constraint=models.UniqueConstraint( + condition=models.Q(("is_deleted", False)), + fields=("tenant_id", "provider", "uid"), + name="unique_provider_uids", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0057_threatscoresnapshot.py b/api/src/backend/api/migrations/0057_threatscoresnapshot.py new file mode 100644 index 0000000000..ee3530a5b6 --- /dev/null +++ b/api/src/backend/api/migrations/0057_threatscoresnapshot.py @@ -0,0 +1,170 @@ +# Generated by Django 5.1.13 on 2025-10-31 09:04 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0056_remove_provider_unique_provider_uids_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="ThreatScoreSnapshot", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ( + "compliance_id", + models.CharField( + help_text="Compliance framework ID (e.g., 'prowler_threatscore_aws')", + max_length=100, + ), + ), + ( + "overall_score", + models.DecimalField( + decimal_places=2, + help_text="Overall ThreatScore percentage (0-100)", + max_digits=5, + ), + ), + ( + "score_delta", + models.DecimalField( + blank=True, + decimal_places=2, + help_text="Score change compared to previous snapshot (positive = improvement)", + max_digits=5, + null=True, + ), + ), + ( + "section_scores", + models.JSONField( + blank=True, + default=dict, + help_text="ThreatScore breakdown by section", + ), + ), + ( + "critical_requirements", + models.JSONField( + blank=True, + default=list, + help_text="List of critical failed requirements (risk >= 4)", + ), + ), + ( + "total_requirements", + models.IntegerField( + default=0, help_text="Total number of requirements evaluated" + ), + ), + ( + "passed_requirements", + models.IntegerField( + default=0, help_text="Number of requirements with PASS status" + ), + ), + ( + "failed_requirements", + models.IntegerField( + default=0, help_text="Number of requirements with FAIL status" + ), + ), + ( + "manual_requirements", + models.IntegerField( + default=0, help_text="Number of requirements with MANUAL status" + ), + ), + ( + "total_findings", + models.IntegerField( + default=0, + help_text="Total number of findings across all requirements", + ), + ), + ( + "passed_findings", + models.IntegerField( + default=0, help_text="Number of findings with PASS status" + ), + ), + ( + "failed_findings", + models.IntegerField( + default=0, help_text="Number of findings with FAIL status" + ), + ), + ( + "provider", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="threatscore_snapshots", + related_query_name="threatscore_snapshot", + to="api.provider", + ), + ), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="threatscore_snapshots", + related_query_name="threatscore_snapshot", + to="api.scan", + ), + ), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + ], + options={ + "db_table": "threatscore_snapshots", + "abstract": False, + }, + ), + migrations.AddIndex( + model_name="threatscoresnapshot", + index=models.Index( + fields=["tenant_id", "scan_id"], name="threatscore_snap_t_scan_idx" + ), + ), + migrations.AddIndex( + model_name="threatscoresnapshot", + index=models.Index( + fields=["tenant_id", "provider_id"], name="threatscore_snap_t_prov_idx" + ), + ), + migrations.AddIndex( + model_name="threatscoresnapshot", + index=models.Index( + fields=["tenant_id", "inserted_at"], name="threatscore_snap_t_time_idx" + ), + ), + migrations.AddConstraint( + model_name="threatscoresnapshot", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_threatscoresnapshot", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index 048f1ca7db..536c5a65f6 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -220,6 +220,8 @@ class Membership(models.Model): class TenantAPIKey(AbstractAPIKey, RowLevelSecurityProtectedModel): id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + name = models.CharField(max_length=100, validators=[MinLengthValidator(3)]) + created = models.DateTimeField(auto_now_add=True, editable=False) prefix = models.CharField( max_length=11, unique=True, @@ -255,6 +257,10 @@ class TenantAPIKey(AbstractAPIKey, RowLevelSecurityProtectedModel): fields=("tenant_id", "prefix"), name="unique_api_key_prefixes", ), + models.UniqueConstraint( + fields=("tenant_id", "name"), + name="unique_api_key_name_per_tenant", + ), ] indexes = [ @@ -278,6 +284,9 @@ class Provider(RowLevelSecurityProtectedModel): KUBERNETES = "kubernetes", _("Kubernetes") M365 = "m365", _("M365") GITHUB = "github", _("GitHub") + MONGODBATLAS = "mongodbatlas", _("MongoDB Atlas") + IAC = "iac", _("IaC") + ORACLECLOUD = "oraclecloud", _("Oracle Cloud Infrastructure") @staticmethod def validate_aws_uid(value): @@ -348,6 +357,40 @@ class Provider(RowLevelSecurityProtectedModel): pointer="/data/attributes/uid", ) + @staticmethod + def validate_iac_uid(value): + # Validate that it's a valid repository URL (git URL format) + if not re.match( + r"^(https?://|git@|ssh://)[^\s/]+[^\s]*\.git$|^(https?://)[^\s/]+[^\s]*$", + value, + ): + raise ModelValidationError( + detail="IaC provider ID must be a valid repository URL (e.g., https://github.com/user/repo or https://github.com/user/repo.git).", + code="iac-uid", + pointer="/data/attributes/uid", + ) + + @staticmethod + def validate_oraclecloud_uid(value): + if not re.match( + r"^ocid1\.([a-z0-9_-]+)\.([a-z0-9_-]+)\.([a-z0-9_-]*)\.([a-z0-9]+)$", value + ): + raise ModelValidationError( + detail="Oracle Cloud Infrastructure provider ID must be a valid tenancy OCID in the format: " + "ocid1....", + code="oraclecloud-uid", + pointer="/data/attributes/uid", + ) + + @staticmethod + def validate_mongodbatlas_uid(value): + if not re.match(r"^[0-9a-fA-F]{24}$", value): + raise ModelValidationError( + detail="MongoDB Atlas organization ID must be a 24-character hexadecimal string.", + code="mongodbatlas-uid", + pointer="/data/attributes/uid", + ) + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) inserted_at = models.DateTimeField(auto_now_add=True, editable=False) updated_at = models.DateTimeField(auto_now=True, editable=False) @@ -382,7 +425,8 @@ class Provider(RowLevelSecurityProtectedModel): constraints = [ models.UniqueConstraint( - fields=("tenant_id", "provider", "uid", "is_deleted"), + fields=("tenant_id", "provider", "uid"), + condition=Q(is_deleted=False), name="unique_provider_uids", ), RowLevelSecurityConstraint( @@ -804,6 +848,9 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): muted_reason = models.TextField( blank=True, null=True, validators=[MinLengthValidator(3)], max_length=500 ) + muted_at = models.DateTimeField( + null=True, blank=True, help_text="Timestamp when this finding was muted" + ) compliance = models.JSONField(default=dict, null=True, blank=True) # Denormalize resource data for performance @@ -1867,22 +1914,6 @@ class LighthouseConfiguration(RowLevelSecurityProtectedModel): def clean(self): super().clean() - # Validate temperature - if not 0 <= self.temperature <= 1: - raise ModelValidationError( - detail="Temperature must be between 0 and 1", - code="invalid_temperature", - pointer="/data/attributes/temperature", - ) - - # Validate max_tokens - if not 500 <= self.max_tokens <= 5000: - raise ModelValidationError( - detail="Max tokens must be between 500 and 5000", - code="invalid_max_tokens", - pointer="/data/attributes/max_tokens", - ) - @property def api_key_decoded(self): """Return the decrypted API key, or None if unavailable or invalid.""" @@ -1907,15 +1938,6 @@ class LighthouseConfiguration(RowLevelSecurityProtectedModel): code="invalid_api_key", pointer="/data/attributes/api_key", ) - - # Validate OpenAI API key format - openai_key_pattern = r"^sk-[\w-]+T3BlbkFJ[\w-]+$" - if not re.match(openai_key_pattern, value): - raise ModelValidationError( - detail="Invalid OpenAI API key format.", - code="invalid_api_key", - pointer="/data/attributes/api_key", - ) self.api_key = fernet.encrypt(value.encode()) def save(self, *args, **kwargs): @@ -1941,6 +1963,59 @@ class LighthouseConfiguration(RowLevelSecurityProtectedModel): resource_name = "lighthouse-configurations" +class MuteRule(RowLevelSecurityProtectedModel): + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + updated_at = models.DateTimeField(auto_now=True, editable=False) + + # Rule metadata + name = models.CharField( + max_length=100, + validators=[MinLengthValidator(3)], + help_text="Human-readable name for this rule", + ) + reason = models.TextField( + validators=[MinLengthValidator(3)], + max_length=500, + help_text="Reason for muting", + ) + enabled = models.BooleanField( + default=True, help_text="Whether this rule is currently enabled" + ) + + # Audit fields + created_by = models.ForeignKey( + User, + on_delete=models.SET_NULL, + null=True, + related_name="created_mute_rules", + help_text="User who created this rule", + ) + + # Rule criteria - array of finding UIDs + finding_uids = ArrayField( + models.CharField(max_length=255), help_text="List of finding UIDs to mute" + ) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "mute_rules" + + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + models.UniqueConstraint( + fields=("tenant_id", "name"), + name="unique_mute_rule_name_per_tenant", + ), + ] + + class JSONAPIMeta: + resource_name = "mute-rules" + + class Processor(RowLevelSecurityProtectedModel): class ProcessorChoices(models.TextChoices): MUTELIST = "mutelist", _("Mutelist") @@ -1978,3 +2053,323 @@ class Processor(RowLevelSecurityProtectedModel): class JSONAPIMeta: resource_name = "processors" + + +class LighthouseProviderConfiguration(RowLevelSecurityProtectedModel): + """ + Per-tenant configuration for an LLM provider (credentials, base URL, activation). + + One configuration per provider type per tenant. + """ + + class LLMProviderChoices(models.TextChoices): + OPENAI = "openai", _("OpenAI") + BEDROCK = "bedrock", _("AWS Bedrock") + OPENAI_COMPATIBLE = "openai_compatible", _("OpenAI Compatible") + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + updated_at = models.DateTimeField(auto_now=True, editable=False) + + provider_type = models.CharField( + max_length=50, + choices=LLMProviderChoices.choices, + help_text="LLM provider name", + ) + + # For OpenAI-compatible providers + base_url = models.URLField(blank=True, null=True) + + # Encrypted JSON for provider-specific auth + credentials = models.BinaryField( + blank=False, null=False, help_text="Encrypted JSON credentials for the provider" + ) + + is_active = models.BooleanField(default=True) + + def __str__(self): + return f"{self.get_provider_type_display()} ({self.tenant_id})" + + def clean(self): + super().clean() + + @property + def credentials_decoded(self): + if not self.credentials: + return None + try: + decrypted_data = fernet.decrypt(bytes(self.credentials)) + return json.loads(decrypted_data.decode()) + except (InvalidToken, json.JSONDecodeError) as e: + logger.warning("Failed to decrypt provider credentials: %s", e) + return None + except Exception as e: + logger.exception( + "Unexpected error while decrypting provider credentials: %s", e + ) + return None + + @credentials_decoded.setter + def credentials_decoded(self, value): + """ + Set and encrypt credentials (assumes serializer performed validation). + """ + if not value: + raise ModelValidationError( + detail="Credentials are required", + code="invalid_credentials", + pointer="/data/attributes/credentials", + ) + self.credentials = fernet.encrypt(json.dumps(value).encode()) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "lighthouse_provider_configurations" + + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + models.UniqueConstraint( + fields=["tenant_id", "provider_type"], + name="unique_provider_config_per_tenant", + ), + ] + + indexes = [ + models.Index( + fields=["tenant_id", "provider_type"], + name="lh_pc_tenant_type_idx", + ), + ] + + class JSONAPIMeta: + resource_name = "lighthouse-providers" + + +class LighthouseTenantConfiguration(RowLevelSecurityProtectedModel): + """ + Tenant-level Lighthouse settings (business context and defaults). + One record per tenant. + """ + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + updated_at = models.DateTimeField(auto_now=True, editable=False) + + business_context = models.TextField(blank=True, default="") + + # Preferred provider key (e.g., "openai", "bedrock", "openai_compatible") + default_provider = models.CharField(max_length=50, blank=True) + + # Mapping of provider -> model id, e.g., {"openai": "gpt-4o", "bedrock": "anthropic.claude-v2"} + default_models = models.JSONField(default=dict, blank=True) + + def __str__(self): + return f"Lighthouse Tenant Config for {self.tenant_id}" + + def clean(self): + super().clean() + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "lighthouse_tenant_config" + + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + models.UniqueConstraint( + fields=["tenant_id"], name="unique_tenant_lighthouse_config" + ), + ] + + class JSONAPIMeta: + resource_name = "lighthouse-configurations" + + +class LighthouseProviderModels(RowLevelSecurityProtectedModel): + """ + Per-tenant, per-provider configuration list of available LLM models. + RLS-protected; populated via provider API using tenant-scoped credentials. + """ + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + updated_at = models.DateTimeField(auto_now=True, editable=False) + + # Scope to a specific provider configuration within a tenant + provider_configuration = models.ForeignKey( + LighthouseProviderConfiguration, + on_delete=models.CASCADE, + related_name="available_models", + ) + model_id = models.CharField(max_length=100) + + # Human-friendly model name + model_name = models.CharField(max_length=100) + + # Model-specific default parameters (e.g., temperature, max_tokens) + default_parameters = models.JSONField(default=dict, blank=True) + + def __str__(self): + return f"{self.provider_configuration.provider_type}:{self.model_id} ({self.tenant_id})" + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "lighthouse_provider_models" + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + models.UniqueConstraint( + fields=["tenant_id", "provider_configuration", "model_id"], + name="unique_provider_model_per_configuration", + ), + ] + indexes = [ + models.Index( + fields=["tenant_id", "provider_configuration"], + name="lh_prov_models_cfg_idx", + ), + ] + + class JSONAPIMeta: + resource_name = "lighthouse-models" + + +class ThreatScoreSnapshot(RowLevelSecurityProtectedModel): + """ + Stores historical ThreatScore metrics for a given scan. + Snapshots are created automatically after each ThreatScore report generation. + """ + + objects = models.Manager() + all_objects = models.Manager() + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + + scan = models.ForeignKey( + Scan, + on_delete=models.CASCADE, + related_name="threatscore_snapshots", + related_query_name="threatscore_snapshot", + ) + + provider = models.ForeignKey( + Provider, + on_delete=models.CASCADE, + related_name="threatscore_snapshots", + related_query_name="threatscore_snapshot", + ) + + compliance_id = models.CharField( + max_length=100, + blank=False, + null=False, + help_text="Compliance framework ID (e.g., 'prowler_threatscore_aws')", + ) + + # Overall ThreatScore metrics + overall_score = models.DecimalField( + max_digits=5, + decimal_places=2, + help_text="Overall ThreatScore percentage (0-100)", + ) + + # Score improvement/degradation compared to previous snapshot + score_delta = models.DecimalField( + max_digits=5, + decimal_places=2, + null=True, + blank=True, + help_text="Score change compared to previous snapshot (positive = improvement)", + ) + + # Section breakdown stored as JSON + # Format: {"1. IAM": 85.5, "2. Attack Surface": 92.3, ...} + section_scores = models.JSONField( + default=dict, + blank=True, + help_text="ThreatScore breakdown by section", + ) + + # Critical requirements metadata stored as JSON + # Format: [{"requirement_id": "...", "risk_level": 5, "weight": 150, ...}, ...] + critical_requirements = models.JSONField( + default=list, + blank=True, + help_text="List of critical failed requirements (risk >= 4)", + ) + + # Summary statistics + total_requirements = models.IntegerField( + default=0, + help_text="Total number of requirements evaluated", + ) + + passed_requirements = models.IntegerField( + default=0, + help_text="Number of requirements with PASS status", + ) + + failed_requirements = models.IntegerField( + default=0, + help_text="Number of requirements with FAIL status", + ) + + manual_requirements = models.IntegerField( + default=0, + help_text="Number of requirements with MANUAL status", + ) + + total_findings = models.IntegerField( + default=0, + help_text="Total number of findings across all requirements", + ) + + passed_findings = models.IntegerField( + default=0, + help_text="Number of findings with PASS status", + ) + + failed_findings = models.IntegerField( + default=0, + help_text="Number of findings with FAIL status", + ) + + def __str__(self): + return f"ThreatScore {self.overall_score}% for scan {self.scan_id} ({self.inserted_at})" + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "threatscore_snapshots" + + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + + indexes = [ + models.Index( + fields=["tenant_id", "scan_id"], + name="threatscore_snap_t_scan_idx", + ), + models.Index( + fields=["tenant_id", "provider_id"], + name="threatscore_snap_t_prov_idx", + ), + models.Index( + fields=["tenant_id", "inserted_at"], + name="threatscore_snap_t_time_idx", + ), + ] + + class JSONAPIMeta: + resource_name = "threatscore-snapshots" diff --git a/api/src/backend/api/renderers.py b/api/src/backend/api/renderers.py index ee03f24758..44fd0edff1 100644 --- a/api/src/backend/api/renderers.py +++ b/api/src/backend/api/renderers.py @@ -11,11 +11,12 @@ class APIJSONRenderer(JSONRenderer): def render(self, data, accepted_media_type=None, renderer_context=None): request = renderer_context.get("request") tenant_id = getattr(request, "tenant_id", None) if request else None + db_alias = getattr(request, "db_alias", None) if request else None include_param_present = "include" in request.query_params if request else False # Use rls_transaction if needed for included resources, otherwise do nothing context_manager = ( - rls_transaction(tenant_id) + rls_transaction(tenant_id, using=db_alias) if tenant_id and include_param_present else nullcontext() ) diff --git a/api/src/backend/api/signals.py b/api/src/backend/api/signals.py index aac89ea33e..d449144bf4 100644 --- a/api/src/backend/api/signals.py +++ b/api/src/backend/api/signals.py @@ -6,7 +6,14 @@ from django.dispatch import receiver from django_celery_results.backends.database import DatabaseBackend from api.db_utils import delete_related_daily_task -from api.models import Membership, Provider, TenantAPIKey, User +from api.models import ( + LighthouseProviderConfiguration, + LighthouseTenantConfiguration, + Membership, + Provider, + TenantAPIKey, + User, +) def create_task_result_on_publish(sender=None, headers=None, **kwargs): # noqa: F841 @@ -56,3 +63,33 @@ def revoke_membership_api_keys(sender, instance, **kwargs): # noqa: F841 TenantAPIKey.objects.filter( entity=instance.user, tenant_id=instance.tenant.id ).update(revoked=True) + + +@receiver(pre_delete, sender=LighthouseProviderConfiguration) +def cleanup_lighthouse_defaults_before_delete(sender, instance, **kwargs): # noqa: F841 + """ + Ensure tenant Lighthouse defaults do not reference a soon-to-be-deleted provider. + + This runs for both per-instance deletes and queryset (bulk) deletes. + """ + try: + tenant_cfg = LighthouseTenantConfiguration.objects.get( + tenant_id=instance.tenant_id + ) + except LighthouseTenantConfiguration.DoesNotExist: + return + + updated = False + defaults = tenant_cfg.default_models or {} + + if instance.provider_type in defaults: + defaults.pop(instance.provider_type, None) + tenant_cfg.default_models = defaults + updated = True + + if tenant_cfg.default_provider == instance.provider_type: + tenant_cfg.default_provider = "" + updated = True + + if updated: + tenant_cfg.save() diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index daec67a79e..e44dba0084 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.14.0 + version: 1.15.0 description: |- Prowler API specification. @@ -869,14 +869,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -884,20 +887,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -907,6 +915,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -1396,14 +1407,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -1411,20 +1425,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -1434,6 +1453,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -1831,14 +1853,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -1846,20 +1871,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -1869,6 +1899,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -2264,14 +2297,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -2279,20 +2315,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -2302,6 +2343,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -2685,14 +2729,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -2700,20 +2747,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -2723,6 +2775,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -3429,6 +3484,7 @@ paths: - Lighthouse AI security: - JWT or API Key: [] + deprecated: true responses: '200': content: @@ -3456,6 +3512,7 @@ paths: required: true security: - JWT or API Key: [] + deprecated: true responses: '201': content: @@ -3490,6 +3547,7 @@ paths: required: true security: - JWT or API Key: [] + deprecated: true responses: '200': content: @@ -3511,6 +3569,7 @@ paths: - Lighthouse AI security: - JWT or API Key: [] + deprecated: true responses: '204': description: No response body @@ -3530,6 +3589,563 @@ paths: - Lighthouse AI security: - JWT or API Key: [] + deprecated: true + responses: + '202': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/TaskResponse' + examples: + Task queued: + summary: Task queued + value: + data: + type: tasks + id: 497f6eca-6276-4993-bfeb-53cbbbba6f08 + attributes: + inserted_at: '2019-08-24T14:15:22Z' + completed_at: '2019-08-24T14:15:22Z' + name: string + state: available + result: null + task_args: null + metadata: null + description: '' + /api/v1/lighthouse/configuration: + get: + operationId: lighthouse_configuration_list + description: Retrieve current tenant-level Lighthouse AI settings. Returns a + single configuration object. + summary: Get Lighthouse AI Tenant config + parameters: + - in: query + name: fields[lighthouse-configurations] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - business_context + - default_provider + - default_models + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - inserted_at + - -inserted_at + - updated_at + - -updated_at + - business_context + - -business_context + - default_provider + - -default_provider + - default_models + - -default_models + - url + - -url + explode: false + tags: + - Lighthouse AI + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedLighthouseTenantConfigList' + description: '' + patch: + operationId: lighthouse_configuration_partial_update + description: Update tenant-level settings. Validates that the default provider + is configured and active and that default model IDs exist for the chosen providers. + Auto-creates configuration if it doesn't exist. + summary: Update Lighthouse AI Tenant config + tags: + - Lighthouse AI + requestBody: + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PatchedLighthouseTenantConfigUpdateRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PatchedLighthouseTenantConfigUpdateRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/PatchedLighthouseTenantConfigUpdateRequest' + required: true + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/LighthouseTenantConfigUpdateResponse' + description: '' + /api/v1/lighthouse/models: + get: + operationId: lighthouse_models_list + description: List available LLM models per configured provider for the current + tenant. + summary: List all LLM models + parameters: + - in: query + name: fields[lighthouse-models] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - provider_configuration + - model_id + - model_name + - default_parameters + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[model_id] + schema: + type: string + - in: query + name: filter[model_id__icontains] + schema: + type: string + - in: query + name: filter[model_id__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + x-spec-enum-id: 30bc523cb9ec0e8b + enum: + - bedrock + - openai + - openai_compatible + description: |- + LLM provider name + + * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + x-spec-enum-id: 30bc523cb9ec0e8b + enum: + - bedrock + - openai + - openai_compatible + description: |- + Multiple values may be separated by commas. + + * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - inserted_at + - -inserted_at + - updated_at + - -updated_at + - provider_configuration + - -provider_configuration + - model_id + - -model_id + - model_name + - -model_name + - default_parameters + - -default_parameters + - url + - -url + explode: false + tags: + - Lighthouse AI + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedLighthouseProviderModelsList' + description: '' + /api/v1/lighthouse/models/{id}: + get: + operationId: lighthouse_models_retrieve + description: Get details for a specific LLM model. + summary: Retrieve LLM model details + parameters: + - in: query + name: fields[lighthouse-models] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - provider_configuration + - model_id + - model_name + - default_parameters + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this lighthouse provider models. + required: true + tags: + - Lighthouse AI + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/LighthouseProviderModelsResponse' + description: '' + /api/v1/lighthouse/providers: + get: + operationId: lighthouse_providers_list + description: Retrieve all LLM provider configurations for the current tenant + summary: List all LLM provider configurations + parameters: + - in: query + name: fields[lighthouse-providers] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - provider_type + - base_url + - is_active + - credentials + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[is_active] + schema: + type: boolean + - in: query + name: filter[provider_type] + schema: + type: string + x-spec-enum-id: 30bc523cb9ec0e8b + enum: + - bedrock + - openai + - openai_compatible + description: |- + LLM provider name + + * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + x-spec-enum-id: 30bc523cb9ec0e8b + enum: + - bedrock + - openai + - openai_compatible + description: |- + Multiple values may be separated by commas. + + * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - inserted_at + - -inserted_at + - updated_at + - -updated_at + - provider_type + - -provider_type + - base_url + - -base_url + - is_active + - -is_active + - credentials + - -credentials + - url + - -url + explode: false + tags: + - Lighthouse AI + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedLighthouseProviderConfigList' + description: '' + post: + operationId: lighthouse_providers_create + description: Create a per-tenant configuration for an LLM provider. Only one + configuration per provider type is allowed per tenant. + summary: Create LLM provider configuration + tags: + - Lighthouse AI + requestBody: + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/LighthouseProviderConfigCreateRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/LighthouseProviderConfigCreateRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/LighthouseProviderConfigCreateRequest' + required: true + security: + - JWT or API Key: [] + responses: + '201': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/LighthouseProviderConfigCreateResponse' + description: '' + /api/v1/lighthouse/providers/{id}: + get: + operationId: lighthouse_providers_retrieve + description: Get details for a specific provider configuration in the current + tenant. + summary: Retrieve LLM provider configuration + parameters: + - in: query + name: fields[lighthouse-providers] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - provider_type + - base_url + - is_active + - credentials + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this lighthouse provider configuration. + required: true + tags: + - Lighthouse AI + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/LighthouseProviderConfigResponse' + description: '' + patch: + operationId: lighthouse_providers_partial_update + description: Partially update a provider configuration (e.g., base_url, is_active). + summary: Update LLM provider configuration + parameters: + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this lighthouse provider configuration. + required: true + tags: + - Lighthouse AI + requestBody: + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PatchedLighthouseProviderConfigUpdateRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PatchedLighthouseProviderConfigUpdateRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/PatchedLighthouseProviderConfigUpdateRequest' + required: true + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/LighthouseProviderConfigUpdateResponse' + description: '' + delete: + operationId: lighthouse_providers_destroy + description: Delete a provider configuration. Any tenant defaults that reference + this provider are cleared during deletion. + summary: Delete LLM provider configuration + parameters: + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this lighthouse provider configuration. + required: true + tags: + - Lighthouse AI + security: + - JWT or API Key: [] + responses: + '204': + description: No response body + /api/v1/lighthouse/providers/{id}/connection: + post: + operationId: lighthouse_providers_connection_create + description: Validate provider credentials asynchronously and toggle is_active. + summary: Check LLM provider connection + parameters: + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this lighthouse provider configuration. + required: true + tags: + - Lighthouse AI + security: + - JWT or API Key: [] responses: '202': content: @@ -3552,6 +4168,330 @@ paths: task_args: null metadata: null description: '' + /api/v1/lighthouse/providers/{id}/refresh-models: + post: + operationId: lighthouse_providers_refresh_models_create + description: Fetch available models for this provider configuration and upsert + into catalog. Supports OpenAI, OpenAI-compatible, and AWS Bedrock providers. + summary: Refresh LLM models catalog + parameters: + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this lighthouse provider configuration. + required: true + tags: + - Lighthouse AI + security: + - JWT or API Key: [] + responses: + '202': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/TaskResponse' + examples: + Task queued: + summary: Task queued + value: + data: + type: tasks + id: 497f6eca-6276-4993-bfeb-53cbbbba6f08 + attributes: + inserted_at: '2019-08-24T14:15:22Z' + completed_at: '2019-08-24T14:15:22Z' + name: string + state: available + result: null + task_args: null + metadata: null + description: '' + /api/v1/mute-rules: + get: + operationId: mute_rules_list + description: Retrieve a list of all mute rules with filtering options. + summary: List all mute rules + parameters: + - in: query + name: fields[mute-rules] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - name + - reason + - enabled + - created_by + - finding_uids + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[created_by] + schema: + type: string + format: uuid + - in: query + name: filter[enabled] + schema: + type: boolean + - in: query + name: filter[id] + schema: + type: string + format: uuid + - in: query + name: filter[id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[inserted_at] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[inserted_at__lte] + schema: + type: string + format: date-time + - in: query + name: filter[name] + schema: + type: string + - in: query + name: filter[name__icontains] + schema: + type: string + - in: query + name: filter[reason__icontains] + schema: + type: string + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[updated_at] + schema: + type: string + format: date + - in: query + name: filter[updated_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[updated_at__lte] + schema: + type: string + format: date-time + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - created_by + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - name + - -name + - enabled + - -enabled + - inserted_at + - -inserted_at + - updated_at + - -updated_at + explode: false + tags: + - Mute Rules + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedMuteRuleList' + description: '' + post: + operationId: mute_rules_create + description: Create a new mute rule by providing finding IDs, name, and reason. + The rule will immediately mute the selected findings and launch a background + task to mute all historical findings with matching UIDs. + summary: Create a new mute rule + tags: + - Mute Rules + requestBody: + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/MuteRuleCreateRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/MuteRuleCreateRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/MuteRuleCreateRequest' + required: true + security: + - JWT or API Key: [] + responses: + '201': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/MuteRuleCreateResponse' + description: '' + /api/v1/mute-rules/{id}: + get: + operationId: mute_rules_retrieve + description: Fetch detailed information about a specific mute rule by ID. + summary: Retrieve a mute rule + parameters: + - in: query + name: fields[mute-rules] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - name + - reason + - enabled + - created_by + - finding_uids + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this mute rule. + required: true + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - created_by + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + tags: + - Mute Rules + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/MuteRuleResponse' + description: '' + patch: + operationId: mute_rules_partial_update + description: Update certain fields of an existing mute rule (e.g., name, reason, + enabled). + summary: Partially update a mute rule + parameters: + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this mute rule. + required: true + tags: + - Mute Rules + requestBody: + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PatchedMuteRuleUpdateRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PatchedMuteRuleUpdateRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/PatchedMuteRuleUpdateRequest' + required: true + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/SerializerMetaclassResponse' + description: '' + delete: + operationId: mute_rules_destroy + description: 'Remove a mute rule from the system. Note: Previously muted findings + remain muted.' + summary: Delete a mute rule + parameters: + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this mute rule. + required: true + tags: + - Mute Rules + security: + - JWT or API Key: [] + responses: + '204': + description: No response body /api/v1/overviews/findings: get: operationId: overviews_findings_retrieve @@ -3611,18 +4551,31 @@ paths: schema: type: string format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -3630,20 +4583,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -3653,6 +4611,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -3778,18 +4739,31 @@ paths: schema: type: string format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -3797,20 +4771,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -3820,6 +4799,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -3931,6 +4913,37 @@ paths: schema: $ref: '#/components/schemas/OverviewProviderResponse' description: '' + /api/v1/overviews/providers/count: + get: + operationId: overviews_providers_count_retrieve + description: Retrieve the number of providers grouped by provider type. This + endpoint counts every provider in the tenant, including those without completed + scans. + summary: Get provider counts grouped by type + parameters: + - in: query + name: fields[providers-count-overview] + schema: + type: array + items: + type: string + enum: + - id + - count + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + tags: + - Overview + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/OverviewProviderCountResponse' + description: '' /api/v1/overviews/services: get: operationId: overviews_services_retrieve @@ -3980,18 +4993,31 @@ paths: schema: type: string format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form - in: query name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -3999,20 +5025,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -4022,6 +5053,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -4707,14 +5741,17 @@ paths: name: filter[provider] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -4722,20 +5759,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -4745,6 +5787,56 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `oraclecloud` - Oracle Cloud Infrastructure + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + x-spec-enum-id: 7087703783970b51 + enum: + - aws + - azure + - gcp + - github + - kubernetes + - m365 + - oraclecloud + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `oraclecloud` - Oracle Cloud Infrastructure + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + x-spec-enum-id: 7087703783970b51 + enum: + - aws + - azure + - gcp + - github + - kubernetes + - m365 + - oraclecloud + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - name: filter[search] @@ -5342,14 +6434,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -5357,20 +6452,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -5380,6 +6480,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -5707,14 +6810,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -5722,20 +6828,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -5745,6 +6856,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -5973,14 +7087,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -5988,20 +7105,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -6011,6 +7133,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -6245,14 +7370,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -6260,20 +7388,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -6283,6 +7416,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -7080,14 +8216,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas + - iac - kubernetes - m365 + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -7095,20 +8234,25 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 enum: - aws - azure - gcp - github + - mongodbatlas - kubernetes - m365 + - oraclecloud description: |- Multiple values may be separated by commas. @@ -7118,6 +8262,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -7491,6 +8638,72 @@ paths: '404': description: The scan has no reports, or the report generation task has not started yet + /api/v1/scans/{id}/threatscore: + get: + operationId: scans_threatscore_retrieve + description: Download a specific threatscore report (e.g., 'prowler_threatscore_aws') + as a PDF file. + summary: Retrieve threatscore report + parameters: + - in: query + name: fields[scans] + schema: + type: array + items: + type: string + enum: + - name + - trigger + - state + - unique_resource_count + - progress + - duration + - provider + - task + - inserted_at + - started_at + - completed_at + - scheduled_at + - next_scan_at + - processor + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this scan. + required: true + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - provider + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + tags: + - Scan + security: + - JWT or API Key: [] + responses: + '200': + description: PDF file containing the threatscore report + '202': + description: The task is in progress + '401': + description: API key missing or user not Authenticated + '403': + description: There is a problem with credentials + '404': + description: The scan has no threatscore reports, or the threatscore report + generation task has not started yet /api/v1/schedules/daily: post: operationId: schedules_daily_create @@ -10823,6 +12036,481 @@ components: $ref: '#/components/schemas/LighthouseConfigUpdate' required: - data + LighthouseProviderConfig: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - lighthouse-providers + id: + type: string + format: uuid + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + provider_type: + enum: + - openai + - bedrock + - openai_compatible + type: string + x-spec-enum-id: 30bc523cb9ec0e8b + description: |- + LLM provider name + + * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible + base_url: + type: string + format: uri + nullable: true + maxLength: 200 + is_active: + type: boolean + readOnly: true + credentials: + readOnly: true + required: + - provider_type + LighthouseProviderConfigCreate: + type: object + required: + - type + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - lighthouse-providers + attributes: + type: object + properties: + provider_type: + enum: + - openai + - bedrock + - openai_compatible + type: string + x-spec-enum-id: 30bc523cb9ec0e8b + description: |- + LLM provider type. Determines which credential format to use. See 'credentials' field documentation for provider-specific requirements. + + * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible + base_url: + type: string + format: uri + nullable: true + description: Base URL for the LLM provider API. Required for 'openai_compatible' + provider type. + credentials: + oneOf: + - type: object + title: OpenAI Credentials + properties: + api_key: + type: string + description: OpenAI API key. Must start with 'sk-' followed by + alphanumeric characters, hyphens, or underscores. + pattern: ^sk-[\w-]+$ + required: + - api_key + - type: object + title: AWS Bedrock Credentials + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - type: object + title: OpenAI Compatible Credentials + properties: + api_key: + type: string + description: 'API key for OpenAI-compatible provider. The format + varies by provider. Note: The ''base_url'' field (separate from + credentials) is required when using this provider type.' + required: + - api_key + writeOnly: true + is_active: + type: boolean + required: + - provider_type + - credentials + LighthouseProviderConfigCreateRequest: + type: object + properties: + data: + type: object + required: + - type + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - lighthouse-providers + attributes: + type: object + properties: + provider_type: + enum: + - openai + - bedrock + - openai_compatible + type: string + x-spec-enum-id: 30bc523cb9ec0e8b + description: |- + LLM provider type. Determines which credential format to use. See 'credentials' field documentation for provider-specific requirements. + + * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible + base_url: + type: string + format: uri + nullable: true + minLength: 1 + description: Base URL for the LLM provider API. Required for 'openai_compatible' + provider type. + credentials: + oneOf: + - type: object + title: OpenAI Credentials + properties: + api_key: + type: string + description: OpenAI API key. Must start with 'sk-' followed + by alphanumeric characters, hyphens, or underscores. + pattern: ^sk-[\w-]+$ + required: + - api_key + - type: object + title: AWS Bedrock Credentials + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - type: object + title: OpenAI Compatible Credentials + properties: + api_key: + type: string + description: 'API key for OpenAI-compatible provider. The + format varies by provider. Note: The ''base_url'' field + (separate from credentials) is required when using this + provider type.' + required: + - api_key + writeOnly: true + is_active: + type: boolean + required: + - provider_type + - credentials + required: + - data + LighthouseProviderConfigCreateResponse: + type: object + properties: + data: + $ref: '#/components/schemas/LighthouseProviderConfigCreate' + required: + - data + LighthouseProviderConfigResponse: + type: object + properties: + data: + $ref: '#/components/schemas/LighthouseProviderConfig' + required: + - data + LighthouseProviderConfigUpdate: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - lighthouse-providers + id: + type: string + format: uuid + attributes: + type: object + properties: + provider_type: + enum: + - openai + - bedrock + - openai_compatible + type: string + x-spec-enum-id: 30bc523cb9ec0e8b + readOnly: true + description: |- + LLM provider name + + * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible + base_url: + type: string + format: uri + nullable: true + description: Base URL for the LLM provider API. Required for 'openai_compatible' + provider type. + credentials: + oneOf: + - type: object + title: OpenAI Credentials + properties: + api_key: + type: string + description: OpenAI API key. Must start with 'sk-' followed by + alphanumeric characters, hyphens, or underscores. + pattern: ^sk-[\w-]+$ + required: + - api_key + - type: object + title: AWS Bedrock Credentials + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - type: object + title: OpenAI Compatible Credentials + properties: + api_key: + type: string + description: 'API key for OpenAI-compatible provider. The format + varies by provider. Note: The ''base_url'' field (separate from + credentials) is required when using this provider type.' + required: + - api_key + writeOnly: true + is_active: + type: boolean + LighthouseProviderConfigUpdateResponse: + type: object + properties: + data: + $ref: '#/components/schemas/LighthouseProviderConfigUpdate' + required: + - data + LighthouseProviderModels: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - lighthouse-models + id: + type: string + format: uuid + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + model_id: + type: string + maxLength: 100 + model_name: + type: string + maxLength: 100 + default_parameters: {} + required: + - model_id + - model_name + relationships: + type: object + properties: + provider_configuration: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - lighthouse-providers + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + LighthouseProviderModelsResponse: + type: object + properties: + data: + $ref: '#/components/schemas/LighthouseProviderModels' + required: + - data + LighthouseTenantConfig: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - lighthouse-configurations + id: + type: string + format: uuid + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + business_context: + type: string + default_provider: + type: string + maxLength: 50 + default_models: {} + url: + type: string + readOnly: true + LighthouseTenantConfigUpdate: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - lighthouse-configurations + id: + type: string + format: uuid + attributes: + type: object + properties: + business_context: + type: string + default_provider: + type: string + maxLength: 50 + default_models: {} + LighthouseTenantConfigUpdateResponse: + type: object + properties: + data: + $ref: '#/components/schemas/LighthouseTenantConfigUpdate' + required: + - data Membership: type: object required: @@ -10914,6 +12602,272 @@ components: $ref: '#/components/schemas/Membership' required: - data + MuteRule: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - mute-rules + id: + type: string + format: uuid + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + description: Human-readable name for this rule + maxLength: 100 + minLength: 3 + reason: + type: string + description: Reason for muting + minLength: 3 + maxLength: 500 + enabled: + type: boolean + description: Whether this rule is currently enabled + finding_uids: + type: array + items: + type: string + readOnly: true + description: List of finding UIDs that are muted by this rule + required: + - name + - reason + relationships: + type: object + properties: + created_by: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - users + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + nullable: true + MuteRuleCreate: + type: object + required: + - type + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - mute-rules + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + description: Human-readable name for this rule + maxLength: 100 + minLength: 3 + reason: + type: string + description: Reason for muting + minLength: 3 + maxLength: 500 + enabled: + type: boolean + readOnly: true + description: Whether this rule is currently enabled + finding_ids: + type: array + items: + type: string + format: uuid + writeOnly: true + description: List of Finding IDs to mute (will be converted to UIDs) + finding_uids: + type: array + items: + type: string + readOnly: true + description: List of finding UIDs that are muted by this rule + required: + - name + - reason + - finding_ids + relationships: + type: object + properties: + created_by: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - users + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + nullable: true + MuteRuleCreateRequest: + type: object + properties: + data: + type: object + required: + - type + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - mute-rules + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + minLength: 3 + description: Human-readable name for this rule + maxLength: 100 + reason: + type: string + minLength: 3 + description: Reason for muting + maxLength: 500 + enabled: + type: boolean + readOnly: true + description: Whether this rule is currently enabled + finding_ids: + type: array + items: + type: string + format: uuid + writeOnly: true + description: List of Finding IDs to mute (will be converted to UIDs) + finding_uids: + type: array + items: + type: string + minLength: 1 + readOnly: true + description: List of finding UIDs that are muted by this rule + required: + - name + - reason + - finding_ids + relationships: + type: object + properties: + created_by: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - users + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share + common attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + nullable: true + required: + - data + MuteRuleCreateResponse: + type: object + properties: + data: + $ref: '#/components/schemas/MuteRuleCreate' + required: + - data + MuteRuleResponse: + type: object + properties: + data: + $ref: '#/components/schemas/MuteRule' + required: + - data OpenApiResponseResponse: type: object properties: @@ -11029,6 +12983,38 @@ components: readOnly: true required: - id + OverviewProviderCount: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - providers-count-overview + id: {} + attributes: + type: object + properties: + id: + type: string + count: + type: integer + required: + - id + - count + OverviewProviderCountResponse: + type: object + properties: + data: + $ref: '#/components/schemas/OverviewProviderCount' + required: + - data OverviewProviderResponse: type: object properties: @@ -11184,6 +13170,33 @@ components: $ref: '#/components/schemas/LighthouseConfig' required: - data + PaginatedLighthouseProviderConfigList: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/LighthouseProviderConfig' + required: + - data + PaginatedLighthouseProviderModelsList: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/LighthouseProviderModels' + required: + - data + PaginatedLighthouseTenantConfigList: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/LighthouseTenantConfig' + required: + - data PaginatedMembershipList: type: object properties: @@ -11193,6 +13206,15 @@ components: $ref: '#/components/schemas/Membership' required: - data + PaginatedMuteRuleList: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/MuteRule' + required: + - data PaginatedProcessorList: type: object properties: @@ -11645,6 +13667,167 @@ components: readOnly: true required: - data + PatchedLighthouseProviderConfigUpdateRequest: + type: object + properties: + data: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - lighthouse-providers + id: + type: string + format: uuid + attributes: + type: object + properties: + provider_type: + enum: + - openai + - bedrock + - openai_compatible + type: string + x-spec-enum-id: 30bc523cb9ec0e8b + readOnly: true + description: |- + LLM provider name + + * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible + base_url: + type: string + format: uri + nullable: true + minLength: 1 + description: Base URL for the LLM provider API. Required for 'openai_compatible' + provider type. + credentials: + oneOf: + - type: object + title: OpenAI Credentials + properties: + api_key: + type: string + description: OpenAI API key. Must start with 'sk-' followed + by alphanumeric characters, hyphens, or underscores. + pattern: ^sk-[\w-]+$ + required: + - api_key + - type: object + title: AWS Bedrock Credentials + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - type: object + title: OpenAI Compatible Credentials + properties: + api_key: + type: string + description: 'API key for OpenAI-compatible provider. The + format varies by provider. Note: The ''base_url'' field + (separate from credentials) is required when using this + provider type.' + required: + - api_key + writeOnly: true + is_active: + type: boolean + required: + - data + PatchedLighthouseTenantConfigUpdateRequest: + type: object + properties: + data: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - lighthouse-configurations + id: + type: string + format: uuid + attributes: + type: object + properties: + business_context: + type: string + default_provider: + type: string + maxLength: 50 + default_models: {} + required: + - data + PatchedMuteRuleUpdateRequest: + type: object + properties: + data: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - mute-rules + id: + type: string + format: uuid + attributes: + type: object + properties: + name: + type: string + minLength: 3 + description: Human-readable name for this rule + maxLength: 100 + reason: + type: string + minLength: 3 + description: Reason for muting + maxLength: 500 + enabled: + type: boolean + description: Whether this rule is currently enabled + required: + - data PatchedProcessorUpdateRequest: type: object properties: @@ -12010,26 +14193,47 @@ components: type: string description: The Azure application (client) ID for authentication in Azure AD. - client_secret: - type: string - description: The client secret associated with the application - (client) ID, providing secure access. tenant_id: type: string description: The Azure tenant ID, representing the directory where the application is registered. + client_secret: + type: string + description: The client secret associated with the application + (client) ID, providing secure access. user: type: email description: User microsoft email address. + deprecated: true password: type: string description: User password. + deprecated: true required: - client_id - client_secret - tenant_id - user - password + - type: object + title: M365 Certificate Credentials + properties: + client_id: + type: string + description: The Azure application (client) ID for authentication + in Azure AD. + tenant_id: + type: string + description: The Azure tenant ID, representing the directory + where the application is registered. + certificate_content: + type: string + description: The certificate content in base64 format for + certificate-based authentication. + required: + - client_id + - tenant_id + - certificate_content - type: object title: GCP Static Credentials properties: @@ -12081,7 +14285,7 @@ components: type: string description: GitHub OAuth App token for authentication. required: - - oauth_app_token + - oauth_app_token - type: object title: GitHub App Credentials properties: @@ -12094,6 +14298,62 @@ components: required: - github_app_id - github_app_key + - type: object + title: MongoDB Atlas API Key + properties: + atlas_public_key: + type: string + description: MongoDB Atlas API public key. + atlas_private_key: + type: string + description: MongoDB Atlas API private key. + required: + - atlas_public_key + - atlas_private_key + - type: object + title: IaC Repository Credentials + properties: + repository_url: + type: string + description: Repository URL to scan for IaC files. + access_token: + type: string + description: Optional access token for private repositories. + required: + - repository_url + - type: object + title: Oracle Cloud Infrastructure (OCI) API Key Credentials + properties: + user: + type: string + description: The OCID of the user to authenticate with. + fingerprint: + type: string + description: The fingerprint of the API signing key. + key_file: + type: string + description: The path to the private key file for API signing. + Either key_file or key_content must be provided. + key_content: + type: string + description: The content of the private key for API signing + (base64 encoded). Either key_file or key_content must be + provided. + tenancy: + type: string + description: The OCID of the tenancy. + region: + type: string + description: The OCI region identifier (e.g., us-ashburn-1, + us-phoenix-1). + pass_phrase: + type: string + description: The passphrase for the private key, if encrypted. + required: + - user + - fingerprint + - tenancy + - region writeOnly: true required: - secret @@ -12433,8 +14693,8 @@ components: properties: name: type: string - nullable: true - maxLength: 255 + minLength: 3 + maxLength: 100 prefix: type: string readOnly: true @@ -12454,6 +14714,8 @@ components: readOnly: true nullable: true description: Last time this API key was used for authentication + required: + - name relationships: type: object properties: @@ -13086,6 +15348,8 @@ components: - kubernetes - m365 - github + - mongodbatlas + - oraclecloud type: string description: |- * `aws` - AWS @@ -13094,7 +15358,9 @@ components: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub - x-spec-enum-id: 4c1e219dad1cc0e7 + * `mongodbatlas` - MongoDB Atlas + * `oraclecloud` - Oracle Cloud Infrastructure + x-spec-enum-id: 7087703783970b51 uid: type: string title: Unique identifier for the provider, set by the provider @@ -13206,8 +15472,10 @@ components: - kubernetes - m365 - github + - mongodbatlas + - oraclecloud type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 description: |- Type of provider to create. @@ -13217,11 +15485,13 @@ components: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `oraclecloud` - Oracle Cloud Infrastructure uid: type: string title: Unique identifier for the provider, set by the provider description: Unique identifier for the provider, set by the provider, - e.g. AWS account ID, Azure subscription ID, GCP project ID, etc. + e.g. AWS account ID, Azure subscription ID, GCP project ID, MongoDB Atlas organization ID, etc. maxLength: 250 minLength: 3 required: @@ -13260,8 +15530,10 @@ components: - kubernetes - m365 - github + - mongodbatlas + - oraclecloud type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: 7087703783970b51 description: |- Type of provider to create. @@ -13271,12 +15543,14 @@ components: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `oraclecloud` - Oracle Cloud Infrastructure uid: type: string minLength: 3 title: Unique identifier for the provider, set by the provider description: Unique identifier for the provider, set by the provider, - e.g. AWS account ID, Azure subscription ID, GCP project ID, etc. + e.g. AWS account ID, Azure subscription ID, GCP project ID, MongoDB Atlas organization ID, etc. maxLength: 250 required: - uid @@ -13874,26 +16148,47 @@ components: type: string description: The Azure application (client) ID for authentication in Azure AD. - client_secret: - type: string - description: The client secret associated with the application - (client) ID, providing secure access. tenant_id: type: string description: The Azure tenant ID, representing the directory where the application is registered. + client_secret: + type: string + description: The client secret associated with the application + (client) ID, providing secure access. user: type: email description: User microsoft email address. + deprecated: true password: type: string description: User password. + deprecated: true required: - client_id - client_secret - tenant_id - user - password + - type: object + title: M365 Certificate Credentials + properties: + client_id: + type: string + description: The Azure application (client) ID for authentication + in Azure AD. + tenant_id: + type: string + description: The Azure tenant ID, representing the directory where + the application is registered. + certificate_content: + type: string + description: The certificate content in base64 format for certificate-based + authentication. + required: + - client_id + - tenant_id + - certificate_content - type: object title: GCP Static Credentials properties: @@ -13958,6 +16253,48 @@ components: required: - github_app_id - github_app_key + - type: object + title: IaC Repository Credentials + properties: + repository_url: + type: string + description: Repository URL to scan for IaC files. + access_token: + type: string + description: Optional access token for private repositories. + required: + - repository_url + - type: object + title: Oracle Cloud Infrastructure (OCI) API Key Credentials + properties: + user: + type: string + description: The OCID of the user to authenticate with. + fingerprint: + type: string + description: The fingerprint of the API signing key. + key_file: + type: string + description: The path to the private key file for API signing. + Either key_file or key_content must be provided. + key_content: + type: string + description: The content of the private key for API signing (base64 + encoded). Either key_file or key_content must be provided. + tenancy: + type: string + description: The OCID of the tenancy. + region: + type: string + description: The OCI region identifier (e.g., us-ashburn-1, us-phoenix-1). + pass_phrase: + type: string + description: The passphrase for the private key, if encrypted. + required: + - user + - fingerprint + - tenancy + - region writeOnly: true required: - secret_type @@ -14122,26 +16459,47 @@ components: type: string description: The Azure application (client) ID for authentication in Azure AD. - client_secret: - type: string - description: The client secret associated with the application - (client) ID, providing secure access. tenant_id: type: string description: The Azure tenant ID, representing the directory where the application is registered. + client_secret: + type: string + description: The client secret associated with the application + (client) ID, providing secure access. user: type: email description: User microsoft email address. + deprecated: true password: type: string description: User password. + deprecated: true required: - client_id - client_secret - tenant_id - user - password + - type: object + title: M365 Certificate Credentials + properties: + client_id: + type: string + description: The Azure application (client) ID for authentication + in Azure AD. + tenant_id: + type: string + description: The Azure tenant ID, representing the directory + where the application is registered. + certificate_content: + type: string + description: The certificate content in base64 format for + certificate-based authentication. + required: + - client_id + - tenant_id + - certificate_content - type: object title: GCP Static Credentials properties: @@ -14206,6 +16564,50 @@ components: required: - github_app_id - github_app_key + - type: object + title: IaC Repository Credentials + properties: + repository_url: + type: string + description: Repository URL to scan for IaC files. + access_token: + type: string + description: Optional access token for private repositories. + required: + - repository_url + - type: object + title: Oracle Cloud Infrastructure (OCI) API Key Credentials + properties: + user: + type: string + description: The OCID of the user to authenticate with. + fingerprint: + type: string + description: The fingerprint of the API signing key. + key_file: + type: string + description: The path to the private key file for API signing. + Either key_file or key_content must be provided. + key_content: + type: string + description: The content of the private key for API signing + (base64 encoded). Either key_file or key_content must be + provided. + tenancy: + type: string + description: The OCID of the tenancy. + region: + type: string + description: The OCI region identifier (e.g., us-ashburn-1, + us-phoenix-1). + pass_phrase: + type: string + description: The passphrase for the private key, if encrypted. + required: + - user + - fingerprint + - tenancy + - region writeOnly: true required: - secret_type @@ -14386,26 +16788,47 @@ components: type: string description: The Azure application (client) ID for authentication in Azure AD. - client_secret: - type: string - description: The client secret associated with the application - (client) ID, providing secure access. tenant_id: type: string description: The Azure tenant ID, representing the directory where the application is registered. + client_secret: + type: string + description: The client secret associated with the application + (client) ID, providing secure access. user: type: email description: User microsoft email address. + deprecated: true password: type: string description: User password. + deprecated: true required: - client_id - client_secret - tenant_id - user - password + - type: object + title: M365 Certificate Credentials + properties: + client_id: + type: string + description: The Azure application (client) ID for authentication + in Azure AD. + tenant_id: + type: string + description: The Azure tenant ID, representing the directory where + the application is registered. + certificate_content: + type: string + description: The certificate content in base64 format for certificate-based + authentication. + required: + - client_id + - tenant_id + - certificate_content - type: object title: GCP Static Credentials properties: @@ -14470,6 +16893,48 @@ components: required: - github_app_id - github_app_key + - type: object + title: IaC Repository Credentials + properties: + repository_url: + type: string + description: Repository URL to scan for IaC files. + access_token: + type: string + description: Optional access token for private repositories. + required: + - repository_url + - type: object + title: Oracle Cloud Infrastructure (OCI) API Key Credentials + properties: + user: + type: string + description: The OCID of the user to authenticate with. + fingerprint: + type: string + description: The fingerprint of the API signing key. + key_file: + type: string + description: The path to the private key file for API signing. + Either key_file or key_content must be provided. + key_content: + type: string + description: The content of the private key for API signing (base64 + encoded). Either key_file or key_content must be provided. + tenancy: + type: string + description: The OCID of the tenancy. + region: + type: string + description: The OCI region identifier (e.g., us-ashburn-1, us-phoenix-1). + pass_phrase: + type: string + description: The passphrase for the private key, if encrypted. + required: + - user + - fingerprint + - tenancy + - region writeOnly: true required: - secret @@ -15523,7 +17988,7 @@ components: type: object properties: data: - $ref: '#/components/schemas/ProviderGroup' + $ref: '#/components/schemas/MuteRule' required: - data Task: @@ -15659,8 +18124,8 @@ components: properties: name: type: string - nullable: true - maxLength: 255 + maxLength: 100 + minLength: 3 prefix: type: string readOnly: true @@ -15682,6 +18147,8 @@ components: format: date-time nullable: true description: Last time this API key was used for authentication + required: + - name relationships: type: object properties: @@ -15728,8 +18195,8 @@ components: properties: name: type: string - nullable: true - maxLength: 255 + maxLength: 100 + minLength: 3 prefix: type: string readOnly: true @@ -15755,6 +18222,8 @@ components: api_key: type: string readOnly: true + required: + - name relationships: type: object properties: @@ -15805,8 +18274,8 @@ components: properties: name: type: string - nullable: true - maxLength: 255 + minLength: 3 + maxLength: 100 prefix: type: string readOnly: true @@ -15833,6 +18302,8 @@ components: api_key: type: string readOnly: true + required: + - name relationships: type: object properties: @@ -15900,8 +18371,8 @@ components: properties: name: type: string - nullable: true - maxLength: 255 + maxLength: 100 + minLength: 3 prefix: type: string readOnly: true @@ -15920,6 +18391,8 @@ components: readOnly: true nullable: true description: Last time this API key was used for authentication + required: + - name relationships: type: object properties: @@ -16559,3 +19032,7 @@ tags: - name: API Keys description: Endpoints for API keys management. These can be used as an alternative to JWT authorization. +- name: Mute Rules + description: Endpoints for simple mute rules management. These can be used as an + alternative to the Mutelist Processor if you need to mute specific findings across + your tenant with a specific reason. diff --git a/api/src/backend/api/tests/integration/test_rls_transaction.py b/api/src/backend/api/tests/integration/test_rls_transaction.py new file mode 100644 index 0000000000..6731b39d71 --- /dev/null +++ b/api/src/backend/api/tests/integration/test_rls_transaction.py @@ -0,0 +1,39 @@ +"""Tests for rls_transaction retry and fallback logic.""" + +import pytest +from django.db import DEFAULT_DB_ALIAS +from rest_framework_json_api.serializers import ValidationError + +from api.db_utils import rls_transaction + + +@pytest.mark.django_db +class TestRLSTransaction: + """Simple integration tests for rls_transaction using real DB.""" + + @pytest.fixture + def tenant(self, tenants_fixture): + return tenants_fixture[0] + + def test_success_on_primary(self, tenant): + """Basic: transaction succeeds on primary database.""" + with rls_transaction(str(tenant.id), using=DEFAULT_DB_ALIAS) as cursor: + cursor.execute("SELECT 1") + result = cursor.fetchone() + assert result == (1,) + + def test_invalid_uuid_raises_validation_error(self): + """Invalid UUID raises ValidationError before DB operations.""" + with pytest.raises(ValidationError, match="Must be a valid UUID"): + with rls_transaction("not-a-uuid", using=DEFAULT_DB_ALIAS): + pass + + def test_custom_parameter_name(self, tenant): + """Test custom RLS parameter name.""" + custom_param = "api.custom_id" + with rls_transaction( + str(tenant.id), parameter=custom_param, using=DEFAULT_DB_ALIAS + ) as cursor: + cursor.execute("SELECT current_setting(%s, true)", [custom_param]) + result = cursor.fetchone() + assert result == (str(tenant.id),) diff --git a/api/src/backend/api/tests/test_db_utils.py b/api/src/backend/api/tests/test_db_utils.py index b4ec73e715..f706ab8c71 100644 --- a/api/src/backend/api/tests/test_db_utils.py +++ b/api/src/backend/api/tests/test_db_utils.py @@ -1,12 +1,15 @@ from datetime import datetime, timezone from enum import Enum -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from django.conf import settings +from django.db import DEFAULT_DB_ALIAS, OperationalError from freezegun import freeze_time +from rest_framework_json_api.serializers import ValidationError from api.db_utils import ( + POSTGRES_TENANT_VAR, _should_create_index_on_partition, batch_delete, create_objects_in_batches, @@ -14,11 +17,22 @@ from api.db_utils import ( generate_api_key_prefix, generate_random_token, one_week_from_now, + rls_transaction, update_objects_in_batches, ) from api.models import Provider +@pytest.fixture +def enable_read_replica(): + """ + Fixture to enable READ_REPLICA_ALIAS for tests that need replica functionality. + This avoids polluting the global test configuration. + """ + with patch("api.db_utils.READ_REPLICA_ALIAS", "replica"): + yield "replica" + + class TestEnumToChoices: def test_enum_to_choices_simple(self): class Color(Enum): @@ -339,3 +353,498 @@ class TestGenerateApiKeyPrefix: prefix = generate_api_key_prefix() random_part = prefix[3:] # Strip 'pk_' assert all(char in allowed_chars for char in random_part) + + +@pytest.mark.django_db +class TestRlsTransaction: + def test_rls_transaction_valid_uuid_string(self, tenants_fixture): + """Test rls_transaction with valid UUID string.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with rls_transaction(tenant_id) as cursor: + assert cursor is not None + cursor.execute("SELECT current_setting(%s)", [POSTGRES_TENANT_VAR]) + result = cursor.fetchone() + assert result[0] == tenant_id + + def test_rls_transaction_valid_uuid_object(self, tenants_fixture): + """Test rls_transaction with UUID object.""" + tenant = tenants_fixture[0] + + with rls_transaction(tenant.id) as cursor: + assert cursor is not None + cursor.execute("SELECT current_setting(%s)", [POSTGRES_TENANT_VAR]) + result = cursor.fetchone() + assert result[0] == str(tenant.id) + + def test_rls_transaction_invalid_uuid_raises_validation_error(self): + """Test rls_transaction raises ValidationError for invalid UUID.""" + invalid_uuid = "not-a-valid-uuid" + + with pytest.raises(ValidationError, match="Must be a valid UUID"): + with rls_transaction(invalid_uuid): + pass + + def test_rls_transaction_uses_default_database_when_no_alias(self, tenants_fixture): + """Test rls_transaction uses DEFAULT_DB_ALIAS when no alias specified.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=None): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with rls_transaction(tenant_id): + pass + + mock_connections.__getitem__.assert_called_with(DEFAULT_DB_ALIAS) + + def test_rls_transaction_uses_specified_alias(self, tenants_fixture): + """Test rls_transaction uses specified database alias via using parameter.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + custom_alias = "custom_db" + + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias") as mock_set_alias: + with patch("api.db_utils.reset_read_db_alias") as mock_reset_alias: + mock_set_alias.return_value = "test_token" + with rls_transaction(tenant_id, using=custom_alias): + pass + + mock_connections.__getitem__.assert_called_with(custom_alias) + mock_set_alias.assert_called_once_with(custom_alias) + mock_reset_alias.assert_called_once_with("test_token") + + def test_rls_transaction_uses_read_replica_from_router( + self, tenants_fixture, enable_read_replica + ): + """Test rls_transaction uses read replica alias from router.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias") as mock_set_alias: + with patch( + "api.db_utils.reset_read_db_alias" + ) as mock_reset_alias: + mock_set_alias.return_value = "test_token" + with rls_transaction(tenant_id): + pass + + mock_connections.__getitem__.assert_called() + mock_set_alias.assert_called_once() + mock_reset_alias.assert_called_once() + + def test_rls_transaction_fallback_to_default_when_alias_not_in_connections( + self, tenants_fixture + ): + """Test rls_transaction falls back to DEFAULT_DB_ALIAS when alias not in connections.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + invalid_alias = "nonexistent_db" + + with patch("api.db_utils.get_read_db_alias", return_value=invalid_alias): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + + def contains_check(alias): + return alias == DEFAULT_DB_ALIAS + + mock_connections.__contains__.side_effect = contains_check + mock_connections.__getitem__.return_value = mock_conn + + with patch("api.db_utils.transaction.atomic"): + with rls_transaction(tenant_id): + pass + + mock_connections.__getitem__.assert_called_with(DEFAULT_DB_ALIAS) + + def test_rls_transaction_successful_execution_on_replica_no_retries( + self, tenants_fixture, enable_read_replica + ): + """Test successful execution on replica without retries.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias", return_value="token"): + with patch("api.db_utils.reset_read_db_alias"): + with rls_transaction(tenant_id): + pass + + assert mock_cursor.execute.call_count == 1 + + def test_rls_transaction_retry_with_exponential_backoff_on_operational_error( + self, tenants_fixture, enable_read_replica + ): + """Test retry with exponential backoff on OperationalError on replica.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + call_count = 0 + + def atomic_side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise OperationalError("Connection error") + return MagicMock( + __enter__=MagicMock(return_value=None), + __exit__=MagicMock(return_value=False), + ) + + with patch( + "api.db_utils.transaction.atomic", side_effect=atomic_side_effect + ): + with patch("api.db_utils.time.sleep") as mock_sleep: + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with patch("api.db_utils.logger") as mock_logger: + with rls_transaction(tenant_id): + pass + + assert mock_sleep.call_count == 2 + mock_sleep.assert_any_call(0.5) + mock_sleep.assert_any_call(1.0) + assert mock_logger.info.call_count == 2 + + def test_rls_transaction_max_three_attempts_for_replica( + self, tenants_fixture, enable_read_replica + ): + """Test maximum 3 attempts for replica database.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = OperationalError("Persistent error") + + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with pytest.raises(OperationalError): + with rls_transaction(tenant_id): + pass + + assert mock_atomic.call_count == 3 + + def test_rls_transaction_only_one_attempt_for_primary(self, tenants_fixture): + """Test only 1 attempt for primary database.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=None): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = OperationalError("Primary error") + + with pytest.raises(OperationalError): + with rls_transaction(tenant_id): + pass + + assert mock_atomic.call_count == 1 + + def test_rls_transaction_fallback_to_primary_after_max_attempts( + self, tenants_fixture, enable_read_replica + ): + """Test fallback to primary DB after max attempts on replica.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + call_count = 0 + + def atomic_side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise OperationalError("Replica error") + return MagicMock( + __enter__=MagicMock(return_value=None), + __exit__=MagicMock(return_value=False), + ) + + with patch( + "api.db_utils.transaction.atomic", side_effect=atomic_side_effect + ): + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with patch("api.db_utils.logger") as mock_logger: + with rls_transaction(tenant_id): + pass + + mock_logger.warning.assert_called_once() + warning_msg = mock_logger.warning.call_args[0][0] + assert "falling back to primary DB" in warning_msg + + def test_rls_transaction_logger_warning_on_fallback( + self, tenants_fixture, enable_read_replica + ): + """Test logger warnings are emitted on fallback to primary.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + call_count = 0 + + def atomic_side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise OperationalError("Replica error") + return MagicMock( + __enter__=MagicMock(return_value=None), + __exit__=MagicMock(return_value=False), + ) + + with patch( + "api.db_utils.transaction.atomic", side_effect=atomic_side_effect + ): + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with patch("api.db_utils.logger") as mock_logger: + with rls_transaction(tenant_id): + pass + + assert mock_logger.info.call_count == 2 + assert mock_logger.warning.call_count == 1 + + def test_rls_transaction_operational_error_raised_immediately_on_primary( + self, tenants_fixture + ): + """Test OperationalError raised immediately on primary without retry.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=None): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = OperationalError("Primary error") + + with patch("api.db_utils.time.sleep") as mock_sleep: + with pytest.raises(OperationalError): + with rls_transaction(tenant_id): + pass + + mock_sleep.assert_not_called() + + def test_rls_transaction_operational_error_raised_after_max_attempts( + self, tenants_fixture, enable_read_replica + ): + """Test OperationalError raised after max attempts on replica.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = OperationalError( + "Persistent replica error" + ) + + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with pytest.raises(OperationalError): + with rls_transaction(tenant_id): + pass + + def test_rls_transaction_router_token_set_for_non_default_alias( + self, tenants_fixture + ): + """Test router token is set when using non-default alias.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + custom_alias = "custom_db" + + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias") as mock_set_alias: + with patch("api.db_utils.reset_read_db_alias") as mock_reset_alias: + mock_set_alias.return_value = "test_token" + with rls_transaction(tenant_id, using=custom_alias): + pass + + mock_set_alias.assert_called_once_with(custom_alias) + mock_reset_alias.assert_called_once_with("test_token") + + def test_rls_transaction_router_token_reset_in_finally_block(self, tenants_fixture): + """Test router token is reset in finally block even on error.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + custom_alias = "custom_db" + + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = Exception("Unexpected error") + + with patch("api.db_utils.set_read_db_alias", return_value="test_token"): + with patch("api.db_utils.reset_read_db_alias") as mock_reset_alias: + with pytest.raises(Exception): + with rls_transaction(tenant_id, using=custom_alias): + pass + + mock_reset_alias.assert_called_once_with("test_token") + + def test_rls_transaction_router_token_not_set_for_default_alias( + self, tenants_fixture + ): + """Test router token is not set when using default alias.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=None): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias") as mock_set_alias: + with patch( + "api.db_utils.reset_read_db_alias" + ) as mock_reset_alias: + with rls_transaction(tenant_id): + pass + + mock_set_alias.assert_not_called() + mock_reset_alias.assert_not_called() + + def test_rls_transaction_set_config_query_executed_with_correct_params( + self, tenants_fixture + ): + """Test SET_CONFIG_QUERY executed with correct parameters.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with rls_transaction(tenant_id) as cursor: + cursor.execute("SELECT current_setting(%s)", [POSTGRES_TENANT_VAR]) + result = cursor.fetchone() + assert result[0] == tenant_id + + def test_rls_transaction_custom_parameter(self, tenants_fixture): + """Test rls_transaction with custom parameter name.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + custom_param = "api.user_id" + + with rls_transaction(tenant_id, parameter=custom_param) as cursor: + cursor.execute("SELECT current_setting(%s)", [custom_param]) + result = cursor.fetchone() + assert result[0] == tenant_id + + def test_rls_transaction_cursor_yielded_correctly(self, tenants_fixture): + """Test cursor is yielded correctly.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with rls_transaction(tenant_id) as cursor: + assert cursor is not None + cursor.execute("SELECT 1") + result = cursor.fetchone() + assert result[0] == 1 diff --git a/api/src/backend/api/tests/test_utils.py b/api/src/backend/api/tests/test_utils.py index 0b36cdf9fe..d67df93ff5 100644 --- a/api/src/backend/api/tests/test_utils.py +++ b/api/src/backend/api/tests/test_utils.py @@ -20,8 +20,11 @@ from prowler.providers.aws.aws_provider import AwsProvider from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection from prowler.providers.azure.azure_provider import AzureProvider from prowler.providers.gcp.gcp_provider import GcpProvider +from prowler.providers.github.github_provider import GithubProvider from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider from prowler.providers.m365.m365_provider import M365Provider +from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider +from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider class TestMergeDicts: @@ -108,6 +111,9 @@ class TestReturnProwlerProvider: (Provider.ProviderChoices.AZURE.value, AzureProvider), (Provider.ProviderChoices.KUBERNETES.value, KubernetesProvider), (Provider.ProviderChoices.M365.value, M365Provider), + (Provider.ProviderChoices.GITHUB.value, GithubProvider), + (Provider.ProviderChoices.MONGODBATLAS.value, MongodbatlasProvider), + (Provider.ProviderChoices.ORACLECLOUD.value, OraclecloudProvider), ], ) def test_return_prowler_provider(self, provider_type, expected_provider): @@ -203,6 +209,14 @@ class TestGetProwlerProviderKwargs: Provider.ProviderChoices.GITHUB.value, {"organizations": ["provider_uid"]}, ), + ( + Provider.ProviderChoices.ORACLECLOUD.value, + {}, + ), + ( + Provider.ProviderChoices.MONGODBATLAS.value, + {"atlas_organization_id": "provider_uid"}, + ), ], ) def test_get_prowler_provider_kwargs(self, provider_type, expected_extra_kwargs): diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index cfad06c777..fc433e1e95 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -4,6 +4,7 @@ import json import os import tempfile from datetime import datetime, timedelta, timezone +from decimal import Decimal from pathlib import Path from types import SimpleNamespace from unittest.mock import ANY, MagicMock, Mock, patch @@ -23,6 +24,7 @@ from conftest import ( today_after_n_days, ) from django.conf import settings +from django.db.models import Count from django.http import JsonResponse from django.test import RequestFactory from django.urls import reverse @@ -33,28 +35,37 @@ from rest_framework.response import Response from api.compliance import get_compliance_frameworks from api.db_router import MainRouter from api.models import ( + Finding, Integration, Invitation, + LighthouseProviderConfiguration, + LighthouseProviderModels, + LighthouseTenantConfiguration, Membership, Processor, Provider, ProviderGroup, ProviderGroupMembership, ProviderSecret, + Resource, Role, RoleProviderGroupRelationship, SAMLConfiguration, SAMLToken, Scan, + ScanSummary, StateChoices, Task, TenantAPIKey, + ThreatScoreSnapshot, User, UserRoleRelationship, ) from api.rls import Tenant from api.v1.serializers import TokenSerializer from api.v1.views import ComplianceOverviewViewSet, TenantFinishACSView +from prowler.lib.check.models import Severity +from prowler.lib.outputs.finding import Status class TestViewSet: @@ -942,6 +953,74 @@ class TestProviderViewSet: assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == len(providers_fixture) + def test_providers_filter_provider_type( + self, authenticated_client, providers_fixture + ): + response = authenticated_client.get( + reverse("provider-list"), {"filter[provider_type]": "aws"} + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + assert all(item["attributes"]["provider"] == "aws" for item in data) + + def test_providers_filter_provider_type_in( + self, authenticated_client, providers_fixture + ): + response = authenticated_client.get( + reverse("provider-list"), {"filter[provider_type__in]": "aws,gcp"} + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 3 + assert {"aws", "gcp"} >= {item["attributes"]["provider"] for item in data} + + def test_providers_filter_provider_type_invalid( + self, authenticated_client, providers_fixture + ): + response = authenticated_client.get( + reverse("provider-list"), {"filter[provider_type]": "invalid"} + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_providers_disable_pagination( + self, authenticated_client, providers_fixture, tenants_fixture + ): + tenant, *_ = tenants_fixture + existing_count = Provider.objects.filter(tenant_id=tenant.id).count() + target_total = settings.REST_FRAMEWORK["PAGE_SIZE"] + 1 + additional_needed = max(0, target_total - existing_count) + + base_uid = 200000000000 + for index in range(additional_needed): + Provider.objects.create( + tenant_id=tenant.id, + provider=Provider.ProviderChoices.AWS, + uid=f"{base_uid + index:012d}", + alias=f"aws_extra_{index}", + ) + + total_providers = Provider.objects.filter(tenant_id=tenant.id).count() + + paginated_response = authenticated_client.get(reverse("provider-list")) + assert paginated_response.status_code == status.HTTP_200_OK + paginated_data = paginated_response.json()["data"] + assert len(paginated_data) == min( + settings.REST_FRAMEWORK["PAGE_SIZE"], total_providers + ) + paginated_meta = paginated_response.json().get("meta", {}) + assert "pagination" in paginated_meta + assert paginated_meta["pagination"]["count"] == total_providers + + unpaginated_response = authenticated_client.get( + reverse("provider-list"), {"page[disable]": "true"} + ) + assert unpaginated_response.status_code == status.HTTP_200_OK + unpaginated_data = unpaginated_response.json()["data"] + assert len(unpaginated_data) == total_providers + unpaginated_meta = unpaginated_response.json().get("meta", {}) + assert "pagination" not in unpaginated_meta + @pytest.mark.parametrize( "include_values, expected_resources", [ @@ -1066,6 +1145,21 @@ class TestProviderViewSet: "uid": "a12345678901234567890123456789012345678", "alias": "Long Username", }, + { + "provider": "iac", + "uid": "https://github.com/user/repo.git", + "alias": "Git Repo", + }, + { + "provider": "iac", + "uid": "https://gitlab.com/user/project", + "alias": "GitLab Repo", + }, + { + "provider": "mongodbatlas", + "uid": "64b1d3c0e4b03b1234567890", + "alias": "Atlas Organization", + }, ] ), ) @@ -1079,6 +1173,161 @@ class TestProviderViewSet: assert Provider.objects.get().uid == provider_json_payload["uid"] assert Provider.objects.get().alias == provider_json_payload["alias"] + @pytest.mark.parametrize( + "provider_json_payload", + ( + [ + {"provider": "aws", "uid": "111111111111", "alias": "test"}, + {"provider": "gcp", "uid": "a12322-test54321", "alias": "test"}, + { + "provider": "kubernetes", + "uid": "kubernetes-test-123456789", + "alias": "test", + }, + { + "provider": "kubernetes", + "uid": "arn:aws:eks:us-east-1:111122223333:cluster/test-cluster-long-name-123456789", + "alias": "EKS", + }, + { + "provider": "kubernetes", + "uid": "gke_aaaa-dev_europe-test1_dev-aaaa-test-cluster-long-name-123456789", + "alias": "GKE", + }, + { + "provider": "kubernetes", + "uid": "gke_project/cluster-name", + "alias": "GKE", + }, + { + "provider": "kubernetes", + "uid": "admin@k8s-demo", + "alias": "test", + }, + { + "provider": "azure", + "uid": "8851db6b-42e5-4533-aa9e-30a32d67e875", + "alias": "test", + }, + { + "provider": "m365", + "uid": "TestingPro.onmicrosoft.com", + "alias": "test", + }, + { + "provider": "m365", + "uid": "subdomain.domain.es", + "alias": "test", + }, + { + "provider": "m365", + "uid": "microsoft.net", + "alias": "test", + }, + { + "provider": "m365", + "uid": "subdomain1.subdomain2.subdomain3.subdomain4.domain.net", + "alias": "test", + }, + { + "provider": "github", + "uid": "test-user", + "alias": "test", + }, + { + "provider": "github", + "uid": "test-organization", + "alias": "GitHub Org", + }, + { + "provider": "github", + "uid": "prowler-cloud", + "alias": "Prowler", + }, + { + "provider": "github", + "uid": "microsoft", + "alias": "Microsoft", + }, + { + "provider": "github", + "uid": "a12345678901234567890123456789012345678", + "alias": "Long Username", + }, + ] + ), + ) + @patch("api.v1.views.Task.objects.get") + @patch("api.v1.views.delete_provider_task.delay") + def test_providers_soft_delete( + self, + mock_delete_task, + mock_task_get, + authenticated_client, + provider_json_payload, + tasks_fixture, + ): + # Mock the Celery task response + prowler_task = tasks_fixture[0] + task_mock = Mock() + task_mock.id = prowler_task.id + mock_delete_task.return_value = task_mock + mock_task_get.return_value = prowler_task + + # 1.Create a provider + response = authenticated_client.post( + reverse("provider-list"), data=provider_json_payload, format="json" + ) + assert response.status_code == status.HTTP_201_CREATED + assert Provider.objects.count() == 1 + provider_id = response.json()["data"]["id"] + + # 2. Soft delete the provider using the actual API endpoint + response = authenticated_client.delete( + reverse("provider-detail", kwargs={"pk": provider_id}) + ) + assert response.status_code == status.HTTP_202_ACCEPTED + assert Provider.objects.count() == 0 + assert Provider.all_objects.count() == 1 + + mock_delete_task.assert_called_once_with( + provider_id=str(provider_id), tenant_id=ANY + ) + + # 3. Create a provider with the same UID should succeed (since the old one is soft deleted) + response = authenticated_client.post( + reverse("provider-list"), data=provider_json_payload, format="json" + ) + assert response.status_code == status.HTTP_201_CREATED + assert Provider.objects.count() == 1 + assert Provider.all_objects.count() == 2 + provider_id = response.json()["data"]["id"] + + # 4. Creating another provider with the same UID should fail (duplicate) + response = authenticated_client.post( + reverse("provider-list"), data=provider_json_payload, format="json" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + mock_delete_task.reset_mock() + mock_delete_task.return_value = task_mock + + # 5. Delete the second provider + response = authenticated_client.delete( + reverse("provider-detail", kwargs={"pk": provider_id}) + ) + assert response.status_code == status.HTTP_202_ACCEPTED + assert Provider.objects.count() == 0 + assert Provider.all_objects.count() == 2 + + # 6. Creating a provider with the same UID should succeed again + response = authenticated_client.post( + reverse("provider-list"), data=provider_json_payload, format="json" + ) + assert response.status_code == status.HTTP_201_CREATED + assert Provider.objects.count() == 1 + assert Provider.all_objects.count() == 3 + @pytest.mark.parametrize( "provider_json_payload, error_code, error_pointer", ( @@ -1215,6 +1464,51 @@ class TestProviderViewSet: "github-uid", "uid", ), + ( + { + "provider": "iac", + "uid": "not-a-url", + "alias": "test", + }, + "iac-uid", + "uid", + ), + ( + { + "provider": "iac", + "uid": "ftp://invalid-protocol.com/repo", + "alias": "test", + }, + "iac-uid", + "uid", + ), + ( + { + "provider": "iac", + "uid": "http://", + "alias": "test", + }, + "iac-uid", + "uid", + ), + ( + { + "provider": "mongodbatlas", + "uid": "64b1d3c0e4b03b123456789g", + "alias": "test", + }, + "mongodbatlas-uid", + "uid", + ), + ( + { + "provider": "mongodbatlas", + "uid": "1234", + "alias": "test", + }, + "mongodbatlas-uid", + "uid", + ), ] ), ) @@ -1385,13 +1679,25 @@ class TestProviderViewSet: ("provider", "aws", 2), ("provider.in", "azure,gcp", 2), ("uid", "123456789012", 1), - ("uid.icontains", "1", 5), + ( + "uid.icontains", + "1", + 7, + ), ("alias", "aws_testing_1", 1), ("alias.icontains", "aws", 2), - ("inserted_at", TODAY, 6), - ("inserted_at.gte", "2024-01-01", 6), + ("inserted_at", TODAY, 8), + ( + "inserted_at.gte", + "2024-01-01", + 8, + ), ("inserted_at.lte", "2024-01-01", 0), - ("updated_at.gte", "2024-01-01", 6), + ( + "updated_at.gte", + "2024-01-01", + 8, + ), ("updated_at.lte", "2024-01-01", 0), ] ), @@ -1870,39 +2176,7 @@ class TestProviderSecretViewSet: "kubeconfig_content": "kubeconfig-content", }, ), - # M365 with STATIC secret - no user or password - ( - Provider.ProviderChoices.M365.value, - ProviderSecret.TypeChoices.STATIC, - { - "client_id": "client-id", - "client_secret": "client-secret", - "tenant_id": "tenant-id", - }, - ), - # M365 with user only - ( - Provider.ProviderChoices.M365.value, - ProviderSecret.TypeChoices.STATIC, - { - "client_id": "client-id", - "client_secret": "client-secret", - "tenant_id": "tenant-id", - "user": "test@domain.com", - }, - ), - # M365 with password only - ( - Provider.ProviderChoices.M365.value, - ProviderSecret.TypeChoices.STATIC, - { - "client_id": "client-id", - "client_secret": "client-secret", - "tenant_id": "tenant-id", - "password": "supersecret", - }, - ), - # M365 with user and password + # M365 client secret credentials ( Provider.ProviderChoices.M365.value, ProviderSecret.TypeChoices.STATIC, @@ -1914,6 +2188,64 @@ class TestProviderSecretViewSet: "password": "supersecret", }, ), + # M365 certificate credentials (valid base64) + ( + Provider.ProviderChoices.M365.value, + ProviderSecret.TypeChoices.STATIC, + { + "client_id": "client-id", + "tenant_id": "tenant-id", + "certificate_content": "VGVzdCBjZXJ0aWZpY2F0ZSBjb250ZW50", + "user": "test@domain.com", + "password": "supersecret", + }, + ), + # OCI with API key credentials (with key_content) + ( + Provider.ProviderChoices.ORACLECLOUD.value, + ProviderSecret.TypeChoices.STATIC, + { + "user": "ocid1.user.oc1..aaaaaaaakldibrbov4ubh25aqdeiroklxjngwka7u6w7no3glmdq3n5sxtkq", + "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", + "key_content": "-----BEGIN RSA PRIVATE KEY-----\ntest-key-content\n-----END RSA PRIVATE KEY-----", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", + "region": "us-ashburn-1", + }, + ), + # OCI with API key credentials (with key_file) + ( + Provider.ProviderChoices.ORACLECLOUD.value, + ProviderSecret.TypeChoices.STATIC, + { + "user": "ocid1.user.oc1..aaaaaaaakldibrbov4ubh25aqdeiroklxjngwka7u6w7no3glmdq3n5sxtkq", + "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", + "key_file": "/path/to/oci_api_key.pem", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", + "region": "us-ashburn-1", + }, + ), + # OCI with API key credentials (with passphrase) + ( + Provider.ProviderChoices.ORACLECLOUD.value, + ProviderSecret.TypeChoices.STATIC, + { + "user": "ocid1.user.oc1..aaaaaaaakldibrbov4ubh25aqdeiroklxjngwka7u6w7no3glmdq3n5sxtkq", + "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", + "key_content": "-----BEGIN RSA PRIVATE KEY-----\ntest-encrypted-key\n-----END RSA PRIVATE KEY-----", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", + "region": "us-ashburn-1", + "pass_phrase": "my-secure-passphrase", + }, + ), + # MongoDB Atlas credentials + ( + Provider.ProviderChoices.MONGODBATLAS.value, + ProviderSecret.TypeChoices.STATIC, + { + "atlas_public_key": "public-key", + "atlas_private_key": "private-key", + }, + ), ], ) def test_provider_secrets_create_valid( @@ -2279,6 +2611,50 @@ class TestProviderSecretViewSet: ) assert response.status_code == status.HTTP_400_BAD_REQUEST + def test_m365_provider_secrets_invalid_certificate_base64( + self, authenticated_client, providers_fixture + ): + """Test M365 provider secret creation with invalid base64 certificate content""" + # Find M365 provider from fixture + m365_provider = None + for provider in providers_fixture: + if provider.provider == Provider.ProviderChoices.M365.value: + m365_provider = provider + break + + assert m365_provider is not None, "M365 provider not found in fixture" + + data = { + "data": { + "type": "provider-secrets", + "attributes": { + "name": "M365 Certificate Invalid Base64", + "secret_type": "static", + "secret": { + "client_id": "client-id", + "tenant_id": "tenant-id", + "certificate_content": "invalid-base64-content!@#$%", + "user": "test@domain.com", + "password": "supersecret", + }, + }, + "relationships": { + "provider": { + "data": {"type": "providers", "id": str(m365_provider.id)} + } + }, + } + } + response = authenticated_client.post( + reverse("providersecret-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "certificate content is not valid base64 encoded data" in str( + response.json() + ) + @pytest.mark.django_db class TestScanViewSet: @@ -2664,6 +3040,55 @@ class TestScanViewSet: == "There is a problem with credentials." ) + @patch("api.v1.views.ScanViewSet._get_task_status") + @patch("api.v1.views.get_s3_client") + @patch("api.v1.views.env.str") + def test_threatscore_s3_wildcard( + self, + mock_env_str, + mock_get_s3_client, + mock_get_task_status, + authenticated_client, + scans_fixture, + ): + """ + When the threatscore endpoint is called with an S3 output_location, + the view should list objects in S3 using wildcard pattern matching, + retrieve the matching PDF file, and return it with HTTP 200 and proper headers. + """ + scan = scans_fixture[0] + scan.state = StateChoices.COMPLETED + bucket = "test-bucket" + zip_key = "tenant-id/scan-id/prowler-output-foo.zip" + scan.output_location = f"s3://{bucket}/{zip_key}" + scan.save() + + pdf_key = os.path.join( + os.path.dirname(zip_key), + "threatscore", + "prowler-output-123_threatscore_report.pdf", + ) + + mock_s3_client = Mock() + mock_s3_client.list_objects_v2.return_value = {"Contents": [{"Key": pdf_key}]} + mock_s3_client.get_object.return_value = {"Body": io.BytesIO(b"pdf-bytes")} + + mock_env_str.return_value = bucket + mock_get_s3_client.return_value = mock_s3_client + mock_get_task_status.return_value = None + + url = reverse("scan-threatscore", kwargs={"pk": scan.id}) + response = authenticated_client.get(url) + + assert response.status_code == status.HTTP_200_OK + assert response["Content-Type"] == "application/pdf" + assert response["Content-Disposition"].endswith( + '"prowler-output-123_threatscore_report.pdf"' + ) + assert response.content == b"pdf-bytes" + mock_s3_client.list_objects_v2.assert_called_once() + mock_s3_client.get_object.assert_called_once_with(Bucket=bucket, Key=pdf_key) + def test_report_s3_success(self, authenticated_client, scans_fixture, monkeypatch): """ When output_location is an S3 URL and the S3 client returns the file successfully, @@ -5707,8 +6132,497 @@ class TestOverviewViewSet: assert response.json()["data"][0]["attributes"]["findings"]["pass"] == 2 assert response.json()["data"][0]["attributes"]["findings"]["fail"] == 1 assert response.json()["data"][0]["attributes"]["findings"]["muted"] == 1 - # Since we rely on completed scans, there are only 2 resources now - assert response.json()["data"][0]["attributes"]["resources"]["total"] == 2 + # Aggregated resources include all AWS providers present in the tenant + assert response.json()["data"][0]["attributes"]["resources"]["total"] == 3 + + def test_overview_providers_aggregates_same_provider_type( + self, + authenticated_client, + scan_summaries_fixture, + resources_fixture, + providers_fixture, + tenants_fixture, + ): + tenant = tenants_fixture[0] + _provider1, provider2, *_ = providers_fixture + + scan = Scan.objects.create( + name="overview scan aws account 2", + provider=provider2, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + ScanSummary.objects.create( + tenant=tenant, + scan=scan, + check_id="check-aws-two", + service="service-extra", + severity="medium", + region="region-extra", + _pass=3, + fail=2, + muted=1, + total=6, + ) + + Resource.objects.create( + tenant_id=tenant.id, + provider=provider2, + uid="arn:aws:ec2:us-west-2:123456789013:instance/i-aggregation", + name="Aggregated Instance", + region="us-west-2", + service="ec2", + type="prowler-test", + ) + + response = authenticated_client.get(reverse("overview-providers")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + attributes = data[0]["attributes"] + + assert attributes["findings"]["total"] == 10 + assert attributes["findings"]["pass"] == 5 + assert attributes["findings"]["fail"] == 3 + assert attributes["findings"]["muted"] == 2 + assert attributes["resources"]["total"] == 4 + + def test_overview_providers_count( + self, + authenticated_client, + scan_summaries_fixture, + resources_fixture, + providers_fixture, + tenants_fixture, + ): + tenant = tenants_fixture[0] + + default_response = authenticated_client.get(reverse("overview-providers")) + assert default_response.status_code == status.HTTP_200_OK + default_data = default_response.json()["data"] + assert len(default_data) == 1 + assert all("count" not in item["attributes"] for item in default_data) + grouped_response = authenticated_client.get(reverse("overview-providers-count")) + assert grouped_response.status_code == status.HTTP_200_OK + grouped_data = grouped_response.json()["data"] + assert len(grouped_data) >= 1 + + aggregated = { + entry["id"]: entry["attributes"]["count"] for entry in grouped_data + } + db_counts = ( + Provider.objects.filter(tenant_id=tenant.id, is_deleted=False) + .values("provider") + .annotate(count=Count("id")) + ) + expected = {row["provider"]: row["count"] for row in db_counts} + + assert aggregated == expected + for entry in grouped_data: + assert "findings" not in entry["attributes"] + + def _create_scan(self, tenant, provider, name, started_at=None): + scan_started = started_at or datetime.now(timezone.utc) - timedelta(hours=1) + return Scan.objects.create( + tenant=tenant, + provider=provider, + name=name, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + started_at=scan_started, + completed_at=scan_started + timedelta(minutes=30), + ) + + def _create_threatscore_snapshot( + self, + tenant, + scan, + provider, + *, + compliance_id, + overall_score, + score_delta, + section_scores, + critical_requirements, + total_requirements, + passed_requirements, + failed_requirements, + manual_requirements, + total_findings, + passed_findings, + failed_findings, + ): + return ThreatScoreSnapshot.objects.create( + tenant=tenant, + scan=scan, + provider=provider, + compliance_id=compliance_id, + overall_score=Decimal(overall_score), + score_delta=Decimal(score_delta) if score_delta is not None else None, + section_scores=section_scores, + critical_requirements=critical_requirements, + total_requirements=total_requirements, + passed_requirements=passed_requirements, + failed_requirements=failed_requirements, + manual_requirements=manual_requirements, + total_findings=total_findings, + passed_findings=passed_findings, + failed_findings=failed_findings, + ) + + def test_overview_threatscore_returns_weighted_aggregate_snapshot( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = self._create_scan(tenant, provider1, "agg-scan-one") + scan2 = self._create_scan(tenant, provider2, "agg-scan-two") + + snapshot1 = self._create_threatscore_snapshot( + tenant, + scan1, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="80.00", + score_delta="5.00", + section_scores={"1. IAM": "70.00", "2. Attack Surface": "60.00"}, + critical_requirements=[ + { + "requirement_id": "req_shared", + "title": "Shared requirement (preferred)", + "section": "1. IAM", + "subsection": "Sub IAM", + "risk_level": 5, + "weight": 150, + "passed_findings": 14, + "total_findings": 20, + "description": "Higher risk duplicate", + }, + { + "requirement_id": "req_unique_one", + "title": "Unique provider one", + "section": "2. Attack Surface", + "subsection": "Sub Attack", + "risk_level": 4, + "weight": 90, + "passed_findings": 20, + "total_findings": 30, + "description": "Lower risk", + }, + ], + total_requirements=120, + passed_requirements=90, + failed_requirements=30, + manual_requirements=0, + total_findings=100, + passed_findings=70, + failed_findings=30, + ) + + snapshot2 = self._create_threatscore_snapshot( + tenant, + scan2, + provider2, + compliance_id="prowler_threatscore_aws", + overall_score="20.00", + score_delta="-2.00", + section_scores={ + "1. IAM": "10.00", + "2. Attack Surface": "40.00", + "3. Logging": "30.00", + }, + critical_requirements=[ + { + "requirement_id": "req_shared", + "title": "Shared requirement (secondary)", + "section": "1. IAM", + "subsection": "Sub IAM", + "risk_level": 4, + "weight": 120, + "passed_findings": 8, + "total_findings": 12, + "description": "Lower risk duplicate", + }, + { + "requirement_id": "req_unique_two", + "title": "Unique provider two", + "section": "3. Logging", + "subsection": "Sub Logging", + "risk_level": 5, + "weight": 110, + "passed_findings": 6, + "total_findings": 10, + "description": "Another critical requirement", + }, + ], + total_requirements=80, + passed_requirements=30, + failed_requirements=50, + manual_requirements=0, + total_findings=50, + passed_findings=15, + failed_findings=35, + ) + + older_inserted = datetime(2025, 1, 1, 12, 0, tzinfo=timezone.utc) + newer_inserted = datetime(2025, 1, 2, 12, 0, tzinfo=timezone.utc) + ThreatScoreSnapshot.objects.filter(id=snapshot1.id).update( + inserted_at=older_inserted + ) + ThreatScoreSnapshot.objects.filter(id=snapshot2.id).update( + inserted_at=newer_inserted + ) + snapshot2.refresh_from_db() + + response = authenticated_client.get(reverse("overview-threatscore")) + + assert response.status_code == status.HTTP_200_OK + body = response.json() + assert len(body["data"]) == 1 + aggregated = body["data"][0] + + assert aggregated["id"] == "n/a" + assert aggregated["relationships"]["scan"]["data"] is None + assert aggregated["relationships"]["provider"]["data"] is None + + attrs = aggregated["attributes"] + assert Decimal(attrs["overall_score"]) == Decimal("60.00") + assert Decimal(attrs["score_delta"]) == Decimal("2.67") + assert attrs["inserted_at"] == snapshot2.inserted_at.isoformat().replace( + "+00:00", "Z" + ) + assert attrs["total_findings"] == 150 + assert attrs["passed_findings"] == 85 + assert attrs["failed_findings"] == 65 + assert attrs["total_requirements"] == 200 + assert attrs["passed_requirements"] == 120 + assert attrs["failed_requirements"] == 80 + assert attrs["manual_requirements"] == 0 + + assert attrs["section_scores"] == { + "1. IAM": "50.00", + "2. Attack Surface": "53.33", + "3. Logging": "30.00", + } + + expected_critical = [ + { + "requirement_id": "req_shared", + "title": "Shared requirement (preferred)", + "section": "1. IAM", + "subsection": "Sub IAM", + "risk_level": 5, + "weight": 150, + "passed_findings": 14, + "total_findings": 20, + "description": "Higher risk duplicate", + }, + { + "requirement_id": "req_unique_two", + "title": "Unique provider two", + "section": "3. Logging", + "subsection": "Sub Logging", + "risk_level": 5, + "weight": 110, + "passed_findings": 6, + "total_findings": 10, + "description": "Another critical requirement", + }, + { + "requirement_id": "req_unique_one", + "title": "Unique provider one", + "section": "2. Attack Surface", + "subsection": "Sub Attack", + "risk_level": 4, + "weight": 90, + "passed_findings": 20, + "total_findings": 30, + "description": "Lower risk", + }, + ] + assert attrs["critical_requirements"] == expected_critical + + def test_overview_threatscore_weight_fallback_to_requirements( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = self._create_scan(tenant, provider1, "fallback-scan-1") + scan2 = self._create_scan(tenant, provider2, "fallback-scan-2") + + self._create_threatscore_snapshot( + tenant, + scan1, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="90.00", + score_delta="4.00", + section_scores={"1. IAM": "90.00"}, + critical_requirements=[], + total_requirements=10, + passed_requirements=8, + failed_requirements=0, + manual_requirements=2, + total_findings=0, + passed_findings=0, + failed_findings=0, + ) + self._create_threatscore_snapshot( + tenant, + scan2, + provider2, + compliance_id="prowler_threatscore_aws", + overall_score="50.00", + score_delta="1.00", + section_scores={"1. IAM": "40.00"}, + critical_requirements=[], + total_requirements=12, + passed_requirements=5, + failed_requirements=7, + manual_requirements=0, + total_findings=10, + passed_findings=4, + failed_findings=6, + ) + + response = authenticated_client.get(reverse("overview-threatscore")) + assert response.status_code == status.HTTP_200_OK + aggregate = response.json()["data"][0]["attributes"] + + assert Decimal(aggregate["overall_score"]) == Decimal("67.78") + assert Decimal(aggregate["score_delta"]) == Decimal("2.33") + assert aggregate["total_findings"] == 10 + assert aggregate["total_requirements"] == 22 + assert aggregate["manual_requirements"] == 2 + assert aggregate["section_scores"] == {"1. IAM": "62.22"} + + def test_overview_threatscore_filter_by_scan_id_returns_snapshot( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, *_ = providers_fixture + scan = self._create_scan(tenant, provider1, "filter-scan") + + snapshot = self._create_threatscore_snapshot( + tenant, + scan, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="75.00", + score_delta="3.00", + section_scores={"1. IAM": "70.00"}, + critical_requirements=[], + total_requirements=50, + passed_requirements=30, + failed_requirements=20, + manual_requirements=0, + total_findings=25, + passed_findings=15, + failed_findings=10, + ) + + response = authenticated_client.get( + reverse("overview-threatscore"), {"filter[scan_id]": str(scan.id)} + ) + + assert response.status_code == status.HTTP_200_OK + body = response.json() + assert len(body["data"]) == 1 + assert body["data"][0]["id"] == str(snapshot.id) + assert body["data"][0]["attributes"]["overall_score"] == "75.00" + + def test_overview_threatscore_snapshot_id_returns_specific_snapshot( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, *_ = providers_fixture + scan = self._create_scan(tenant, provider1, "snapshot-id-scan") + + snapshot = self._create_threatscore_snapshot( + tenant, + scan, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="88.50", + score_delta=None, + section_scores={"1. IAM": "80.00"}, + critical_requirements=[], + total_requirements=60, + passed_requirements=45, + failed_requirements=15, + manual_requirements=0, + total_findings=30, + passed_findings=25, + failed_findings=5, + ) + + response = authenticated_client.get( + reverse("overview-threatscore"), {"snapshot_id": str(snapshot.id)} + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["data"]["id"] == str(snapshot.id) + assert data["data"]["attributes"]["score_delta"] is None + + def test_overview_threatscore_provider_filter_returns_unaggregated_snapshot( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = self._create_scan(tenant, provider1, "provider-filter-scan-1") + scan2 = self._create_scan(tenant, provider2, "provider-filter-scan-2") + + snapshot1 = self._create_threatscore_snapshot( + tenant, + scan1, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="55.55", + score_delta="1.10", + section_scores={"1. IAM": "50.00"}, + critical_requirements=[], + total_requirements=40, + passed_requirements=25, + failed_requirements=15, + manual_requirements=0, + total_findings=12, + passed_findings=7, + failed_findings=5, + ) + self._create_threatscore_snapshot( + tenant, + scan2, + provider2, + compliance_id="prowler_threatscore_aws", + overall_score="44.44", + score_delta="0.80", + section_scores={"1. IAM": "40.00"}, + critical_requirements=[], + total_requirements=30, + passed_requirements=18, + failed_requirements=12, + manual_requirements=0, + total_findings=10, + passed_findings=6, + failed_findings=4, + ) + + response = authenticated_client.get( + reverse("overview-threatscore"), + {"filter[provider_id__in]": str(provider1.id)}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["id"] == str(snapshot1.id) + assert data[0]["attributes"]["overall_score"] == "55.55" def test_overview_services_list_no_required_filters( self, authenticated_client, scan_summaries_fixture @@ -5742,6 +6656,171 @@ class TestOverviewViewSet: assert service1_data["attributes"]["muted"] == 1 assert service2_data["attributes"]["muted"] == 0 + def test_overview_findings_provider_id_in_filter( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = Scan.objects.create( + name="scan-one", + provider=provider1, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + scan2 = Scan.objects.create( + name="scan-two", + provider=provider2, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + ScanSummary.objects.create( + tenant=tenant, + scan=scan1, + check_id="check-provider-one", + service="service-a", + severity="high", + region="region-a", + _pass=5, + fail=1, + muted=2, + total=8, + new=5, + changed=2, + unchanged=1, + fail_new=1, + fail_changed=0, + pass_new=3, + pass_changed=2, + muted_new=1, + muted_changed=1, + ) + + ScanSummary.objects.create( + tenant=tenant, + scan=scan2, + check_id="check-provider-two", + service="service-b", + severity="medium", + region="region-b", + _pass=2, + fail=3, + muted=1, + total=6, + new=3, + changed=2, + unchanged=1, + fail_new=2, + fail_changed=1, + pass_new=1, + pass_changed=1, + muted_new=1, + muted_changed=0, + ) + + single_response = authenticated_client.get( + reverse("overview-findings"), + {"filter[provider_id__in]": str(provider1.id)}, + ) + assert single_response.status_code == status.HTTP_200_OK + single_attributes = single_response.json()["data"]["attributes"] + assert single_attributes["pass"] == 5 + assert single_attributes["fail"] == 1 + assert single_attributes["muted"] == 2 + assert single_attributes["total"] == 8 + + combined_response = authenticated_client.get( + reverse("overview-findings"), + {"filter[provider_id__in]": f"{provider1.id},{provider2.id}"}, + ) + assert combined_response.status_code == status.HTTP_200_OK + combined_attributes = combined_response.json()["data"]["attributes"] + assert combined_attributes["pass"] == 7 + assert combined_attributes["fail"] == 4 + assert combined_attributes["muted"] == 3 + assert combined_attributes["total"] == 14 + + def test_overview_findings_severity_provider_id_in_filter( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = Scan.objects.create( + name="severity-scan-one", + provider=provider1, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + scan2 = Scan.objects.create( + name="severity-scan-two", + provider=provider2, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + ScanSummary.objects.create( + tenant=tenant, + scan=scan1, + check_id="severity-check-one", + service="service-a", + severity="high", + region="region-a", + _pass=4, + fail=4, + muted=0, + total=8, + ) + ScanSummary.objects.create( + tenant=tenant, + scan=scan1, + check_id="severity-check-two", + service="service-a", + severity="medium", + region="region-b", + _pass=2, + fail=2, + muted=0, + total=4, + ) + ScanSummary.objects.create( + tenant=tenant, + scan=scan2, + check_id="severity-check-three", + service="service-b", + severity="critical", + region="region-c", + _pass=1, + fail=2, + muted=0, + total=3, + ) + + single_response = authenticated_client.get( + reverse("overview-findings_severity"), + {"filter[provider_id__in]": str(provider1.id)}, + ) + assert single_response.status_code == status.HTTP_200_OK + single_attributes = single_response.json()["data"]["attributes"] + assert single_attributes["high"] == 8 + assert single_attributes["medium"] == 4 + assert single_attributes["critical"] == 0 + + combined_response = authenticated_client.get( + reverse("overview-findings_severity"), + {"filter[provider_id__in]": f"{provider1.id},{provider2.id}"}, + ) + assert combined_response.status_code == status.HTTP_200_OK + combined_attributes = combined_response.json()["data"]["attributes"] + assert combined_attributes["high"] == 8 + assert combined_attributes["medium"] == 4 + assert combined_attributes["critical"] == 3 + @pytest.mark.django_db class TestScheduleViewSet: @@ -5781,10 +6860,12 @@ class TestScheduleViewSet: ) assert response.status_code == status.HTTP_404_NOT_FOUND + @patch("tasks.beat.perform_scheduled_scan_task.apply_async") @patch("api.v1.views.Task.objects.get") def test_schedule_daily_already_scheduled( self, mock_task_get, + mock_apply_async, authenticated_client, providers_fixture, tasks_fixture, @@ -5792,6 +6873,7 @@ class TestScheduleViewSet: provider, *_ = providers_fixture prowler_task = tasks_fixture[0] mock_task_get.return_value = prowler_task + mock_apply_async.return_value.id = prowler_task.id json_payload = { "provider_id": str(provider.id), } @@ -6892,6 +7974,188 @@ class TestTenantFinishACSView: assert response.status_code == 302 assert "sso_saml_failed=true" in response.url + def test_dispatch_skips_role_mapping_when_single_manage_account_user( + self, create_test_user, tenants_fixture, saml_setup, settings, monkeypatch + ): + """Test that role mapping is skipped when tenant has only one user with MANAGE_ACCOUNT role""" + monkeypatch.setenv("SAML_SSO_CALLBACK_URL", "http://localhost/sso-complete") + user = create_test_user + tenant = tenants_fixture[0] + + # Create a single role with manage_account=True for the user + admin_role = Role.objects.using(MainRouter.admin_db).create( + name="admin", + tenant=tenant, + manage_account=True, + manage_users=True, + manage_billing=True, + manage_providers=True, + manage_integrations=True, + manage_scans=True, + unlimited_visibility=True, + ) + UserRoleRelationship.objects.using(MainRouter.admin_db).create( + user=user, role=admin_role, tenant_id=tenant.id + ) + + social_account = SocialAccount( + user=user, + provider="saml", + extra_data={ + "firstName": ["John"], + "lastName": ["Doe"], + "organization": ["testing_company"], + "userType": ["no_permissions"], # This should be ignored + }, + ) + + request = RequestFactory().get( + reverse("saml_finish_acs", kwargs={"organization_slug": "testtenant"}) + ) + request.user = user + request.session = {} + + with ( + patch( + "allauth.socialaccount.providers.saml.views.get_app_or_404" + ) as mock_get_app_or_404, + patch( + "allauth.socialaccount.models.SocialApp.objects.get" + ) as mock_socialapp_get, + patch( + "allauth.socialaccount.models.SocialAccount.objects.get" + ) as mock_sa_get, + patch("api.models.SAMLDomainIndex.objects.get") as mock_saml_domain_get, + patch("api.models.SAMLConfiguration.objects.get") as mock_saml_config_get, + patch("api.models.User.objects.get") as mock_user_get, + ): + mock_get_app_or_404.return_value = MagicMock( + provider="saml", client_id="testtenant", name="Test App", settings={} + ) + mock_sa_get.return_value = social_account + mock_socialapp_get.return_value = MagicMock(provider_id="saml") + mock_saml_domain_get.return_value = SimpleNamespace(tenant_id=tenant.id) + mock_saml_config_get.return_value = MagicMock() + mock_user_get.return_value = user + + view = TenantFinishACSView.as_view() + response = view(request, organization_slug="testtenant") + + assert response.status_code == 302 + + # Verify the admin role is still assigned (not changed to no_permissions) + assert ( + UserRoleRelationship.objects.using(MainRouter.admin_db) + .filter(user=user, role=admin_role, tenant_id=tenant.id) + .exists() + ) + + # Verify no_permissions role was NOT created in the database + assert ( + not Role.objects.using(MainRouter.admin_db) + .filter(name="no_permissions", tenant=tenant) + .exists() + ) + + # Verify no_permissions role was NOT assigned to the user + assert not ( + UserRoleRelationship.objects.using(MainRouter.admin_db) + .filter(user=user, role__name="no_permissions", tenant_id=tenant.id) + .exists() + ) + + def test_dispatch_applies_role_mapping_when_multiple_manage_account_users( + self, create_test_user, tenants_fixture, saml_setup, settings, monkeypatch + ): + """Test that role mapping is applied when tenant has multiple users with MANAGE_ACCOUNT role""" + monkeypatch.setenv("SAML_SSO_CALLBACK_URL", "http://localhost/sso-complete") + user = create_test_user + tenant = tenants_fixture[0] + + # Create a second user with manage_account=True + second_admin = User.objects.using(MainRouter.admin_db).create( + email="admin2@prowler.com", name="Second Admin" + ) + admin_role = Role.objects.using(MainRouter.admin_db).create( + name="admin", + tenant=tenant, + manage_account=True, + manage_users=True, + manage_billing=True, + manage_providers=True, + manage_integrations=True, + manage_scans=True, + unlimited_visibility=True, + ) + UserRoleRelationship.objects.using(MainRouter.admin_db).create( + user=user, role=admin_role, tenant_id=tenant.id + ) + UserRoleRelationship.objects.using(MainRouter.admin_db).create( + user=second_admin, role=admin_role, tenant_id=tenant.id + ) + + social_account = SocialAccount( + user=user, + provider="saml", + extra_data={ + "firstName": ["John"], + "lastName": ["Doe"], + "organization": ["testing_company"], + "userType": ["viewer"], # This SHOULD be applied + }, + ) + + request = RequestFactory().get( + reverse("saml_finish_acs", kwargs={"organization_slug": "testtenant"}) + ) + request.user = user + request.session = {} + + with ( + patch( + "allauth.socialaccount.providers.saml.views.get_app_or_404" + ) as mock_get_app_or_404, + patch( + "allauth.socialaccount.models.SocialApp.objects.get" + ) as mock_socialapp_get, + patch( + "allauth.socialaccount.models.SocialAccount.objects.get" + ) as mock_sa_get, + patch("api.models.SAMLDomainIndex.objects.get") as mock_saml_domain_get, + patch("api.models.SAMLConfiguration.objects.get") as mock_saml_config_get, + patch("api.models.User.objects.get") as mock_user_get, + ): + mock_get_app_or_404.return_value = MagicMock( + provider="saml", client_id="testtenant", name="Test App", settings={} + ) + mock_sa_get.return_value = social_account + mock_socialapp_get.return_value = MagicMock(provider_id="saml") + mock_saml_domain_get.return_value = SimpleNamespace(tenant_id=tenant.id) + mock_saml_config_get.return_value = MagicMock() + mock_user_get.return_value = user + + view = TenantFinishACSView.as_view() + response = view(request, organization_slug="testtenant") + + assert response.status_code == 302 + + # Verify the viewer role was created and assigned (role mapping was applied) + viewer_role = Role.objects.using(MainRouter.admin_db).get( + name="viewer", tenant=tenant + ) + assert ( + UserRoleRelationship.objects.using(MainRouter.admin_db) + .filter(user=user, role=viewer_role, tenant_id=tenant.id) + .exists() + ) + + # Verify the admin role was removed (replaced by viewer) + assert not ( + UserRoleRelationship.objects.using(MainRouter.admin_db) + .filter(user=user, role=admin_role, tenant_id=tenant.id) + .exists() + ) + @pytest.mark.django_db class TestLighthouseConfigViewSet: @@ -7530,8 +8794,6 @@ class TestTenantApiKeyViewSet: ( [ {"name": "New API Key"}, - {"name": ""}, - {}, ] ), ) @@ -7572,6 +8834,18 @@ class TestTenantApiKeyViewSet: {"name": "Invalid Expiry", "expires_at": "not-a-date"}, "expires_at", ), + ( + {"name": ""}, + "name", + ), + ( + {}, + "name", + ), + ( + {"name": "AB"}, # Too short (min length is 3) + "name", + ), ] ), ) @@ -7600,6 +8874,58 @@ class TestTenantApiKeyViewSet: == f"/data/attributes/{error_pointer}" ) + def test_api_keys_create_duplicate_name( + self, authenticated_client, api_keys_fixture + ): + """Test creating an API key with a duplicate name fails.""" + # Use the name of an existing API key + existing_name = api_keys_fixture[0].name + data = { + "data": { + "type": "api-keys", + "attributes": { + "name": existing_name, + }, + } + } + response = authenticated_client.post( + reverse("api-key-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + error_detail = response.json()["errors"][0]["detail"] + assert "already exists" in error_detail.lower() + + def test_api_keys_update_duplicate_name( + self, authenticated_client, api_keys_fixture + ): + """Test updating an API key with a duplicate name fails.""" + # Get two different API keys + first_api_key = api_keys_fixture[0] + second_api_key = api_keys_fixture[1] + + # Try to update the second API key to have the same name as the first one + data = { + "data": { + "type": "api-keys", + "id": str(second_api_key.id), + "attributes": { + "name": first_api_key.name, + }, + } + } + response = authenticated_client.patch( + reverse("api-key-detail", kwargs={"pk": second_api_key.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + error_detail = response.json()["errors"][0]["detail"] + assert "already exists" in error_detail.lower() + def test_api_keys_create_multiple_unique_prefixes( self, authenticated_client, api_keys_fixture ): @@ -7677,6 +9003,27 @@ class TestTenantApiKeyViewSet: api_key.refresh_from_db() assert api_key.revoked is True + def test_api_keys_revoke_preserves_created_field( + self, authenticated_client, api_keys_fixture + ): + """Test that revoking an API key preserves the created timestamp.""" + api_key = api_keys_fixture[0] # Not revoked + assert api_key.revoked is False + + # Record the original created timestamp + original_created = api_key.created + + response = authenticated_client.delete( + reverse("api-key-revoke", kwargs={"pk": api_key.id}) + ) + assert response.status_code == status.HTTP_200_OK + + # Verify in database + api_key.refresh_from_db() + assert api_key.revoked is True + # Verify created field has not changed + assert api_key.created == original_created + def test_api_keys_revoke_already_revoked( self, authenticated_client, api_keys_fixture ): @@ -8051,6 +9398,10 @@ class TestTenantApiKeyViewSet: assert included_user["type"] == "users" assert included_user["id"] == str(api_key.entity.id) + # Refresh entity from database to get current state + # (in case other tests modified the shared session-scoped user fixture) + api_key.entity.refresh_from_db() + # Verify UserIncludeSerializer fields are present user_attrs = included_user["attributes"] assert "name" in user_attrs @@ -8192,3 +9543,1414 @@ class TestTenantApiKeyViewSet: # Verify error object structure error = response_data["errors"][0] assert "detail" in error or "title" in error + + +@pytest.mark.django_db +class TestLighthouseTenantConfigViewSet: + """Test Lighthouse tenant configuration endpoint (singleton pattern)""" + + def test_lighthouse_tenant_config_create_via_patch(self, authenticated_client): + """Test creating a tenant config successfully via PATCH (upsert)""" + payload = { + "data": { + "type": "lighthouse-configurations", + "attributes": { + "business_context": "Test business context for security analysis", + "default_provider": "", + "default_models": {}, + }, + } + } + response = authenticated_client.patch( + reverse("lighthouse-configurations"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert ( + data["attributes"]["business_context"] + == "Test business context for security analysis" + ) + assert data["attributes"]["default_provider"] == "" + assert data["attributes"]["default_models"] == {} + + def test_lighthouse_tenant_config_upsert_behavior(self, authenticated_client): + """Test that PATCH creates config if not exists and updates if exists (upsert)""" + payload = { + "data": { + "type": "lighthouse-configurations", + "attributes": { + "business_context": "First config", + }, + } + } + + # First PATCH creates the config + response = authenticated_client.patch( + reverse("lighthouse-configurations"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert response.status_code == status.HTTP_200_OK + first_data = response.json()["data"] + assert first_data["attributes"]["business_context"] == "First config" + + # Second PATCH updates the same config (not creating a duplicate) + payload["data"]["attributes"]["business_context"] = "Updated config" + response = authenticated_client.patch( + reverse("lighthouse-configurations"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert response.status_code == status.HTTP_200_OK + second_data = response.json()["data"] + assert second_data["attributes"]["business_context"] == "Updated config" + # Verify it's the same config (same ID) + assert first_data["id"] == second_data["id"] + + @patch("openai.OpenAI") + def test_lighthouse_tenant_config_retrieve( + self, mock_openai_client, authenticated_client, tenants_fixture + ): + """Test retrieving the singleton tenant config with proper provider and model validation""" + + # Mock OpenAI client and models response + mock_models_response = Mock() + mock_models_response.data = [ + Mock(id="gpt-4o"), + Mock(id="gpt-4o-mini"), + Mock(id="gpt-5"), + ] + mock_openai_client.return_value.models.list.return_value = mock_models_response + + # Create OpenAI provider configuration + provider_config = LighthouseProviderConfiguration.objects.create( + tenant_id=tenants_fixture[0].id, + provider_type="openai", + credentials=b'{"api_key": "sk-test1234567890T3BlbkFJtest1234567890"}', + is_active=True, + ) + + # Create provider models (simulating refresh) + LighthouseProviderModels.objects.create( + tenant_id=tenants_fixture[0].id, + provider_configuration=provider_config, + model_id="gpt-4o", + default_parameters={}, + ) + LighthouseProviderModels.objects.create( + tenant_id=tenants_fixture[0].id, + provider_configuration=provider_config, + model_id="gpt-4o-mini", + default_parameters={}, + ) + + # Create tenant configuration with valid provider and model + config = LighthouseTenantConfiguration.objects.create( + tenant_id=tenants_fixture[0].id, + business_context="Test context", + default_provider="openai", + default_models={"openai": "gpt-4o"}, + ) + + # Retrieve and verify the configuration + response = authenticated_client.get(reverse("lighthouse-configurations")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data["id"] == str(config.id) + assert data["attributes"]["business_context"] == "Test context" + assert data["attributes"]["default_provider"] == "openai" + assert data["attributes"]["default_models"] == {"openai": "gpt-4o"} + + def test_lighthouse_tenant_config_retrieve_not_found(self, authenticated_client): + """Test GET when config doesn't exist returns 404""" + response = authenticated_client.get(reverse("lighthouse-configurations")) + assert response.status_code == status.HTTP_404_NOT_FOUND + assert "not found" in response.json()["errors"][0]["detail"].lower() + + def test_lighthouse_tenant_config_partial_update( + self, authenticated_client, tenants_fixture + ): + """Test updating tenant config fields""" + from api.models import LighthouseTenantConfiguration + + # Create config first + config = LighthouseTenantConfiguration.objects.create( + tenant_id=tenants_fixture[0].id, + business_context="Original context", + default_provider="", + default_models={}, + ) + + # Update it + payload = { + "data": { + "type": "lighthouse-configurations", + "attributes": { + "business_context": "Updated context for cloud security", + }, + } + } + response = authenticated_client.patch( + reverse("lighthouse-configurations"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert response.status_code == status.HTTP_200_OK + + # Verify update + config.refresh_from_db() + assert config.business_context == "Updated context for cloud security" + + def test_lighthouse_tenant_config_update_invalid_provider( + self, authenticated_client, tenants_fixture + ): + """Test validation fails when default_provider is not configured and active""" + from api.models import LighthouseTenantConfiguration + + # Create config first + LighthouseTenantConfiguration.objects.create( + tenant_id=tenants_fixture[0].id, + business_context="Test", + ) + + # Try to set invalid provider + payload = { + "data": { + "type": "lighthouse-configurations", + "attributes": { + "default_provider": "nonexistent-provider", + }, + } + } + response = authenticated_client.patch( + reverse("lighthouse-configurations"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "provider" in response.json()["errors"][0]["detail"].lower() + + def test_lighthouse_tenant_config_update_invalid_json_format( + self, authenticated_client, tenants_fixture + ): + """Test that invalid JSON payload is rejected""" + from api.models import LighthouseTenantConfiguration + + # Create config first + LighthouseTenantConfiguration.objects.create( + tenant_id=tenants_fixture[0].id, + business_context="Test", + ) + + # Send invalid JSON + response = authenticated_client.patch( + reverse("lighthouse-configurations"), + data="invalid json", + content_type=API_JSON_CONTENT_TYPE, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +@pytest.mark.django_db +class TestLighthouseProviderConfigViewSet: + """Tests for LighthouseProviderConfiguration create validations""" + + def test_invalid_provider_type(self, authenticated_client): + """Add invalid provider (testprovider) should error""" + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "testprovider", + "credentials": {"api_key": "sk-testT3BlbkFJkey"}, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + + def test_openai_missing_credentials(self, authenticated_client): + """OpenAI provider without credentials should error""" + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai", + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + + @pytest.mark.parametrize( + "credentials", + [ + {}, # empty credentials + {"token": "sk-testT3BlbkFJkey"}, # wrong key name + {"api_key": "ks-invalid-format"}, # wrong format + ], + ) + def test_openai_invalid_credentials(self, authenticated_client, credentials): + """OpenAI provider with invalid credentials should error""" + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai", + "credentials": credentials, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + + def test_openai_valid_credentials_success(self, authenticated_client): + """OpenAI provider with valid sk-xxx format should succeed""" + valid_key = "sk-abc123T3BlbkFJxyz456" + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai", + "credentials": {"api_key": valid_key}, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_201_CREATED + data = resp.json()["data"] + + masked_creds = data["attributes"].get("credentials") + assert masked_creds is not None + assert "api_key" in masked_creds + assert masked_creds["api_key"] == ("*" * len(valid_key)) + + def test_openai_provider_duplicate_per_tenant(self, authenticated_client): + """If an OpenAI provider exists for tenant, creating again should error""" + valid_key = "sk-dup123T3BlbkFJdup456" + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai", + "credentials": {"api_key": valid_key}, + }, + } + } + # First creation succeeds + resp1 = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp1.status_code == status.HTTP_201_CREATED + + # Second creation should fail with validation error + resp2 = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp2.status_code == status.HTTP_400_BAD_REQUEST + assert "already exists" in str(resp2.json()).lower() + + def test_openai_patch_base_url_and_is_active(self, authenticated_client): + """After creating, should be able to patch base_url and is_active""" + valid_key = "sk-patch123T3BlbkFJpatch456" + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai", + "credentials": {"api_key": valid_key}, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "base_url": "https://api.example.com/v1", + "is_active": False, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + updated = patch_resp.json()["data"]["attributes"] + assert updated["base_url"] == "https://api.example.com/v1" + assert updated["is_active"] is False + + def test_openai_patch_invalid_credentials(self, authenticated_client): + """PATCH with invalid credentials.api_key should error (400)""" + valid_key = "sk-ok123T3BlbkFJok456" + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai", + "credentials": {"api_key": valid_key}, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Try patch with invalid api_key format + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "credentials": {"api_key": "ks-invalid-format"}, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_400_BAD_REQUEST + + def test_openai_get_masking_and_fields_filter(self, authenticated_client): + valid_key = "sk-get123T3BlbkFJget456" + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai", + "credentials": {"api_key": valid_key}, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Default GET should return masked credentials + get_resp = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + ) + assert get_resp.status_code == status.HTTP_200_OK + masked = get_resp.json()["data"]["attributes"]["credentials"]["api_key"] + assert masked == ("*" * len(valid_key)) + + # Fields filter should return decrypted credentials structure + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + assert creds["api_key"] == valid_key + + def test_delete_provider_updates_tenant_defaults( + self, authenticated_client, tenants_fixture + ): + """Deleting a provider config should clear tenant default_provider and its default_model entry.""" + + tenant = tenants_fixture[0] + + # Create provider configuration to delete + provider = LighthouseProviderConfiguration.objects.create( + tenant_id=tenant.id, + provider_type="openai", + credentials=b'{"api_key":"sk-test123T3BlbkFJ"}', + is_active=True, + ) + + # Seed tenant defaults referencing the provider we will delete + cfg = LighthouseTenantConfiguration.objects.create( + tenant_id=tenant.id, + business_context="Test", + default_provider="openai", + default_models={"openai": "gpt-4o", "other": "model-x"}, + ) + + # Delete via API and validate response + url = reverse("lighthouse-providers-detail", kwargs={"pk": str(provider.id)}) + resp = authenticated_client.delete(url) + assert resp.status_code in ( + status.HTTP_204_NO_CONTENT, + status.HTTP_200_OK, + ) + + # Tenant defaults should be updated + cfg.refresh_from_db() + assert cfg.default_provider == "" + assert "openai" not in cfg.default_models + + # Unrelated entries should remain untouched + assert cfg.default_models.get("other") == "model-x" + + +@pytest.mark.django_db +class TestMuteRuleViewSet: + """Tests for MuteRule endpoints.""" + + def test_mute_rules_list(self, authenticated_client, mute_rules_fixture): + """Test listing all mute rules for the tenant.""" + response = authenticated_client.get(reverse("mute-rule-list")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == len(mute_rules_fixture) + + def test_mute_rules_list_empty(self, authenticated_client, tenants_fixture): + """Test listing mute rules when none exist returns empty list.""" + response = authenticated_client.get(reverse("mute-rule-list")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 0 + assert isinstance(data, list) + + def test_mute_rules_list_default_ordering( + self, authenticated_client, mute_rules_fixture + ): + """Test that mute rules are ordered by -inserted_at by default.""" + response = authenticated_client.get(reverse("mute-rule-list")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + + if len(data) >= 2: + first_date = data[0]["attributes"]["inserted_at"] + second_date = data[1]["attributes"]["inserted_at"] + assert first_date >= second_date + + def test_mute_rules_retrieve(self, authenticated_client, mute_rules_fixture): + """Test retrieving a single mute rule by ID.""" + mute_rule = mute_rules_fixture[0] + response = authenticated_client.get( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}) + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data["id"] == str(mute_rule.id) + assert data["attributes"]["name"] == mute_rule.name + assert data["attributes"]["reason"] == mute_rule.reason + assert data["attributes"]["enabled"] == mute_rule.enabled + assert "finding_uids" in data["attributes"] + assert "inserted_at" in data["attributes"] + assert "updated_at" in data["attributes"] + + def test_mute_rules_retrieve_invalid(self, authenticated_client): + """Test retrieving non-existent mute rule returns 404.""" + response = authenticated_client.get( + reverse( + "mute-rule-detail", + kwargs={"pk": "f498b103-c760-4785-9a3e-e23fafbb7b02"}, + ) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + @pytest.mark.parametrize( + "filter_name, filter_value, expected_count", + ( + [ + ("name", "Test Rule 1", 1), + ("name.icontains", "rule", 2), + ("reason.icontains", "security", 1), + ("enabled", True, 1), + ("enabled", False, 1), + ] + ), + ) + def test_mute_rule_filters( + self, + authenticated_client, + mute_rules_fixture, + filter_name, + filter_value, + expected_count, + ): + """Test filtering mute rules by various fields.""" + filters = {f"filter[{filter_name}]": filter_value} + response = authenticated_client.get(reverse("mute-rule-list"), filters) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == expected_count + + def test_mute_rule_filter_by_created_by( + self, authenticated_client, mute_rules_fixture, create_test_user + ): + """Test filtering mute rules by creator.""" + response = authenticated_client.get( + reverse("mute-rule-list"), + {"filter[created_by]": create_test_user.id}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + + def test_mute_rule_search(self, authenticated_client, mute_rules_fixture): + """Test searching mute rules by name and reason.""" + response = authenticated_client.get( + reverse("mute-rule-list"), {"filter[search]": "Rule 1"} + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 1 + + @pytest.mark.parametrize( + "sort_field, first_index", + ( + [ + ("name", 0), + ("-name", 1), + ("inserted_at", 0), + ("-inserted_at", 1), + ] + ), + ) + def test_mute_rule_ordering( + self, authenticated_client, mute_rules_fixture, sort_field, first_index + ): + """Test ordering mute rules by various fields.""" + response = authenticated_client.get( + reverse("mute-rule-list"), {"sort": sort_field} + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + assert data[0]["id"] == str(mute_rules_fixture[first_index].id) + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_valid( + self, + mock_task, + authenticated_client, + findings_fixture, + create_test_user, + ): + """Test creating a valid mute rule.""" + finding_ids = [str(findings_fixture[0].id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "New Mute Rule", + "reason": "Security exception approved", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_201_CREATED + + # Verify response contains the created mute rule + response_data = response.json()["data"] + assert response_data["type"] == "mute-rules" + assert response_data["attributes"]["name"] == "New Mute Rule" + assert response_data["attributes"]["reason"] == "Security exception approved" + + # Verify the finding was immediately muted + from api.models import Finding + + finding = Finding.objects.get(id=findings_fixture[0].id) + assert finding.muted is True + assert finding.muted_at is not None + assert finding.muted_reason == "Security exception approved" + + # Verify background task was called + mock_task.assert_called_once() + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_converts_finding_ids_to_uids( + self, + mock_task, + authenticated_client, + findings_fixture, + ): + """Test that finding_ids are converted to finding UIDs.""" + finding_ids = [str(findings_fixture[0].id), str(findings_fixture[1].id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "UID Conversion Test", + "reason": "Testing UID conversion", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_201_CREATED + + # Verify finding_uids contains the UIDs, not IDs + from api.models import MuteRule + + mute_rule = MuteRule.objects.get(name="UID Conversion Test") + expected_uids = [ + findings_fixture[0].uid, + findings_fixture[1].uid, + ] + assert set(mute_rule.finding_uids) == set(expected_uids) + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_deduplicates_uids( + self, + mock_task, + authenticated_client, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + """Test that multiple findings with same UID result in only one UID in the rule.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + shared_uid = "prowler-aws-dedupe-test-001" + + finding1 = Finding.objects.create( + tenant=tenant, + uid=shared_uid, + scan=scan, + status=Status.FAIL, + status_extended="test", + severity=Severity.high, + impact=Severity.high, + check_id="test_check", + check_metadata={"CheckId": "test_check"}, + raw_result={}, + ) + + finding2 = Finding.objects.create( + tenant=tenant, + uid=shared_uid, + scan=scan, + status=Status.FAIL, + status_extended="test", + severity=Severity.high, + impact=Severity.high, + check_id="test_check", + check_metadata={"CheckId": "test_check"}, + raw_result={}, + ) + + finding_ids = [str(finding1.id), str(finding2.id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Dedupe Test Rule", + "reason": "Testing UID deduplication", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_201_CREATED + + from api.models import MuteRule + + mute_rule = MuteRule.objects.get(name="Dedupe Test Rule") + assert len(mute_rule.finding_uids) == 1 + assert mute_rule.finding_uids[0] == shared_uid + + finding1.refresh_from_db() + finding2.refresh_from_db() + assert finding1.muted is True + assert finding2.muted is True + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_overlap_detection_active( + self, + mock_task, + authenticated_client, + mute_rules_fixture, + findings_fixture, + ): + """Test that creating a rule with overlapping UIDs in active rule fails.""" + # mute_rules_fixture[0] is active and has findings_fixture[0] UID + finding_ids = [str(findings_fixture[0].id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Overlapping Rule", + "reason": "This should fail", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_409_CONFLICT + assert "errors" in response.json() + error_detail = response.json()["errors"][0]["detail"] + assert ( + "already muted" in error_detail.lower() or "overlap" in error_detail.lower() + ) + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_no_overlap_with_inactive( + self, + mock_task, + authenticated_client, + mute_rules_fixture, + findings_fixture, + ): + """Test that disabled rules don't prevent new rules with same UIDs.""" + # mute_rules_fixture[1] is disabled + # Disable the enabled rule first + mute_rules_fixture[0].enabled = False + mute_rules_fixture[0].save() + + finding_ids = [str(findings_fixture[0].id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Non-overlapping Rule", + "reason": "Inactive rules don't block", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_201_CREATED + + def test_mute_rules_create_invalid_empty_finding_ids(self, authenticated_client): + """Test creating mute rule with empty finding_ids fails.""" + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Valid", + "reason": "Valid", + "finding_ids": [], + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + assert ( + response.json()["errors"][0]["source"]["pointer"] + == "/data/attributes/finding_ids" + ) + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_invalid_finding_ids( + self, mock_task, authenticated_client + ): + """Test creating mute rule with non-existent finding IDs fails.""" + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Invalid Findings", + "reason": "This should fail", + "finding_ids": ["f498b103-c760-4785-9a3e-e23fafbb7b02"], + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + + def test_mute_rules_create_duplicate_name( + self, authenticated_client, mute_rules_fixture + ): + """Test creating a mute rule with duplicate name fails.""" + existing_name = mute_rules_fixture[0].name + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": existing_name, + "reason": "Duplicate name test", + "finding_ids": ["f498b103-c760-4785-9a3e-e23fafbb7b02"], + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + + def test_mute_rules_update_name(self, authenticated_client, mute_rules_fixture): + """Test updating mute rule name.""" + mute_rule = mute_rules_fixture[0] + data = { + "data": { + "type": "mute-rules", + "id": str(mute_rule.id), + "attributes": { + "name": "Updated Name", + }, + } + } + response = authenticated_client.patch( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_200_OK + response_data = response.json()["data"] + assert response_data["attributes"]["name"] == "Updated Name" + + # Verify database was updated + mute_rule.refresh_from_db() + assert mute_rule.name == "Updated Name" + + def test_mute_rules_update_reason(self, authenticated_client, mute_rules_fixture): + """Test updating mute rule reason.""" + mute_rule = mute_rules_fixture[0] + data = { + "data": { + "type": "mute-rules", + "id": str(mute_rule.id), + "attributes": { + "reason": "Updated reason for muting", + }, + } + } + response = authenticated_client.patch( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_200_OK + response_data = response.json()["data"] + assert response_data["attributes"]["reason"] == "Updated reason for muting" + + mute_rule.refresh_from_db() + assert mute_rule.reason == "Updated reason for muting" + + def test_mute_rules_update_enabled(self, authenticated_client, mute_rules_fixture): + """Test disabling a mute rule.""" + mute_rule = mute_rules_fixture[0] + assert mute_rule.enabled is True + + data = { + "data": { + "type": "mute-rules", + "id": str(mute_rule.id), + "attributes": { + "enabled": False, + }, + } + } + response = authenticated_client.patch( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_200_OK + response_data = response.json()["data"] + assert response_data["attributes"]["enabled"] is False + + mute_rule.refresh_from_db() + assert mute_rule.enabled is False + + def test_mute_rules_update_duplicate_name( + self, authenticated_client, mute_rules_fixture + ): + """Test updating mute rule with duplicate name fails.""" + first_rule = mute_rules_fixture[0] + second_rule = mute_rules_fixture[1] + + data = { + "data": { + "type": "mute-rules", + "id": str(second_rule.id), + "attributes": { + "name": first_rule.name, + }, + } + } + response = authenticated_client.patch( + reverse("mute-rule-detail", kwargs={"pk": second_rule.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + + def test_mute_rules_delete(self, authenticated_client, mute_rules_fixture): + """Test deleting a mute rule.""" + mute_rule = mute_rules_fixture[0] + response = authenticated_client.delete( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + + # Verify rule was deleted + from api.models import MuteRule + + assert not MuteRule.objects.filter(id=mute_rule.id).exists() + + def test_mute_rules_tenant_isolation( + self, authenticated_client, mute_rules_fixture, tenants_fixture + ): + """Test that users can only access mute rules from their tenant.""" + # Create a second tenant with a mute rule + from api.models import MuteRule, Tenant + + other_tenant = Tenant.objects.create(name="Other Tenant") + other_rule = MuteRule.objects.create( + tenant=other_tenant, + name="Other Tenant Rule", + reason="Should not be visible", + finding_uids=["test-uid"], + ) + + # Try to access other tenant's rule + response = authenticated_client.get( + reverse("mute-rule-detail", kwargs={"pk": other_rule.id}) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + # List should only show current tenant's rules + response = authenticated_client.get(reverse("mute-rule-list")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == len(mute_rules_fixture) + for rule_data in data: + assert rule_data["id"] != str(other_rule.id) + + @pytest.mark.parametrize( + "credentials", + [ + {}, # empty credentials + { + "access_key_id": "AKIAIOSFODNN7EXAMPLE" + }, # missing secret_access_key and region + { + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + }, # missing access_key_id and region + { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, # missing region + { # invalid access_key_id format (not starting with AKIA) + "access_key_id": "ABCD0123456789ABCDEF", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + }, + { # invalid access_key_id format (wrong length) + "access_key_id": "AKIAIOSFODNN7EXAMPL", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + }, + { # invalid secret_access_key format (wrong length) + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEK", + "region": "us-east-1", + }, + { # invalid region format + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "invalid-region", + }, + { # invalid region format (uppercase) + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "US-EAST-1", + }, + ], + ) + def test_bedrock_invalid_credentials(self, authenticated_client, credentials): + """Bedrock provider with invalid credentials should error""" + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": credentials, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + + def test_bedrock_valid_credentials_success(self, authenticated_client): + """Bedrock provider with valid AWS credentials should succeed and mask credentials""" + valid_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + } + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": valid_credentials, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_201_CREATED + data = resp.json()["data"] + + # Verify credentials are returned masked + masked_creds = data["attributes"].get("credentials") + assert masked_creds is not None + assert "access_key_id" in masked_creds + assert "secret_access_key" in masked_creds + assert "region" in masked_creds + # Verify all characters are masked with asterisks + assert all(c == "*" for c in masked_creds["access_key_id"]) + assert all(c == "*" for c in masked_creds["secret_access_key"]) + + def test_bedrock_provider_duplicate_per_tenant(self, authenticated_client): + """Creating a second Bedrock provider for same tenant should fail""" + valid_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-west-2", + } + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": valid_credentials, + }, + } + } + # First creation succeeds + resp1 = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp1.status_code == status.HTTP_201_CREATED + + # Second creation should fail with validation error + resp2 = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp2.status_code == status.HTTP_400_BAD_REQUEST + assert "already exists" in str(resp2.json()).lower() + + def test_bedrock_patch_credentials_and_fields_filter(self, authenticated_client): + """PATCH credentials and verify fields filter returns decrypted values""" + valid_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "eu-west-1", + } + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": valid_credentials, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Update credentials with new valid ones + new_credentials = { + "access_key_id": "AKIAZZZZZZZZZZZZZZZZ", + "secret_access_key": "aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789+/==", + "region": "ap-south-1", + } + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "credentials": new_credentials, + "is_active": False, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + updated = patch_resp.json()["data"]["attributes"] + assert updated["is_active"] is False + + # Default GET should return masked credentials + get_resp = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + ) + assert get_resp.status_code == status.HTTP_200_OK + masked = get_resp.json()["data"]["attributes"]["credentials"] + assert all(c == "*" for c in masked["access_key_id"]) + assert all(c == "*" for c in masked["secret_access_key"]) + + # Fields filter should return decrypted credentials + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + assert creds["access_key_id"] == new_credentials["access_key_id"] + assert creds["secret_access_key"] == new_credentials["secret_access_key"] + assert creds["region"] == new_credentials["region"] + + def test_bedrock_partial_credential_update(self, authenticated_client): + """Test partial update of Bedrock credentials (e.g., only region)""" + # Create provider with full credentials + initial_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + } + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": initial_credentials, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Update only the region field + partial_update = { + "region": "eu-west-1", + } + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "credentials": partial_update, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + + # Verify credentials with fields filter - region should be updated, keys preserved + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + + # Original keys should be preserved + assert creds["access_key_id"] == initial_credentials["access_key_id"] + assert creds["secret_access_key"] == initial_credentials["secret_access_key"] + # Region should be updated + assert creds["region"] == "eu-west-1" + + @pytest.mark.parametrize( + "attributes", + [ + pytest.param( + { + "provider_type": "openai_compatible", + "credentials": {"api_key": "compat-key"}, + }, + id="missing", + ), + pytest.param( + { + "provider_type": "openai_compatible", + "credentials": {"api_key": "compat-key"}, + "base_url": "", + }, + id="empty", + ), + ], + ) + def test_openai_compatible_missing_base_url(self, authenticated_client, attributes): + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": attributes, + } + } + + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + error_detail = str(resp.json()).lower() + assert "base_url" in error_detail + + def test_openai_compatible_invalid_credentials(self, authenticated_client): + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai_compatible", + "base_url": "https://compat.example/v1", + "credentials": {"api_key": ""}, + }, + } + } + + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + errors = resp.json().get("errors", []) + assert any( + error.get("source", {}).get("pointer") + == "/data/attributes/credentials/api_key" + for error in errors + ) + assert any( + "may not be blank" in error.get("detail", "").lower() for error in errors + ) + + def test_openai_compatible_patch_credentials_and_fields(self, authenticated_client): + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai_compatible", + "base_url": "https://compat.example/v1", + "credentials": {"api_key": "compat-key-123"}, + }, + } + } + + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + updated_base_url = "https://compat.example/v2" + updated_api_key = "compat-key-456" + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "base_url": updated_base_url, + "credentials": {"api_key": updated_api_key}, + }, + } + } + + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + updated_attrs = patch_resp.json()["data"]["attributes"] + assert updated_attrs["base_url"] == updated_base_url + assert updated_attrs["credentials"]["api_key"] == "*" * len(updated_api_key) + + get_resp = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + ) + assert get_resp.status_code == status.HTTP_200_OK + masked = get_resp.json()["data"]["attributes"]["credentials"]["api_key"] + assert masked == "*" * len(updated_api_key) + + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + assert creds["api_key"] == updated_api_key diff --git a/api/src/backend/api/utils.py b/api/src/backend/api/utils.py index 97325d9240..958f24b3f0 100644 --- a/api/src/backend/api/utils.py +++ b/api/src/backend/api/utils.py @@ -18,8 +18,11 @@ from prowler.providers.azure.azure_provider import AzureProvider from prowler.providers.common.models import Connection from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.github.github_provider import GithubProvider +from prowler.providers.iac.iac_provider import IacProvider from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider from prowler.providers.m365.m365_provider import M365Provider +from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider +from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider class CustomOAuth2Client(OAuth2Client): @@ -65,8 +68,11 @@ def return_prowler_provider( | AzureProvider | GcpProvider | GithubProvider + | IacProvider | KubernetesProvider | M365Provider + | MongodbatlasProvider + | OraclecloudProvider ]: """Return the Prowler provider class based on the given provider type. @@ -74,7 +80,7 @@ def return_prowler_provider( provider (Provider): The provider object containing the provider type and associated secrets. Returns: - AwsProvider | AzureProvider | GcpProvider | GithubProvider | KubernetesProvider | M365Provider: The corresponding provider class. + AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: The corresponding provider class. Raises: ValueError: If the provider type specified in `provider.provider` is not supported. @@ -92,6 +98,12 @@ def return_prowler_provider( prowler_provider = M365Provider case Provider.ProviderChoices.GITHUB.value: prowler_provider = GithubProvider + case Provider.ProviderChoices.MONGODBATLAS.value: + prowler_provider = MongodbatlasProvider + case Provider.ProviderChoices.IAC.value: + prowler_provider = IacProvider + case Provider.ProviderChoices.ORACLECLOUD.value: + prowler_provider = OraclecloudProvider case _: raise ValueError(f"Provider type {provider.provider} not supported") return prowler_provider @@ -128,6 +140,21 @@ def get_prowler_provider_kwargs( **prowler_provider_kwargs, "organizations": [provider.uid], } + elif provider.provider == Provider.ProviderChoices.IAC.value: + # For IaC provider, uid contains the repository URL + # Extract the access token if present in the secret + prowler_provider_kwargs = { + "scan_repository_url": provider.uid, + } + if "access_token" in provider.secret.secret: + prowler_provider_kwargs["oauth_app_token"] = provider.secret.secret[ + "access_token" + ] + elif provider.provider == Provider.ProviderChoices.MONGODBATLAS.value: + prowler_provider_kwargs = { + **prowler_provider_kwargs, + "atlas_organization_id": provider.uid, + } if mutelist_processor: mutelist_content = mutelist_processor.configuration.get("Mutelist", {}) @@ -145,8 +172,11 @@ def initialize_prowler_provider( | AzureProvider | GcpProvider | GithubProvider + | IacProvider | KubernetesProvider | M365Provider + | MongodbatlasProvider + | OraclecloudProvider ): """Initialize a Prowler provider instance based on the given provider type. @@ -155,8 +185,8 @@ def initialize_prowler_provider( mutelist_processor (Processor): The mutelist processor object containing the mutelist configuration. Returns: - AwsProvider | AzureProvider | GcpProvider | GithubProvider | KubernetesProvider | M365Provider: An instance of the corresponding provider class - (`AwsProvider`, `AzureProvider`, `GcpProvider`, `GithubProvider`, `KubernetesProvider` or `M365Provider`) initialized with the + AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: An instance of the corresponding provider class + (`AwsProvider`, `AzureProvider`, `GcpProvider`, `GithubProvider`, `IacProvider`, `KubernetesProvider`, `M365Provider`, `OraclecloudProvider` or `MongodbatlasProvider`) initialized with the provider's secrets. """ prowler_provider = return_prowler_provider(provider) @@ -180,9 +210,23 @@ def prowler_provider_connection_test(provider: Provider) -> Connection: except Provider.secret.RelatedObjectDoesNotExist as secret_error: return Connection(is_connected=False, error=secret_error) - return prowler_provider.test_connection( - **prowler_provider_kwargs, provider_id=provider.uid, raise_on_exception=False - ) + # For IaC provider, construct the kwargs properly for test_connection + if provider.provider == Provider.ProviderChoices.IAC.value: + # Don't pass repository_url from secret, use scan_repository_url with the UID + iac_test_kwargs = { + "scan_repository_url": provider.uid, + "raise_on_exception": False, + } + # Add access_token if present in the secret + if "access_token" in prowler_provider_kwargs: + iac_test_kwargs["access_token"] = prowler_provider_kwargs["access_token"] + return prowler_provider.test_connection(**iac_test_kwargs) + else: + return prowler_provider.test_connection( + **prowler_provider_kwargs, + provider_id=provider.uid, + raise_on_exception=False, + ) def prowler_integration_connection_test(integration: Integration) -> Connection: diff --git a/api/src/backend/api/v1/mixins.py b/api/src/backend/api/v1/mixins.py index 1cef76a84c..e1a5d3470f 100644 --- a/api/src/backend/api/v1/mixins.py +++ b/api/src/backend/api/v1/mixins.py @@ -12,6 +12,24 @@ from api.models import StateChoices, Task from api.v1.serializers import TaskSerializer +class DisablePaginationMixin: + disable_pagination_query_param = "page[disable]" + disable_pagination_truthy_values = {"true"} + + def should_disable_pagination(self) -> bool: + if not hasattr(self, "request"): + return False + value = self.request.query_params.get(self.disable_pagination_query_param) + if value is None: + return False + return str(value).lower() in self.disable_pagination_truthy_values + + def paginate_queryset(self, queryset): + if self.should_disable_pagination(): + return None + return super().paginate_queryset(queryset) + + class PaginateByPkMixin: """ Mixin to paginate on a list of PKs (cheaper than heavy JOINs), diff --git a/api/src/backend/api/v1/serializer_utils/lighthouse.py b/api/src/backend/api/v1/serializer_utils/lighthouse.py new file mode 100644 index 0000000000..e29c679ce6 --- /dev/null +++ b/api/src/backend/api/v1/serializer_utils/lighthouse.py @@ -0,0 +1,209 @@ +import re + +from drf_spectacular.utils import extend_schema_field +from rest_framework_json_api import serializers + + +class OpenAICredentialsSerializer(serializers.Serializer): + api_key = serializers.CharField() + + def validate_api_key(self, value: str) -> str: + pattern = r"^sk-[\w-]+$" + if not re.match(pattern, value or ""): + raise serializers.ValidationError("Invalid OpenAI API key format.") + return value + + def to_internal_value(self, data): + """Check for unknown fields before DRF filters them out.""" + if not isinstance(data, dict): + raise serializers.ValidationError( + {"non_field_errors": ["Credentials must be an object"]} + ) + + allowed_fields = set(self.fields.keys()) + provided_fields = set(data.keys()) + extra_fields = provided_fields - allowed_fields + + if extra_fields: + raise serializers.ValidationError( + { + "non_field_errors": [ + f"Unknown fields in credentials: {', '.join(sorted(extra_fields))}" + ] + } + ) + + return super().to_internal_value(data) + + +class BedrockCredentialsSerializer(serializers.Serializer): + """ + Serializer for AWS Bedrock credentials validation. + + Validates long-term AWS credentials (AKIA) and region format. + """ + + access_key_id = serializers.CharField() + secret_access_key = serializers.CharField() + region = serializers.CharField() + + def validate_access_key_id(self, value: str) -> str: + """Validate AWS access key ID format (AKIA for long-term credentials).""" + pattern = r"^AKIA[0-9A-Z]{16}$" + if not re.match(pattern, value or ""): + raise serializers.ValidationError( + "Invalid AWS access key ID format. Must be AKIA followed by 16 alphanumeric characters." + ) + return value + + def validate_secret_access_key(self, value: str) -> str: + """Validate AWS secret access key format (40 base64 characters).""" + pattern = r"^[A-Za-z0-9/+=]{40}$" + if not re.match(pattern, value or ""): + raise serializers.ValidationError( + "Invalid AWS secret access key format. Must be 40 base64 characters." + ) + return value + + def validate_region(self, value: str) -> str: + """Validate AWS region format.""" + pattern = r"^[a-z]{2}-[a-z]+-\d+$" + if not re.match(pattern, value or ""): + raise serializers.ValidationError( + "Invalid AWS region format. Expected format like 'us-east-1' or 'eu-west-2'." + ) + return value + + def to_internal_value(self, data): + """Check for unknown fields before DRF filters them out.""" + if not isinstance(data, dict): + raise serializers.ValidationError( + {"non_field_errors": ["Credentials must be an object"]} + ) + + allowed_fields = set(self.fields.keys()) + provided_fields = set(data.keys()) + extra_fields = provided_fields - allowed_fields + + if extra_fields: + raise serializers.ValidationError( + { + "non_field_errors": [ + f"Unknown fields in credentials: {', '.join(sorted(extra_fields))}" + ] + } + ) + + return super().to_internal_value(data) + + +class BedrockCredentialsUpdateSerializer(BedrockCredentialsSerializer): + """ + Serializer for AWS Bedrock credentials during UPDATE operations. + + Inherits all validation logic from BedrockCredentialsSerializer but makes + all fields optional to support partial updates. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Make all fields optional for updates + for field in self.fields.values(): + field.required = False + + +class OpenAICompatibleCredentialsSerializer(serializers.Serializer): + """ + Minimal serializer for OpenAI-compatible credentials. + + Many OpenAI-compatible providers do not use the same key format as OpenAI. + We only require a non-empty API key string. Additional fields can be added later + without breaking existing configurations. + """ + + api_key = serializers.CharField() + + def validate_api_key(self, value: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise serializers.ValidationError("API key is required.") + return value.strip() + + def to_internal_value(self, data): + """Check for unknown fields before DRF filters them out.""" + if not isinstance(data, dict): + raise serializers.ValidationError( + {"non_field_errors": ["Credentials must be an object"]} + ) + + allowed_fields = set(self.fields.keys()) + provided_fields = set(data.keys()) + extra_fields = provided_fields - allowed_fields + + if extra_fields: + raise serializers.ValidationError( + { + "non_field_errors": [ + f"Unknown fields in credentials: {', '.join(sorted(extra_fields))}" + ] + } + ) + + return super().to_internal_value(data) + + +@extend_schema_field( + { + "oneOf": [ + { + "type": "object", + "title": "OpenAI Credentials", + "properties": { + "api_key": { + "type": "string", + "description": "OpenAI API key. Must start with 'sk-' followed by alphanumeric characters, " + "hyphens, or underscores.", + "pattern": "^sk-[\\w-]+$", + } + }, + "required": ["api_key"], + }, + { + "type": "object", + "title": "AWS Bedrock Credentials", + "properties": { + "access_key_id": { + "type": "string", + "description": "AWS access key ID.", + "pattern": "^AKIA[0-9A-Z]{16}$", + }, + "secret_access_key": { + "type": "string", + "description": "AWS secret access key.", + "pattern": "^[A-Za-z0-9/+=]{40}$", + }, + "region": { + "type": "string", + "description": "AWS region identifier where Bedrock is available. Examples: us-east-1, " + "us-west-2, eu-west-1, ap-northeast-1.", + "pattern": "^[a-z]{2}-[a-z]+-\\d+$", + }, + }, + "required": ["access_key_id", "secret_access_key", "region"], + }, + { + "type": "object", + "title": "OpenAI Compatible Credentials", + "properties": { + "api_key": { + "type": "string", + "description": "API key for OpenAI-compatible provider. The format varies by provider. " + "Note: The 'base_url' field (separate from credentials) is required when using this provider type.", + } + }, + "required": ["api_key"], + }, + ] + } +) +class LighthouseCredentialsField(serializers.JSONField): + pass diff --git a/api/src/backend/api/v1/serializer_utils/providers.py b/api/src/backend/api/v1/serializer_utils/providers.py index 76fa0b4911..4ec772e02f 100644 --- a/api/src/backend/api/v1/serializer_utils/providers.py +++ b/api/src/backend/api/v1/serializer_utils/providers.py @@ -105,23 +105,25 @@ from rest_framework_json_api import serializers "type": "string", "description": "The Azure application (client) ID for authentication in Azure AD.", }, - "client_secret": { - "type": "string", - "description": "The client secret associated with the application (client) ID, providing " - "secure access.", - }, "tenant_id": { "type": "string", "description": "The Azure tenant ID, representing the directory where the application is " "registered.", }, + "client_secret": { + "type": "string", + "description": "The client secret associated with the application (client) ID, providing " + "secure access.", + }, "user": { "type": "email", "description": "User microsoft email address.", + "deprecated": True, }, "password": { "type": "string", "description": "User password.", + "deprecated": True, }, }, "required": [ @@ -132,6 +134,30 @@ from rest_framework_json_api import serializers "password", ], }, + { + "type": "object", + "title": "M365 Certificate Credentials", + "properties": { + "client_id": { + "type": "string", + "description": "The Azure application (client) ID for authentication in Azure AD.", + }, + "tenant_id": { + "type": "string", + "description": "The Azure tenant ID, representing the directory where the application is " + "registered.", + }, + "certificate_content": { + "type": "string", + "description": "The certificate content in base64 format for certificate-based authentication.", + }, + }, + "required": [ + "client_id", + "tenant_id", + "certificate_content", + ], + }, { "type": "object", "title": "GCP Static Credentials", @@ -213,6 +239,71 @@ from rest_framework_json_api import serializers }, "required": ["github_app_id", "github_app_key"], }, + { + "type": "object", + "title": "IaC Repository Credentials", + "properties": { + "repository_url": { + "type": "string", + "description": "Repository URL to scan for IaC files.", + }, + "access_token": { + "type": "string", + "description": "Optional access token for private repositories.", + }, + }, + "required": ["repository_url"], + }, + { + "type": "object", + "title": "Oracle Cloud Infrastructure (OCI) API Key Credentials", + "properties": { + "user": { + "type": "string", + "description": "The OCID of the user to authenticate with.", + }, + "fingerprint": { + "type": "string", + "description": "The fingerprint of the API signing key.", + }, + "key_file": { + "type": "string", + "description": "The path to the private key file for API signing. Either key_file or key_content must be provided.", + }, + "key_content": { + "type": "string", + "description": "The content of the private key for API signing (base64 encoded). Either key_file or key_content must be provided.", + }, + "tenancy": { + "type": "string", + "description": "The OCID of the tenancy.", + }, + "region": { + "type": "string", + "description": "The OCI region identifier (e.g., us-ashburn-1, us-phoenix-1).", + }, + "pass_phrase": { + "type": "string", + "description": "The passphrase for the private key, if encrypted.", + }, + }, + "required": ["user", "fingerprint", "tenancy", "region"], + }, + { + "type": "object", + "title": "MongoDB Atlas API Key", + "properties": { + "atlas_public_key": { + "type": "string", + "description": "MongoDB Atlas API public key.", + }, + "atlas_private_key": { + "type": "string", + "description": "MongoDB Atlas API private key.", + }, + }, + "required": ["atlas_public_key", "atlas_private_key"], + }, ] } ) diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 32ebbb040d..77b4840361 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -1,3 +1,4 @@ +import base64 import json from datetime import datetime, timedelta, timezone @@ -5,8 +6,10 @@ from django.conf import settings from django.contrib.auth import authenticate from django.contrib.auth.models import update_last_login from django.contrib.auth.password_validation import validate_password +from django.db import IntegrityError from drf_spectacular.utils import extend_schema_field from jwt.exceptions import InvalidKeyError +from rest_framework.reverse import reverse from rest_framework.validators import UniqueTogetherValidator from rest_framework_json_api import serializers from rest_framework_json_api.relations import SerializerMethodResourceRelatedField @@ -24,7 +27,11 @@ from api.models import ( Invitation, InvitationRoleRelationship, LighthouseConfiguration, + LighthouseProviderConfiguration, + LighthouseProviderModels, + LighthouseTenantConfiguration, Membership, + MuteRule, Processor, Provider, ProviderGroup, @@ -40,6 +47,7 @@ from api.models import ( StatusChoices, Task, TenantAPIKey, + ThreatScoreSnapshot, User, UserRoleRelationship, ) @@ -53,6 +61,13 @@ from api.v1.serializer_utils.integrations import ( S3ConfigSerializer, SecurityHubConfigSerializer, ) +from api.v1.serializer_utils.lighthouse import ( + BedrockCredentialsSerializer, + BedrockCredentialsUpdateSerializer, + LighthouseCredentialsField, + OpenAICompatibleCredentialsSerializer, + OpenAICredentialsSerializer, +) from api.v1.serializer_utils.processors import ProcessorConfigField from api.v1.serializer_utils.providers import ProviderSecretField from prowler.lib.mutelist.mutelist import Mutelist @@ -1348,10 +1363,16 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer): serializer = GCPProviderSecret(data=secret) elif provider_type == Provider.ProviderChoices.GITHUB.value: serializer = GithubProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.IAC.value: + serializer = IacProviderSecret(data=secret) elif provider_type == Provider.ProviderChoices.KUBERNETES.value: serializer = KubernetesProviderSecret(data=secret) elif provider_type == Provider.ProviderChoices.M365.value: serializer = M365ProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.ORACLECLOUD.value: + serializer = OracleCloudProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.MONGODBATLAS.value: + serializer = MongoDBAtlasProviderSecret(data=secret) else: raise serializers.ValidationError( {"provider": f"Provider type not supported {provider_type}"} @@ -1395,10 +1416,38 @@ class AzureProviderSecret(serializers.Serializer): class M365ProviderSecret(serializers.Serializer): client_id = serializers.CharField() - client_secret = serializers.CharField() + client_secret = serializers.CharField(required=False) tenant_id = serializers.CharField() user = serializers.EmailField(required=False) password = serializers.CharField(required=False) + certificate_content = serializers.CharField(required=False) + + def validate(self, attrs): + if attrs.get("client_secret") and attrs.get("certificate_content"): + raise serializers.ValidationError( + "You cannot provide both client_secret and certificate_content." + ) + if not attrs.get("client_secret") and not attrs.get("certificate_content"): + raise serializers.ValidationError( + "You must provide either client_secret or certificate_content." + ) + return super().validate(attrs) + + def validate_certificate_content(self, certificate_content): + """Validate that M365 certificate content is valid base64 encoded data.""" + if certificate_content: + try: + base64.b64decode(certificate_content, validate=True) + except Exception as e: + raise ValidationError( + { + "certificate_content": [ + f"The provided certificate content is not valid base64 encoded data: {str(e)}" + ] + }, + code="m365-certificate-content", + ) + return certificate_content class Meta: resource_name = "provider-secrets" @@ -1420,6 +1469,14 @@ class GCPServiceAccountProviderSecret(serializers.Serializer): resource_name = "provider-secrets" +class MongoDBAtlasProviderSecret(serializers.Serializer): + atlas_public_key = serializers.CharField() + atlas_private_key = serializers.CharField() + + class Meta: + resource_name = "provider-secrets" + + class KubernetesProviderSecret(serializers.Serializer): kubeconfig_content = serializers.CharField() @@ -1437,6 +1494,27 @@ class GithubProviderSecret(serializers.Serializer): resource_name = "provider-secrets" +class IacProviderSecret(serializers.Serializer): + repository_url = serializers.CharField() + access_token = serializers.CharField(required=False) + + class Meta: + resource_name = "provider-secrets" + + +class OracleCloudProviderSecret(serializers.Serializer): + user = serializers.CharField() + fingerprint = serializers.CharField() + key_file = serializers.CharField(required=False) + key_content = serializers.CharField(required=False) + tenancy = serializers.CharField() + region = serializers.CharField() + pass_phrase = serializers.CharField(required=False) + + class Meta: + resource_name = "provider-secrets" + + class AWSRoleAssumptionProviderSecret(serializers.Serializer): role_arn = serializers.CharField() external_id = serializers.CharField() @@ -2070,6 +2148,17 @@ class OverviewProviderSerializer(serializers.Serializer): } +class OverviewProviderCountSerializer(serializers.Serializer): + id = serializers.CharField(source="provider") + count = serializers.IntegerField() + + class JSONAPIMeta: + resource_name = "providers-count-overview" + + def get_root_meta(self, _resource, _many): + return {"version": "v1"} + + class OverviewFindingSerializer(serializers.Serializer): id = serializers.CharField(default="n/a") new = serializers.IntegerField() @@ -2721,6 +2810,16 @@ class LighthouseConfigCreateSerializer(RLSSerializer, BaseWriteSerializer): "updated_at": {"read_only": True}, } + def validate_temperature(self, value): + if not 0 <= value <= 1: + raise ValidationError("Temperature must be between 0 and 1.") + return value + + def validate_max_tokens(self, value): + if not 500 <= value <= 5000: + raise ValidationError("Max tokens must be between 500 and 5000.") + return value + def validate(self, attrs): tenant_id = self.context.get("request").tenant_id if LighthouseConfiguration.objects.filter(tenant_id=tenant_id).exists(): @@ -2729,6 +2828,11 @@ class LighthouseConfigCreateSerializer(RLSSerializer, BaseWriteSerializer): "tenant_id": "Lighthouse configuration already exists for this tenant." } ) + api_key = attrs.get("api_key") + if api_key is not None: + OpenAICredentialsSerializer(data={"api_key": api_key}).is_valid( + raise_exception=True + ) return super().validate(attrs) def create(self, validated_data): @@ -2773,6 +2877,24 @@ class LighthouseConfigUpdateSerializer(BaseWriteSerializer): "max_tokens": {"required": False}, } + def validate_temperature(self, value): + if not 0 <= value <= 1: + raise ValidationError("Temperature must be between 0 and 1.") + return value + + def validate_max_tokens(self, value): + if not 500 <= value <= 5000: + raise ValidationError("Max tokens must be between 500 and 5000.") + return value + + def validate(self, attrs): + api_key = attrs.get("api_key", None) + if api_key is not None: + OpenAICredentialsSerializer(data={"api_key": api_key}).is_valid( + raise_exception=True + ) + return super().validate(attrs) + def update(self, instance, validated_data): api_key = validated_data.pop("api_key", None) instance = super().update(instance, validated_data) @@ -2843,6 +2965,13 @@ class TenantApiKeyCreateSerializer(RLSSerializer, BaseWriteSerializer): "api_key": {"read_only": True}, } + def validate_name(self, value): + """Validate that the name is unique within the tenant.""" + tenant_id = self.context.get("tenant_id") + if TenantAPIKey.objects.filter(tenant_id=tenant_id, name=value).exists(): + raise ValidationError("An API key with this name already exists.") + return value + def get_api_key(self, obj): """Return the raw API key if it was stored during creation.""" return getattr(obj, "_raw_api_key", None) @@ -2884,3 +3013,678 @@ class TenantApiKeyUpdateSerializer(RLSSerializer, BaseWriteSerializer): "inserted_at": {"read_only": True}, "last_used_at": {"read_only": True}, } + + def validate_name(self, value): + """Validate that the name is unique within the tenant, excluding current instance.""" + tenant_id = self.context.get("tenant_id") + if ( + TenantAPIKey.objects.filter(tenant_id=tenant_id, name=value) + .exclude(id=self.instance.id) + .exists() + ): + raise ValidationError("An API key with this name already exists.") + return value + + +# Lighthouse: Provider configurations + + +class LighthouseProviderConfigSerializer(RLSSerializer): + """ + Read serializer for LighthouseProviderConfiguration. + """ + + # Decrypted credentials are only returned in to_representation when requested + credentials = serializers.JSONField(required=False, read_only=True) + + class Meta: + model = LighthouseProviderConfiguration + fields = [ + "id", + "inserted_at", + "updated_at", + "provider_type", + "base_url", + "is_active", + "credentials", + "url", + ] + extra_kwargs = { + "id": {"read_only": True}, + "inserted_at": {"read_only": True}, + "updated_at": {"read_only": True}, + "is_active": {"read_only": True}, + "url": {"read_only": True, "view_name": "lighthouse-providers-detail"}, + } + + class JSONAPIMeta: + resource_name = "lighthouse-providers" + + def to_representation(self, instance): + data = super().to_representation(instance) + # Support JSON:API fields filter: fields[lighthouse-providers]=credentials,base_url + fields_param = self.context.get("request", None) and self.context[ + "request" + ].query_params.get("fields[lighthouse-providers]", "") + + creds = instance.credentials_decoded + + requested_fields = ( + [f.strip() for f in fields_param.split(",")] if fields_param else [] + ) + + if "credentials" in requested_fields: + # Return full decrypted credentials JSON + data["credentials"] = creds + else: + # Return masked credentials by default + def mask_value(value): + if isinstance(value, str): + return "*" * len(value) + if isinstance(value, dict): + return {k: mask_value(v) for k, v in value.items()} + if isinstance(value, list): + return [mask_value(v) for v in value] + return value + + # Always return masked credentials, even if creds is None + if creds is not None: + data["credentials"] = mask_value(creds) + else: + # If credentials_decoded returns None, return None for credentials field + data["credentials"] = None + + return data + + +class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerializer): + """ + Create serializer for LighthouseProviderConfiguration. + Accepts credentials as JSON; stored encrypted via credentials_decoded. + """ + + credentials = LighthouseCredentialsField(write_only=True, required=True) + base_url = serializers.URLField( + required=False, + allow_null=True, + help_text="Base URL for the LLM provider API. Required for 'openai_compatible' provider type.", + ) + + class Meta: + model = LighthouseProviderConfiguration + fields = [ + "provider_type", + "base_url", + "credentials", + "is_active", + ] + extra_kwargs = { + "is_active": {"required": False}, + "provider_type": { + "help_text": "LLM provider type. Determines which credential format to use. " + "See 'credentials' field documentation for provider-specific requirements." + }, + } + + def create(self, validated_data): + credentials = validated_data.pop("credentials") + + instance = LighthouseProviderConfiguration(**validated_data) + instance.tenant_id = self.context.get("tenant_id") + instance.credentials_decoded = credentials + + try: + instance.save() + return instance + except IntegrityError: + raise ValidationError( + { + "provider_type": "Configuration for this provider already exists for the tenant." + } + ) + + def validate(self, attrs): + provider_type = attrs.get("provider_type") + credentials = attrs.get("credentials") or {} + base_url = attrs.get("base_url") + + if provider_type == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI: + try: + OpenAICredentialsSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e + elif ( + provider_type == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + try: + BedrockCredentialsSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e + elif ( + provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE + ): + if not base_url: + raise ValidationError({"base_url": "Base URL is required."}) + try: + OpenAICompatibleCredentialsSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e + + return super().validate(attrs) + + +class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): + """ + Update serializer for LighthouseProviderConfiguration. + """ + + credentials = LighthouseCredentialsField(write_only=True, required=False) + base_url = serializers.URLField( + required=False, + allow_null=True, + help_text="Base URL for the LLM provider API. Required for 'openai_compatible' provider type.", + ) + + class Meta: + model = LighthouseProviderConfiguration + fields = [ + "id", + "provider_type", + "base_url", + "credentials", + "is_active", + ] + extra_kwargs = { + "id": {"read_only": True}, + "provider_type": {"read_only": True}, + "is_active": {"required": False}, + } + + def update(self, instance, validated_data): + credentials = validated_data.pop("credentials", None) + + for attr, value in validated_data.items(): + setattr(instance, attr, value) + + if credentials is not None: + # Merge partial credentials with existing ones + # New values overwrite existing ones, but unspecified fields are preserved + existing_credentials = instance.credentials_decoded or {} + merged_credentials = {**existing_credentials, **credentials} + instance.credentials_decoded = merged_credentials + + instance.save() + return instance + + def validate(self, attrs): + provider_type = getattr(self.instance, "provider_type", None) + credentials = attrs.get("credentials", None) + base_url = attrs.get("base_url", None) + + if ( + credentials is not None + and provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI + ): + try: + OpenAICredentialsSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e + elif ( + credentials is not None + and provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + try: + BedrockCredentialsUpdateSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e + elif ( + credentials is not None + and provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE + ): + if base_url is None: + pass + elif not base_url: + raise ValidationError({"base_url": "Base URL cannot be empty."}) + try: + OpenAICompatibleCredentialsSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e + + return super().validate(attrs) + + +# Lighthouse: Tenant configuration + + +class LighthouseTenantConfigSerializer(RLSSerializer): + """ + Read serializer for LighthouseTenantConfiguration. + """ + + # Build singleton URL without pk + url = serializers.SerializerMethodField() + + def get_url(self, obj): + request = self.context.get("request") + return reverse("lighthouse-configurations", request=request) + + class Meta: + model = LighthouseTenantConfiguration + fields = [ + "id", + "inserted_at", + "updated_at", + "business_context", + "default_provider", + "default_models", + "url", + ] + extra_kwargs = { + "id": {"read_only": True}, + "inserted_at": {"read_only": True}, + "updated_at": {"read_only": True}, + "url": {"read_only": True}, + } + + +class LighthouseTenantConfigUpdateSerializer(BaseWriteSerializer): + class Meta: + model = LighthouseTenantConfiguration + fields = [ + "id", + "business_context", + "default_provider", + "default_models", + ] + extra_kwargs = { + "id": {"read_only": True}, + } + + def validate(self, attrs): + request = self.context.get("request") + tenant_id = self.context.get("tenant_id") or ( + getattr(request, "tenant_id", None) if request else None + ) + + default_provider = attrs.get( + "default_provider", getattr(self.instance, "default_provider", "") + ) + default_models = attrs.get( + "default_models", getattr(self.instance, "default_models", {}) + ) + + if default_provider: + supported = set(LighthouseProviderConfiguration.LLMProviderChoices.values) + if default_provider not in supported: + raise ValidationError( + {"default_provider": f"Unsupported provider '{default_provider}'."} + ) + if not LighthouseProviderConfiguration.objects.filter( + tenant_id=tenant_id, provider_type=default_provider, is_active=True + ).exists(): + raise ValidationError( + { + "default_provider": f"No active configuration found for '{default_provider}'." + } + ) + + if default_models is not None and not isinstance(default_models, dict): + raise ValidationError( + {"default_models": "Must be an object mapping provider -> model_id."} + ) + + for provider_type, model_id in (default_models or {}).items(): + provider_cfg = LighthouseProviderConfiguration.objects.filter( + tenant_id=tenant_id, provider_type=provider_type, is_active=True + ).first() + if not provider_cfg: + raise ValidationError( + { + "default_models": f"No active configuration for provider '{provider_type}'." + } + ) + if not LighthouseProviderModels.objects.filter( + tenant_id=tenant_id, + provider_configuration=provider_cfg, + model_id=model_id, + ).exists(): + raise ValidationError( + { + "default_models": f"Invalid model '{model_id}' for provider '{provider_type}'." + } + ) + + return super().validate(attrs) + + +# Lighthouse: Provider models + + +class LighthouseProviderModelsSerializer(RLSSerializer): + """ + Read serializer for LighthouseProviderModels. + """ + + provider_configuration = serializers.ResourceRelatedField(read_only=True) + + class Meta: + model = LighthouseProviderModels + fields = [ + "id", + "inserted_at", + "updated_at", + "provider_configuration", + "model_id", + "model_name", + "default_parameters", + "url", + ] + extra_kwargs = { + "id": {"read_only": True}, + "inserted_at": {"read_only": True}, + "updated_at": {"read_only": True}, + "url": {"read_only": True, "view_name": "lighthouse-models-detail"}, + } + + +class LighthouseProviderModelsCreateSerializer(RLSSerializer, BaseWriteSerializer): + provider_configuration = serializers.ResourceRelatedField( + queryset=LighthouseProviderConfiguration.objects.all() + ) + + class Meta: + model = LighthouseProviderModels + fields = [ + "provider_configuration", + "model_id", + "default_parameters", + ] + extra_kwargs = { + "default_parameters": {"required": False}, + } + + +class LighthouseProviderModelsUpdateSerializer(BaseWriteSerializer): + class Meta: + model = LighthouseProviderModels + fields = [ + "id", + "default_parameters", + ] + extra_kwargs = { + "id": {"read_only": True}, + } + + +# Mute Rules + + +class MuteRuleSerializer(RLSSerializer): + """ + Serializer for reading MuteRule instances. + """ + + finding_uids = serializers.ListField( + child=serializers.CharField(), + read_only=True, + help_text="List of finding UIDs that are muted by this rule", + ) + + class Meta: + model = MuteRule + fields = [ + "id", + "inserted_at", + "updated_at", + "name", + "reason", + "enabled", + "created_by", + "finding_uids", + ] + + included_serializers = { + "created_by": "api.v1.serializers.UserIncludeSerializer", + } + + +class MuteRuleCreateSerializer(RLSSerializer, BaseWriteSerializer): + """ + Serializer for creating new MuteRule instances. + + Accepts finding_ids in the request, converts them to UIDs, and stores in finding_uids. + """ + + finding_ids = serializers.ListField( + child=serializers.UUIDField(), + write_only=True, + required=True, + help_text="List of Finding IDs to mute (will be converted to UIDs)", + ) + finding_uids = serializers.ListField( + child=serializers.CharField(), + read_only=True, + help_text="List of finding UIDs that are muted by this rule", + ) + + class Meta: + model = MuteRule + fields = [ + "id", + "inserted_at", + "updated_at", + "name", + "reason", + "enabled", + "created_by", + "finding_ids", + "finding_uids", + ] + extra_kwargs = { + "id": {"read_only": True}, + "inserted_at": {"read_only": True}, + "updated_at": {"read_only": True}, + "enabled": {"read_only": True}, + "created_by": {"read_only": True}, + } + + def validate_name(self, value): + """Validate that the name is unique within the tenant.""" + tenant_id = self.context.get("tenant_id") + if MuteRule.objects.filter(tenant_id=tenant_id, name=value).exists(): + raise ValidationError("A mute rule with this name already exists.") + return value + + def validate_finding_ids(self, value): + """Validate that all finding IDs exist and belong to the tenant.""" + if not value: + raise ValidationError("At least one finding_id must be provided.") + + tenant_id = self.context.get("tenant_id") + + # Check that all findings exist and belong to this tenant + findings = Finding.all_objects.filter(tenant_id=tenant_id, id__in=value) + found_ids = set(findings.values_list("id", flat=True)) + provided_ids = set(value) + + missing_ids = provided_ids - found_ids + if missing_ids: + raise ValidationError( + f"The following finding IDs do not exist or do not belong to your tenant: {missing_ids}" + ) + + return value + + def validate(self, data): + """Validate the entire mute rule, including overlap detection.""" + data = super().validate(data) + + tenant_id = self.context.get("tenant_id") + finding_ids = data.get("finding_ids", []) + + if not finding_ids: + return data + + # Convert finding IDs to UIDs (deduplicate in case multiple findings have same UID) + findings = Finding.all_objects.filter(id__in=finding_ids, tenant_id=tenant_id) + finding_uids = list(set(findings.values_list("uid", flat=True))) + + # Check for overlaps with existing enabled rules + existing_rules = MuteRule.objects.filter(tenant_id=tenant_id, enabled=True) + + for rule in existing_rules: + overlap = set(finding_uids) & set(rule.finding_uids) + if overlap: + raise ConflictException( + detail=f"The following finding UIDs are already muted by rule '{rule.name}': {overlap}" + ) + + # Store finding_uids in validated_data for create + data["finding_uids"] = finding_uids + + return data + + def create(self, validated_data): + """Create a new mute rule and set created_by.""" + # Remove finding_ids from validated_data (we've already converted to finding_uids) + validated_data.pop("finding_ids", None) + + # Set created_by to the current user + request = self.context.get("request") + if request and hasattr(request, "user"): + validated_data["created_by"] = request.user + + return super().create(validated_data) + + +class MuteRuleUpdateSerializer(BaseWriteSerializer): + """ + Serializer for updating MuteRule instances. + """ + + class Meta: + model = MuteRule + fields = [ + "id", + "name", + "reason", + "enabled", + ] + extra_kwargs = { + "id": {"read_only": True}, + "name": {"required": False}, + "reason": {"required": False}, + "enabled": {"required": False}, + } + + def validate_name(self, value): + """Validate that the name is unique within the tenant, excluding current instance.""" + tenant_id = self.context.get("tenant_id") + if ( + MuteRule.objects.filter(tenant_id=tenant_id, name=value) + .exclude(id=self.instance.id) + .exists() + ): + raise ValidationError("A mute rule with this name already exists.") + return value + + +# ThreatScore Snapshots + + +class ThreatScoreSnapshotSerializer(RLSSerializer): + """ + Serializer for ThreatScore snapshots. + Read-only serializer for retrieving historical ThreatScore metrics. + """ + + id = serializers.SerializerMethodField() + + class Meta: + model = ThreatScoreSnapshot + fields = [ + "id", + "inserted_at", + "scan", + "provider", + "compliance_id", + "overall_score", + "score_delta", + "section_scores", + "critical_requirements", + "total_requirements", + "passed_requirements", + "failed_requirements", + "manual_requirements", + "total_findings", + "passed_findings", + "failed_findings", + ] + extra_kwargs = { + "id": {"read_only": True}, + "inserted_at": {"read_only": True}, + "scan": {"read_only": True}, + "provider": {"read_only": True}, + "compliance_id": {"read_only": True}, + "overall_score": {"read_only": True}, + "score_delta": {"read_only": True}, + "section_scores": {"read_only": True}, + "critical_requirements": {"read_only": True}, + "total_requirements": {"read_only": True}, + "passed_requirements": {"read_only": True}, + "failed_requirements": {"read_only": True}, + "manual_requirements": {"read_only": True}, + "total_findings": {"read_only": True}, + "passed_findings": {"read_only": True}, + "failed_findings": {"read_only": True}, + } + + included_serializers = { + "scan": "api.v1.serializers.ScanIncludeSerializer", + "provider": "api.v1.serializers.ProviderIncludeSerializer", + } + + def get_id(self, obj): + if getattr(obj, "_aggregated", False): + return "n/a" + return str(obj.id) diff --git a/api/src/backend/api/v1/urls.py b/api/src/backend/api/v1/urls.py index 1e8694ef23..d879d1476b 100644 --- a/api/src/backend/api/v1/urls.py +++ b/api/src/backend/api/v1/urls.py @@ -17,7 +17,11 @@ from api.v1.views import ( InvitationAcceptViewSet, InvitationViewSet, LighthouseConfigViewSet, + LighthouseProviderConfigViewSet, + LighthouseProviderModelsViewSet, + LighthouseTenantConfigViewSet, MembershipViewSet, + MuteRuleViewSet, OverviewViewSet, ProcessorViewSet, ProviderGroupProvidersRelationshipView, @@ -34,12 +38,12 @@ from api.v1.views import ( ScheduleViewSet, SchemaView, TaskViewSet, + TenantApiKeyViewSet, TenantFinishACSView, TenantMembersViewSet, TenantViewSet, UserRoleRelationshipView, UserViewSet, - TenantApiKeyViewSet, ) router = routers.DefaultRouter(trailing_slash=False) @@ -67,6 +71,17 @@ router.register( basename="lighthouseconfiguration", ) router.register(r"api-keys", TenantApiKeyViewSet, basename="api-key") +router.register( + r"lighthouse/providers", + LighthouseProviderConfigViewSet, + basename="lighthouse-providers", +) +router.register( + r"lighthouse/models", + LighthouseProviderModelsViewSet, + basename="lighthouse-models", +) +router.register(r"mute-rules", MuteRuleViewSet, basename="mute-rule") tenants_router = routers.NestedSimpleRouter(router, r"tenants", lookup="tenant") tenants_router.register( @@ -137,6 +152,13 @@ urlpatterns = [ ), name="provider_group-providers-relationship", ), + path( + "lighthouse/configuration", + LighthouseTenantConfigViewSet.as_view( + {"get": "list", "patch": "partial_update"} + ), + name="lighthouse-configurations", + ), # API endpoint to start SAML SSO flow path( "auth/saml/initiate/", SAMLInitiateAPIView.as_view(), name="api_saml_initiate" diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 777afac907..6724565605 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -1,7 +1,12 @@ +import fnmatch import glob +import json import logging import os +from collections import defaultdict +from copy import deepcopy from datetime import datetime, timedelta, timezone +from decimal import ROUND_HALF_UP, Decimal, InvalidOperation from urllib.parse import urljoin import sentry_sdk @@ -22,9 +27,23 @@ from django.conf import settings as django_settings from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.search import SearchQuery from django.db import transaction -from django.db.models import Count, F, Prefetch, Q, Subquery, Sum +from django.db.models import ( + Case, + Count, + DecimalField, + ExpressionWrapper, + F, + IntegerField, + Max, + Prefetch, + Q, + Subquery, + Sum, + Value, + When, +) from django.db.models.functions import Coalesce -from django.http import HttpResponse +from django.http import HttpResponse, QueryDict from django.shortcuts import redirect from django.urls import reverse from django.utils.dateparse import parse_date @@ -59,11 +78,14 @@ from tasks.tasks import ( backfill_scan_resource_summaries_task, check_integration_connection_task, check_lighthouse_connection_task, + check_lighthouse_provider_connection_task, check_provider_connection_task, delete_provider_task, delete_tenant_task, jira_integration_task, + mute_historical_findings_task, perform_scan_task, + refresh_lighthouse_provider_models_task, ) from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset @@ -83,7 +105,10 @@ from api.filters import ( InvitationFilter, LatestFindingFilter, LatestResourceFilter, + LighthouseProviderConfigFilter, + LighthouseProviderModelsFilter, MembershipFilter, + MuteRuleFilter, ProcessorFilter, ProviderFilter, ProviderGroupFilter, @@ -97,6 +122,7 @@ from api.filters import ( TaskFilter, TenantApiKeyFilter, TenantFilter, + ThreatScoreSnapshotFilter, UserFilter, ) from api.models import ( @@ -105,7 +131,11 @@ from api.models import ( Integration, Invitation, LighthouseConfiguration, + LighthouseProviderConfiguration, + LighthouseProviderModels, + LighthouseTenantConfiguration, Membership, + MuteRule, Processor, Provider, ProviderGroup, @@ -126,6 +156,7 @@ from api.models import ( StateChoices, Task, TenantAPIKey, + ThreatScoreSnapshot, User, UserRoleRelationship, ) @@ -138,7 +169,7 @@ from api.utils import ( validate_invitation, ) from api.uuid_utils import datetime_to_uuid7, uuid7_start -from api.v1.mixins import PaginateByPkMixin, TaskManagementMixin +from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin from api.v1.serializers import ( ComplianceOverviewAttributesSerializer, ComplianceOverviewDetailSerializer, @@ -159,8 +190,18 @@ from api.v1.serializers import ( LighthouseConfigCreateSerializer, LighthouseConfigSerializer, LighthouseConfigUpdateSerializer, + LighthouseProviderConfigCreateSerializer, + LighthouseProviderConfigSerializer, + LighthouseProviderConfigUpdateSerializer, + LighthouseProviderModelsSerializer, + LighthouseTenantConfigSerializer, + LighthouseTenantConfigUpdateSerializer, MembershipSerializer, + MuteRuleCreateSerializer, + MuteRuleSerializer, + MuteRuleUpdateSerializer, OverviewFindingSerializer, + OverviewProviderCountSerializer, OverviewProviderSerializer, OverviewServiceSerializer, OverviewSeveritySerializer, @@ -196,6 +237,7 @@ from api.v1.serializers import ( TenantApiKeySerializer, TenantApiKeyUpdateSerializer, TenantSerializer, + ThreatScoreSnapshotSerializer, TokenRefreshSerializer, TokenSerializer, TokenSocialLoginSerializer, @@ -306,7 +348,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.14.0" + spectacular_settings.VERSION = "1.15.0" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -398,6 +440,12 @@ class SchemaView(SpectacularAPIView): "description": "Endpoints for API keys management. These can be used as an alternative to JWT " "authorization.", }, + { + "name": "Mute Rules", + "description": "Endpoints for simple mute rules management. These can be used as an alternative to the" + " Mutelist Processor if you need to mute specific findings across your tenant with a " + "specific reason.", + }, ] return super().get(request, *args, **kwargs) @@ -666,36 +714,48 @@ class TenantFinishACSView(FinishACSView): .get(email_domain=email_domain) .tenant ) - role_name = ( - extra.get("userType", ["no_permissions"])[0].strip() - if extra.get("userType") - else "no_permissions" + + # Check if tenant has only one user with MANAGE_ACCOUNT role + users_with_manage_account = ( + UserRoleRelationship.objects.using(MainRouter.admin_db) + .filter(role__manage_account=True, tenant_id=tenant.id) + .values("user") + .distinct() + .count() ) - try: - role = Role.objects.using(MainRouter.admin_db).get( - name=role_name, tenant=tenant + + # Only apply role mapping from userType if tenant does NOT have exactly one user with MANAGE_ACCOUNT + if users_with_manage_account != 1: + role_name = ( + extra.get("userType", ["no_permissions"])[0].strip() + if extra.get("userType") + else "no_permissions" ) - except Role.DoesNotExist: - role = Role.objects.using(MainRouter.admin_db).create( - name=role_name, - tenant=tenant, - manage_users=False, - manage_account=False, - manage_billing=False, - manage_providers=False, - manage_integrations=False, - manage_scans=False, - unlimited_visibility=False, + try: + role = Role.objects.using(MainRouter.admin_db).get( + name=role_name, tenant=tenant + ) + except Role.DoesNotExist: + role = Role.objects.using(MainRouter.admin_db).create( + name=role_name, + tenant=tenant, + manage_users=False, + manage_account=False, + manage_billing=False, + manage_providers=False, + manage_integrations=False, + manage_scans=False, + unlimited_visibility=False, + ) + UserRoleRelationship.objects.using(MainRouter.admin_db).filter( + user=user, + tenant_id=tenant.id, + ).delete() + UserRoleRelationship.objects.using(MainRouter.admin_db).create( + user=user, + role=role, + tenant_id=tenant.id, ) - UserRoleRelationship.objects.using(MainRouter.admin_db).filter( - user=user, - tenant_id=tenant.id, - ).delete() - UserRoleRelationship.objects.using(MainRouter.admin_db).create( - user=user, - role=role, - tenant_id=tenant.id, - ) membership, _ = Membership.objects.using(MainRouter.admin_db).get_or_create( user=user, tenant=tenant, @@ -1404,7 +1464,7 @@ class ProviderGroupProvidersRelationshipView(RelationshipView, BaseRLSViewSet): ) @method_decorator(CACHE_DECORATOR, name="list") @method_decorator(CACHE_DECORATOR, name="retrieve") -class ProviderViewSet(BaseRLSViewSet): +class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): queryset = Provider.objects.all() serializer_class = ProviderSerializer http_method_names = ["get", "post", "patch", "delete"] @@ -1581,6 +1641,25 @@ class ProviderViewSet(BaseRLSViewSet): }, request=None, ), + threatscore=extend_schema( + tags=["Scan"], + summary="Retrieve threatscore report", + description="Download a specific threatscore report (e.g., 'prowler_threatscore_aws') as a PDF file.", + request=None, + responses={ + 200: OpenApiResponse( + description="PDF file containing the threatscore report" + ), + 202: OpenApiResponse(description="The task is in progress"), + 401: OpenApiResponse( + description="API key missing or user not Authenticated" + ), + 403: OpenApiResponse(description="There is a problem with credentials"), + 404: OpenApiResponse( + description="The scan has no threatscore reports, or the threatscore report generation task has not started yet" + ), + }, + ), ) @method_decorator(CACHE_DECORATOR, name="list") @method_decorator(CACHE_DECORATOR, name="retrieve") @@ -1637,6 +1716,9 @@ class ScanViewSet(BaseRLSViewSet): if hasattr(self, "response_serializer_class"): return self.response_serializer_class return ScanComplianceReportSerializer + elif self.action == "threatscore": + if hasattr(self, "response_serializer_class"): + return self.response_serializer_class return super().get_serializer_class() def partial_update(self, request, *args, **kwargs): @@ -1741,7 +1823,18 @@ class ScanViewSet(BaseRLSViewSet): status=status.HTTP_502_BAD_GATEWAY, ) contents = resp.get("Contents", []) - keys = [obj["Key"] for obj in contents if obj["Key"].endswith(suffix)] + keys = [] + for obj in contents: + key = obj["Key"] + key_basename = os.path.basename(key) + if any(ch in suffix for ch in ("*", "?", "[")): + if fnmatch.fnmatch(key_basename, suffix): + keys.append(key) + elif key_basename == suffix: + keys.append(key) + elif key.endswith(suffix): + # Backward compatibility if suffix already includes directories + keys.append(key) if not keys: return Response( { @@ -1868,6 +1961,45 @@ class ScanViewSet(BaseRLSViewSet): content, filename = loader return self._serve_file(content, filename, "text/csv") + @action( + detail=True, + methods=["get"], + url_name="threatscore", + ) + def threatscore(self, request, pk=None): + scan = self.get_object() + running_resp = self._get_task_status(scan) + if running_resp: + return running_resp + + if not scan.output_location: + return Response( + { + "detail": "The scan has no reports, or the threatscore report generation task has not started yet." + }, + status=status.HTTP_404_NOT_FOUND, + ) + + if scan.output_location.startswith("s3://"): + bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "") + key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/") + prefix = os.path.join( + os.path.dirname(key_prefix), + "threatscore", + "*_threatscore_report.pdf", + ) + loader = self._load_file(prefix, s3=True, bucket=bucket, list_objects=True) + else: + base = os.path.dirname(scan.output_location) + pattern = os.path.join(base, "threatscore", "*_threatscore_report.pdf") + loader = self._load_file(pattern, s3=False) + + if isinstance(loader, Response): + return loader + + content, filename = loader + return self._serve_file(content, filename, "application/pdf") + def create(self, request, *args, **kwargs): input_serializer = self.get_serializer(data=request.data) input_serializer.is_valid(raise_exception=True) @@ -3437,20 +3569,16 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): ) filtered_queryset = self.filter_queryset(self.get_queryset()) - all_requirements = ( - filtered_queryset.values( - "requirement_id", - "framework", - "version", - "description", - "passed_findings", - "total_findings", - ) - .distinct() - .annotate( - total_instances=Count("id"), - manual_count=Count("id", filter=Q(requirement_status="MANUAL")), - ) + all_requirements = filtered_queryset.values( + "requirement_id", + "framework", + "version", + "description", + ).annotate( + total_instances=Count("id"), + manual_count=Count("id", filter=Q(requirement_status="MANUAL")), + passed_findings_sum=Sum("passed_findings"), + total_findings_sum=Sum("total_findings"), ) passed_instances = ( @@ -3469,8 +3597,8 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): total_instances = requirement["total_instances"] passed_count = passed_counts.get(requirement_id, 0) is_manual = requirement["manual_count"] == total_instances - passed_findings = requirement["passed_findings"] - total_findings = requirement["total_findings"] + passed_findings = requirement["passed_findings_sum"] or 0 + total_findings = requirement["total_findings_sum"] or 0 if is_manual: requirement_status = "MANUAL" elif passed_count == total_instances: @@ -3596,6 +3724,13 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): "each provider are considered in the aggregation to ensure accurate and up-to-date insights." ), ), + providers_count=extend_schema( + summary="Get provider counts grouped by type", + description=( + "Retrieve the number of providers grouped by provider type. " + "This endpoint counts every provider in the tenant, including those without completed scans." + ), + ), findings=extend_schema( summary="Get aggregated findings data", description=( @@ -3647,12 +3782,16 @@ class OverviewViewSet(BaseRLSViewSet): def get_serializer_class(self): if self.action == "providers": return OverviewProviderSerializer + elif self.action == "providers_count": + return OverviewProviderCountSerializer elif self.action == "findings": return OverviewFindingSerializer elif self.action == "findings_severity": return OverviewSeveritySerializer elif self.action == "services": return OverviewServiceSerializer + elif self.action == "threatscore": + return ThreatScoreSnapshotSerializer return super().get_serializer_class() def get_filterset_class(self): @@ -3695,10 +3834,7 @@ class OverviewViewSet(BaseRLSViewSet): findings_aggregated = ( queryset.filter(scan_id__in=latest_scan_ids) - .values( - "scan__provider_id", - provider=F("scan__provider__provider"), - ) + .values(provider=F("scan__provider__provider")) .annotate( findings_passed=Coalesce(Sum("_pass"), 0), findings_failed=Coalesce(Sum("fail"), 0), @@ -3707,13 +3843,16 @@ class OverviewViewSet(BaseRLSViewSet): ) ) - resources_aggregated = ( - Resource.all_objects.filter(tenant_id=tenant_id) - .values("provider_id") - .annotate(total_resources=Count("id")) - ) + resources_queryset = Resource.all_objects.filter(tenant_id=tenant_id) + if hasattr(self, "allowed_providers"): + resources_queryset = resources_queryset.filter( + provider__in=self.allowed_providers + ) + resources_aggregated = resources_queryset.values( + provider_type=F("provider__provider") + ).annotate(total_resources=Count("id")) resource_map = { - row["provider_id"]: row["total_resources"] for row in resources_aggregated + row["provider_type"]: row["total_resources"] for row in resources_aggregated } overview = [] @@ -3721,7 +3860,7 @@ class OverviewViewSet(BaseRLSViewSet): overview.append( { "provider": row["provider"], - "total_resources": resource_map.get(row["scan__provider_id"], 0), + "total_resources": resource_map.get(row["provider"], 0), "total_findings": row["total_findings"], "findings_passed": row["findings_passed"], "findings_failed": row["findings_failed"], @@ -3734,6 +3873,36 @@ class OverviewViewSet(BaseRLSViewSet): status=status.HTTP_200_OK, ) + @action( + detail=False, + methods=["get"], + url_path="providers/count", + url_name="providers-count", + ) + def providers_count(self, request): + tenant_id = self.request.tenant_id + providers_qs = Provider.objects.filter(tenant_id=tenant_id) + + if hasattr(self, "allowed_providers"): + allowed_ids = list(self.allowed_providers.values_list("id", flat=True)) + if not allowed_ids: + overview = [] + return Response( + self.get_serializer(overview, many=True).data, + status=status.HTTP_200_OK, + ) + providers_qs = providers_qs.filter(id__in=allowed_ids) + + overview = ( + providers_qs.values("provider") + .annotate(count=Count("id")) + .order_by("provider") + ) + return Response( + self.get_serializer(overview, many=True).data, + status=status.HTTP_200_OK, + ) + @action(detail=False, methods=["get"], url_name="findings") def findings(self, request): tenant_id = self.request.tenant_id @@ -3864,6 +4033,332 @@ class OverviewViewSet(BaseRLSViewSet): return Response(serializer.data, status=status.HTTP_200_OK) + @extend_schema( + summary="Get ThreatScore snapshots", + description=( + "Retrieve ThreatScore metrics. By default, returns the latest snapshot for each provider. " + "Use snapshot_id to retrieve a specific historical snapshot." + ), + tags=["Overviews"], + parameters=[ + OpenApiParameter( + name="snapshot_id", + type=OpenApiTypes.UUID, + location=OpenApiParameter.QUERY, + description="Retrieve a specific snapshot by ID. If not provided, returns latest snapshots.", + ), + OpenApiParameter( + name="provider_id", + type=OpenApiTypes.UUID, + location=OpenApiParameter.QUERY, + description="Filter by specific provider ID", + ), + OpenApiParameter( + name="provider_id__in", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Filter by multiple provider IDs (comma-separated UUIDs)", + ), + OpenApiParameter( + name="provider_type", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Filter by provider type (aws, azure, gcp, etc.)", + ), + OpenApiParameter( + name="provider_type__in", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Filter by multiple provider types (comma-separated)", + ), + ], + ) + @action(detail=False, methods=["get"], url_name="threatscore") + def threatscore(self, request): + """ + Get ThreatScore snapshots. + + Default behavior: Returns the latest snapshot for each provider. + With snapshot_id: Returns the specific snapshot requested. + """ + tenant_id = self.request.tenant_id + snapshot_id = request.query_params.get("snapshot_id") + + # Base queryset with RLS + base_queryset = ThreatScoreSnapshot.objects.filter(tenant_id=tenant_id) + + # Apply RBAC filtering + if hasattr(self, "allowed_providers"): + base_queryset = base_queryset.filter(provider__in=self.allowed_providers) + + # Case 1: Specific snapshot requested + if snapshot_id: + try: + snapshot = base_queryset.get(id=snapshot_id) + serializer = ThreatScoreSnapshotSerializer( + snapshot, context={"request": request} + ) + return Response(serializer.data, status=status.HTTP_200_OK) + except ThreatScoreSnapshot.DoesNotExist: + raise NotFound(detail="ThreatScore snapshot not found") + + # Case 2: Latest snapshot per provider (default) + # Apply filters manually: this @action is outside the standard list endpoint flow, + # so DRF's filter backends don't execute and we must flatten JSON:API params ourselves. + normalized_params = QueryDict(mutable=True) + for param_key, values in request.query_params.lists(): + normalized_key = param_key + if param_key.startswith("filter[") and param_key.endswith("]"): + normalized_key = param_key[7:-1] + if normalized_key == "snapshot_id": + continue + normalized_params.setlist(normalized_key, values) + + filterset = ThreatScoreSnapshotFilter(normalized_params, queryset=base_queryset) + filtered_queryset = filterset.qs + + # Get distinct provider IDs from filtered queryset + # Pick the latest snapshot per provider using Postgres DISTINCT ON pattern. + # This avoids issuing one query per provider (N+1) when the filtered dataset is large. + latest_snapshot_ids = list( + filtered_queryset.order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + latest_snapshot_map = { + snapshot.id: snapshot + for snapshot in filtered_queryset.filter(id__in=latest_snapshot_ids) + } + latest_snapshots = [ + latest_snapshot_map[snapshot_id] + for snapshot_id in latest_snapshot_ids + if snapshot_id in latest_snapshot_map + ] + + if len(latest_snapshots) <= 1: + serializer = ThreatScoreSnapshotSerializer( + latest_snapshots, many=True, context={"request": request} + ) + return Response(serializer.data, status=status.HTTP_200_OK) + + snapshot_ids = [ + snapshot.id for snapshot in latest_snapshots if snapshot and snapshot.id + ] + aggregated_snapshot = self._build_threatscore_overview_snapshot( + snapshot_ids, tenant_id + ) + serializer = ThreatScoreSnapshotSerializer( + [aggregated_snapshot], many=True, context={"request": request} + ) + return Response(serializer.data, status=status.HTTP_200_OK) + + def _build_threatscore_overview_snapshot(self, snapshot_ids, tenant_id): + """ + Aggregate the latest snapshots into a single overview snapshot for the tenant. + """ + if not snapshot_ids: + raise ValueError( + "Snapshot id list cannot be empty when aggregating threatscore overview" + ) + + base_queryset = ThreatScoreSnapshot.objects.filter( + tenant_id=tenant_id, id__in=snapshot_ids + ) + + annotated_queryset = ( + base_queryset.annotate( + active_requirements=ExpressionWrapper( + F("total_requirements") - F("manual_requirements"), + output_field=IntegerField(), + ) + ) + .annotate( + weight=Case( + When(total_findings__gt=0, then=F("total_findings")), + When( + active_requirements__gt=0, + then=F("active_requirements"), + ), + default=Value(1, output_field=IntegerField()), + output_field=IntegerField(), + ) + ) + .order_by() + ) + + aggregated_metrics = annotated_queryset.aggregate( + total_requirements=Sum("total_requirements"), + passed_requirements=Sum("passed_requirements"), + failed_requirements=Sum("failed_requirements"), + manual_requirements=Sum("manual_requirements"), + total_findings=Sum("total_findings"), + passed_findings=Sum("passed_findings"), + failed_findings=Sum("failed_findings"), + weighted_overall_sum=Sum( + ExpressionWrapper( + F("overall_score") * F("weight"), + output_field=DecimalField(max_digits=14, decimal_places=4), + ) + ), + overall_weight=Sum("weight"), + unweighted_overall_sum=Sum("overall_score"), + weighted_delta_sum=Sum( + Case( + When( + score_delta__isnull=False, + then=ExpressionWrapper( + F("score_delta") * F("weight"), + output_field=DecimalField(max_digits=14, decimal_places=4), + ), + ), + default=Value( + Decimal("0"), + output_field=DecimalField(max_digits=14, decimal_places=4), + ), + output_field=DecimalField(max_digits=14, decimal_places=4), + ) + ), + delta_weight=Sum( + Case( + When(score_delta__isnull=False, then=F("weight")), + default=Value(0, output_field=IntegerField()), + output_field=IntegerField(), + ) + ), + provider_count=Count("id"), + latest_inserted_at=Max("inserted_at"), + ) + + total_requirements = aggregated_metrics["total_requirements"] or 0 + passed_requirements = aggregated_metrics["passed_requirements"] or 0 + failed_requirements = aggregated_metrics["failed_requirements"] or 0 + manual_requirements = aggregated_metrics["manual_requirements"] or 0 + total_findings = aggregated_metrics["total_findings"] or 0 + passed_findings = aggregated_metrics["passed_findings"] or 0 + failed_findings = aggregated_metrics["failed_findings"] or 0 + + weighted_overall_sum = aggregated_metrics["weighted_overall_sum"] + if weighted_overall_sum is None: + weighted_overall_sum = Decimal("0") + unweighted_overall_sum = aggregated_metrics["unweighted_overall_sum"] + if unweighted_overall_sum is None: + unweighted_overall_sum = Decimal("0") + + overall_weight = aggregated_metrics["overall_weight"] or 0 + provider_count = aggregated_metrics["provider_count"] or 0 + + weighted_delta_sum = aggregated_metrics["weighted_delta_sum"] + if weighted_delta_sum is None: + weighted_delta_sum = Decimal("0") + delta_weight = aggregated_metrics["delta_weight"] or 0 + + if overall_weight > 0: + overall_score = (weighted_overall_sum / Decimal(overall_weight)).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + elif provider_count > 0: + overall_score = (unweighted_overall_sum / Decimal(provider_count)).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + else: + overall_score = Decimal("0.00") + + if delta_weight > 0: + score_delta = (weighted_delta_sum / Decimal(delta_weight)).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + else: + score_delta = None + + section_weighted_sums = defaultdict(lambda: Decimal("0")) + section_weights = defaultdict(lambda: Decimal("0")) + + combined_critical_requirements = {} + + snapshots_with_weight = list(annotated_queryset) + + for snapshot in snapshots_with_weight: + weight_value = getattr(snapshot, "weight", None) + try: + weight_decimal = Decimal(weight_value) + except (InvalidOperation, TypeError): + weight_decimal = Decimal("1") + if weight_decimal <= 0: + weight_decimal = Decimal("1") + + section_scores = snapshot.section_scores or {} + for section, score in section_scores.items(): + try: + score_decimal = Decimal(str(score)) + except (InvalidOperation, TypeError): + continue + section_weighted_sums[section] += score_decimal * weight_decimal + section_weights[section] += weight_decimal + + for requirement in snapshot.critical_requirements or []: + key = requirement.get("requirement_id") or requirement.get("title") + if not key: + continue + existing = combined_critical_requirements.get(key) + + def requirement_sort_key(item): + return ( + item.get("risk_level") or 0, + item.get("weight") or 0, + ) + + if existing is None or requirement_sort_key( + requirement + ) > requirement_sort_key(existing): + combined_critical_requirements[key] = deepcopy(requirement) + + aggregated_section_scores = {} + for section, total in section_weighted_sums.items(): + weight_total = section_weights[section] + if weight_total > 0: + aggregated_section_scores[section] = str( + (total / weight_total).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + ) + + aggregated_section_scores = dict(sorted(aggregated_section_scores.items())) + + aggregated_critical_requirements = sorted( + combined_critical_requirements.values(), + key=lambda item: ( + item.get("risk_level") or 0, + item.get("weight") or 0, + ), + reverse=True, + ) + + aggregated_snapshot = ThreatScoreSnapshot( + tenant_id=tenant_id, + scan=None, + provider=None, + compliance_id="prowler_threatscore_overview", + overall_score=overall_score, + score_delta=score_delta, + section_scores=aggregated_section_scores, + critical_requirements=aggregated_critical_requirements, + total_requirements=total_requirements, + passed_requirements=passed_requirements, + failed_requirements=failed_requirements, + manual_requirements=manual_requirements, + total_findings=total_findings, + passed_findings=passed_findings, + failed_findings=failed_findings, + ) + + latest_inserted_at = aggregated_metrics["latest_inserted_at"] + if latest_inserted_at is not None: + aggregated_snapshot.inserted_at = latest_inserted_at + + aggregated_snapshot._aggregated = True + + return aggregated_snapshot + @extend_schema(tags=["Schedule"]) @extend_schema_view( @@ -4096,21 +4591,25 @@ class IntegrationJiraViewSet(BaseRLSViewSet): tags=["Lighthouse AI"], summary="List all Lighthouse AI configurations", description="Retrieve a list of all Lighthouse AI configurations.", + deprecated=True, ), create=extend_schema( tags=["Lighthouse AI"], summary="Create a new Lighthouse AI configuration", description="Create a new Lighthouse AI configuration with the specified details.", + deprecated=True, ), partial_update=extend_schema( tags=["Lighthouse AI"], summary="Partially update a Lighthouse AI configuration", description="Update certain fields of an existing Lighthouse AI configuration.", + deprecated=True, ), destroy=extend_schema( tags=["Lighthouse AI"], summary="Delete a Lighthouse AI configuration", description="Remove a Lighthouse AI configuration by its ID.", + deprecated=True, ), connection=extend_schema( tags=["Lighthouse AI"], @@ -4118,6 +4617,7 @@ class IntegrationJiraViewSet(BaseRLSViewSet): description="Verify the connection to the OpenAI API for a specific Lighthouse AI configuration.", request=None, responses={202: OpenApiResponse(response=TaskSerializer)}, + deprecated=True, ), ) class LighthouseConfigViewSet(BaseRLSViewSet): @@ -4168,6 +4668,255 @@ class LighthouseConfigViewSet(BaseRLSViewSet): ) +@extend_schema_view( + list=extend_schema( + tags=["Lighthouse AI"], + summary="List all LLM provider configurations", + description="Retrieve all LLM provider configurations for the current tenant", + ), + retrieve=extend_schema( + tags=["Lighthouse AI"], + summary="Retrieve LLM provider configuration", + description="Get details for a specific provider configuration in the current tenant.", + ), + create=extend_schema( + tags=["Lighthouse AI"], + summary="Create LLM provider configuration", + description="Create a per-tenant configuration for an LLM provider. Only one configuration per provider type " + "is allowed per tenant.", + ), + partial_update=extend_schema( + tags=["Lighthouse AI"], + summary="Update LLM provider configuration", + description="Partially update a provider configuration (e.g., base_url, is_active).", + ), + destroy=extend_schema( + tags=["Lighthouse AI"], + summary="Delete LLM provider configuration", + description="Delete a provider configuration. Any tenant defaults that reference this provider are cleared " + "during deletion.", + ), +) +class LighthouseProviderConfigViewSet(BaseRLSViewSet): + queryset = LighthouseProviderConfiguration.objects.all() + serializer_class = LighthouseProviderConfigSerializer + http_method_names = ["get", "post", "patch", "delete"] + filterset_class = LighthouseProviderConfigFilter + + def get_queryset(self): + if getattr(self, "swagger_fake_view", False): + return LighthouseProviderConfiguration.objects.none() + return LighthouseProviderConfiguration.objects.filter( + tenant_id=self.request.tenant_id + ) + + def get_serializer_class(self): + if self.action == "create": + return LighthouseProviderConfigCreateSerializer + elif self.action == "partial_update": + return LighthouseProviderConfigUpdateSerializer + elif self.action in ["connection", "refresh_models"]: + return TaskSerializer + return super().get_serializer_class() + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + instance = serializer.save() + + read_serializer = LighthouseProviderConfigSerializer( + instance, context=self.get_serializer_context() + ) + headers = self.get_success_headers(read_serializer.data) + return Response( + data=read_serializer.data, + status=status.HTTP_201_CREATED, + headers=headers, + ) + + def partial_update(self, request, *args, **kwargs): + instance = self.get_object() + serializer = self.get_serializer( + instance, + data=request.data, + partial=True, + context=self.get_serializer_context(), + ) + serializer.is_valid(raise_exception=True) + serializer.save() + read_serializer = LighthouseProviderConfigSerializer( + instance, context=self.get_serializer_context() + ) + return Response(data=read_serializer.data, status=status.HTTP_200_OK) + + @extend_schema( + tags=["Lighthouse AI"], + summary="Check LLM provider connection", + description="Validate provider credentials asynchronously and toggle is_active.", + request=None, + responses={202: OpenApiResponse(response=TaskSerializer)}, + ) + @action(detail=True, methods=["post"], url_name="connection") + def connection(self, request, pk=None): + instance = self.get_object() + + with transaction.atomic(): + task = check_lighthouse_provider_connection_task.delay( + provider_config_id=str(instance.id), tenant_id=self.request.tenant_id + ) + + prowler_task = Task.objects.get(id=task.id) + serializer = TaskSerializer(prowler_task) + return Response( + data=serializer.data, + status=status.HTTP_202_ACCEPTED, + headers={ + "Content-Location": reverse( + "task-detail", kwargs={"pk": prowler_task.id} + ) + }, + ) + + @extend_schema( + tags=["Lighthouse AI"], + summary="Refresh LLM models catalog", + description="Fetch available models for this provider configuration and upsert into catalog. Supports OpenAI, OpenAI-compatible, and AWS Bedrock providers.", + request=None, + responses={202: OpenApiResponse(response=TaskSerializer)}, + ) + @action( + detail=True, + methods=["post"], + url_path="refresh-models", + url_name="refresh-models", + ) + def refresh_models(self, request, pk=None): + instance = self.get_object() + + with transaction.atomic(): + task = refresh_lighthouse_provider_models_task.delay( + provider_config_id=str(instance.id), tenant_id=self.request.tenant_id + ) + + prowler_task = Task.objects.get(id=task.id) + serializer = TaskSerializer(prowler_task) + return Response( + data=serializer.data, + status=status.HTTP_202_ACCEPTED, + headers={ + "Content-Location": reverse( + "task-detail", kwargs={"pk": prowler_task.id} + ) + }, + ) + + +@extend_schema_view( + list=extend_schema( + tags=["Lighthouse AI"], + summary="Get Lighthouse AI Tenant config", + description="Retrieve current tenant-level Lighthouse AI settings. Returns a single configuration object.", + ), + partial_update=extend_schema( + tags=["Lighthouse AI"], + summary="Update Lighthouse AI Tenant config", + description="Update tenant-level settings. Validates that the default provider is configured and active and that default model IDs exist for the chosen providers. Auto-creates configuration if it doesn't exist.", + ), +) +class LighthouseTenantConfigViewSet(BaseRLSViewSet): + """ + Singleton endpoint for tenant-level Lighthouse AI configuration. + + This viewset implements a true singleton pattern: + - GET returns the single configuration object (or 404 if not found) + - PATCH updates/creates the configuration (upsert semantics) + - No ID is required in the URL + """ + + queryset = LighthouseTenantConfiguration.objects.all() + serializer_class = LighthouseTenantConfigSerializer + http_method_names = ["get", "patch"] + + def get_queryset(self): + if getattr(self, "swagger_fake_view", False): + return LighthouseTenantConfiguration.objects.none() + return LighthouseTenantConfiguration.objects.filter( + tenant_id=self.request.tenant_id + ) + + def get_serializer_class(self): + if self.action == "partial_update": + return LighthouseTenantConfigUpdateSerializer + return super().get_serializer_class() + + def get_object(self): + """Retrieve the singleton instance for the current tenant.""" + obj = LighthouseTenantConfiguration.objects.filter( + tenant_id=self.request.tenant_id + ).first() + if obj is None: + raise NotFound("Tenant Lighthouse configuration not found") + self.check_object_permissions(self.request, obj) + return obj + + def list(self, request, *args, **kwargs): + """GET endpoint for singleton - returns single object, not an array.""" + instance = self.get_object() + serializer = self.get_serializer(instance) + return Response(serializer.data) + + def partial_update(self, request, *args, **kwargs): + """PATCH endpoint for singleton - no pk required. Auto-creates if not exists.""" + # Auto-create tenant config if it doesn't exist (upsert semantics) + instance, created = LighthouseTenantConfiguration.objects.get_or_create( + tenant_id=self.request.tenant_id, + defaults={}, + ) + + # Extract attributes from JSON:API payload + try: + payload = json.loads(request.body) + attributes = payload.get("data", {}).get("attributes", {}) + except (json.JSONDecodeError, AttributeError): + raise ValidationError("Invalid JSON:API payload") + + serializer = self.get_serializer(instance, data=attributes, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + read_serializer = LighthouseTenantConfigSerializer( + instance, context=self.get_serializer_context() + ) + return Response(read_serializer.data, status=status.HTTP_200_OK) + + +@extend_schema_view( + list=extend_schema( + tags=["Lighthouse AI"], + summary="List all LLM models", + description="List available LLM models per configured provider for the current tenant.", + ), + retrieve=extend_schema( + tags=["Lighthouse AI"], + summary="Retrieve LLM model details", + description="Get details for a specific LLM model.", + ), +) +class LighthouseProviderModelsViewSet(BaseRLSViewSet): + queryset = LighthouseProviderModels.objects.all() + serializer_class = LighthouseProviderModelsSerializer + filterset_class = LighthouseProviderModelsFilter + # Expose as read-only catalog collection + http_method_names = ["get"] + + def get_queryset(self): + if getattr(self, "swagger_fake_view", False): + return LighthouseProviderModels.objects.none() + return LighthouseProviderModels.objects.filter(tenant_id=self.request.tenant_id) + + def get_serializer_class(self): + return super().get_serializer_class() + + @extend_schema_view( list=extend_schema( tags=["Processor"], @@ -4298,3 +5047,95 @@ class TenantApiKeyViewSet(BaseRLSViewSet): serializer = self.get_serializer(instance) return Response(data=serializer.data, status=status.HTTP_200_OK) + + +# MuteRules +@extend_schema_view( + list=extend_schema( + tags=["Mute Rules"], + summary="List all mute rules", + description="Retrieve a list of all mute rules with filtering options.", + ), + retrieve=extend_schema( + tags=["Mute Rules"], + summary="Retrieve a mute rule", + description="Fetch detailed information about a specific mute rule by ID.", + ), + create=extend_schema( + tags=["Mute Rules"], + summary="Create a new mute rule", + description="Create a new mute rule by providing finding IDs, name, and reason. " + "The rule will immediately mute the selected findings and launch a background task " + "to mute all historical findings with matching UIDs.", + request=MuteRuleCreateSerializer, + ), + partial_update=extend_schema( + tags=["Mute Rules"], + summary="Partially update a mute rule", + description="Update certain fields of an existing mute rule (e.g., name, reason, enabled).", + request=MuteRuleUpdateSerializer, + responses={200: MuteRuleSerializer}, + ), + destroy=extend_schema( + tags=["Mute Rules"], + summary="Delete a mute rule", + description="Remove a mute rule from the system. Note: Previously muted findings remain muted.", + ), +) +class MuteRuleViewSet(BaseRLSViewSet): + queryset = MuteRule.objects.all() + serializer_class = MuteRuleSerializer + filterset_class = MuteRuleFilter + http_method_names = ["get", "post", "patch", "delete"] + search_fields = ["name", "reason"] + ordering = ["-inserted_at"] + ordering_fields = [ + "name", + "enabled", + "inserted_at", + "updated_at", + ] + required_permissions = [Permissions.MANAGE_SCANS] + + def get_queryset(self): + queryset = MuteRule.objects.filter(tenant_id=self.request.tenant_id) + return queryset.select_related("created_by") + + def get_serializer_class(self): + if self.action == "create": + return MuteRuleCreateSerializer + elif self.action == "partial_update": + return MuteRuleUpdateSerializer + return super().get_serializer_class() + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + # Create the mute rule + mute_rule = serializer.save() + + tenant_id = str(request.tenant_id) + finding_ids = request.data.get("finding_ids", []) + + # Immediately mute the selected findings + Finding.all_objects.filter( + id__in=finding_ids, tenant_id=tenant_id, muted=False + ).update( + muted=True, + muted_at=mute_rule.inserted_at, + muted_reason=mute_rule.reason, + ) + + # Launch background task for historical muting + with transaction.atomic(): + mute_historical_findings_task.apply_async( + kwargs={"tenant_id": tenant_id, "mute_rule_id": str(mute_rule.id)} + ) + + # Return the created mute rule + serializer = self.get_serializer(mute_rule) + return Response( + data=serializer.data, + status=status.HTTP_201_CREATED, + ) diff --git a/api/src/backend/config/django/devel.py b/api/src/backend/config/django/devel.py index 825e1ce36a..12c3c384c7 100644 --- a/api/src/backend/config/django/devel.py +++ b/api/src/backend/config/django/devel.py @@ -5,24 +5,39 @@ DEBUG = env.bool("DJANGO_DEBUG", default=True) ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"]) # Database +default_db_name = env("POSTGRES_DB", default="prowler_db") +default_db_user = env("POSTGRES_USER", default="prowler_user") +default_db_password = env("POSTGRES_PASSWORD", default="prowler") +default_db_host = env("POSTGRES_HOST", default="postgres-db") +default_db_port = env("POSTGRES_PORT", default="5432") + DATABASES = { "prowler_user": { "ENGINE": "psqlextra.backend", - "NAME": env("POSTGRES_DB", default="prowler_db"), - "USER": env("POSTGRES_USER", default="prowler_user"), - "PASSWORD": env("POSTGRES_PASSWORD", default="prowler"), - "HOST": env("POSTGRES_HOST", default="postgres-db"), - "PORT": env("POSTGRES_PORT", default="5432"), + "NAME": default_db_name, + "USER": default_db_user, + "PASSWORD": default_db_password, + "HOST": default_db_host, + "PORT": default_db_port, }, "admin": { "ENGINE": "psqlextra.backend", - "NAME": env("POSTGRES_DB", default="prowler_db"), + "NAME": default_db_name, "USER": env("POSTGRES_ADMIN_USER", default="prowler"), "PASSWORD": env("POSTGRES_ADMIN_PASSWORD", default="S3cret"), - "HOST": env("POSTGRES_HOST", default="postgres-db"), - "PORT": env("POSTGRES_PORT", default="5432"), + "HOST": default_db_host, + "PORT": default_db_port, + }, + "replica": { + "ENGINE": "psqlextra.backend", + "NAME": env("POSTGRES_REPLICA_DB", default=default_db_name), + "USER": env("POSTGRES_REPLICA_USER", default=default_db_user), + "PASSWORD": env("POSTGRES_REPLICA_PASSWORD", default=default_db_password), + "HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host), + "PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port), }, } + DATABASES["default"] = DATABASES["prowler_user"] REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] = tuple( # noqa: F405 diff --git a/api/src/backend/config/django/production.py b/api/src/backend/config/django/production.py index 0f9479625c..5a4cc73044 100644 --- a/api/src/backend/config/django/production.py +++ b/api/src/backend/config/django/production.py @@ -6,22 +6,37 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0. # Database # TODO Use Django database routers https://docs.djangoproject.com/en/5.0/topics/db/multi-db/#automatic-database-routing +default_db_name = env("POSTGRES_DB") +default_db_user = env("POSTGRES_USER") +default_db_password = env("POSTGRES_PASSWORD") +default_db_host = env("POSTGRES_HOST") +default_db_port = env("POSTGRES_PORT") + DATABASES = { "prowler_user": { - "ENGINE": "django.db.backends.postgresql", - "NAME": env("POSTGRES_DB"), - "USER": env("POSTGRES_USER"), - "PASSWORD": env("POSTGRES_PASSWORD"), - "HOST": env("POSTGRES_HOST"), - "PORT": env("POSTGRES_PORT"), + "ENGINE": "psqlextra.backend", + "NAME": default_db_name, + "USER": default_db_user, + "PASSWORD": default_db_password, + "HOST": default_db_host, + "PORT": default_db_port, }, "admin": { "ENGINE": "psqlextra.backend", - "NAME": env("POSTGRES_DB"), + "NAME": default_db_name, "USER": env("POSTGRES_ADMIN_USER"), "PASSWORD": env("POSTGRES_ADMIN_PASSWORD"), - "HOST": env("POSTGRES_HOST"), - "PORT": env("POSTGRES_PORT"), + "HOST": default_db_host, + "PORT": default_db_port, + }, + "replica": { + "ENGINE": "psqlextra.backend", + "NAME": env("POSTGRES_REPLICA_DB", default=default_db_name), + "USER": env("POSTGRES_REPLICA_USER", default=default_db_user), + "PASSWORD": env("POSTGRES_REPLICA_PASSWORD", default=default_db_password), + "HOST": env("POSTGRES_REPLICA_HOST", default=default_db_host), + "PORT": env("POSTGRES_REPLICA_PORT", default=default_db_port), }, } + DATABASES["default"] = DATABASES["prowler_user"] diff --git a/api/src/backend/config/urls.py b/api/src/backend/config/urls.py index 195329d171..c307bcc310 100644 --- a/api/src/backend/config/urls.py +++ b/api/src/backend/config/urls.py @@ -1,7 +1,5 @@ -from django.contrib import admin from django.urls import include, path urlpatterns = [ - path("admin/", admin.site.urls), path("api/v1/", include("api.v1.urls")), ] diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index 5d1acf1e88..d5fa21a787 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -23,6 +23,7 @@ from api.models import ( Invitation, LighthouseConfiguration, Membership, + MuteRule, Processor, Provider, ProviderGroup, @@ -499,8 +500,29 @@ def providers_fixture(tenants_fixture): alias="m365_testing", tenant_id=tenant.id, ) + provider7 = Provider.objects.create( + provider="oraclecloud", + uid="ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", + alias="oci_testing", + tenant_id=tenant.id, + ) + provider8 = Provider.objects.create( + provider="mongodbatlas", + uid="64b1d3c0e4b03b1234567890", + alias="mongodbatlas_testing", + tenant_id=tenant.id, + ) - return provider1, provider2, provider3, provider4, provider5, provider6 + return ( + provider1, + provider2, + provider3, + provider4, + provider5, + provider6, + provider7, + provider8, + ) @pytest.fixture @@ -1419,6 +1441,34 @@ def api_keys_fixture(tenants_fixture, create_test_user): return [api_key1, api_key2, api_key3] +@pytest.fixture +def mute_rules_fixture(tenants_fixture, create_test_user, findings_fixture): + """Create test mute rules for testing.""" + tenant = tenants_fixture[0] + user = create_test_user + + # Create two mute rules: one enabled, one disabled + mute_rule1 = MuteRule.objects.create( + tenant_id=tenant.id, + name="Test Rule 1", + reason="Security exception for testing", + enabled=True, + created_by=user, + finding_uids=[findings_fixture[0].uid], + ) + + mute_rule2 = MuteRule.objects.create( + tenant_id=tenant.id, + name="Test Rule 2", + reason="Compliance exception approved", + enabled=False, + created_by=user, + finding_uids=[findings_fixture[1].uid], + ) + + return mute_rule1, mute_rule2 + + def get_authorization_header(access_token: str) -> dict: return {"Authorization": f"Bearer {access_token}"} diff --git a/api/src/backend/tasks/assets/fonts/FiraCode-Regular.ttf b/api/src/backend/tasks/assets/fonts/FiraCode-Regular.ttf new file mode 100644 index 0000000000..3a57209a97 Binary files /dev/null and b/api/src/backend/tasks/assets/fonts/FiraCode-Regular.ttf differ diff --git a/api/src/backend/tasks/assets/fonts/PlusJakartaSans-Regular.ttf b/api/src/backend/tasks/assets/fonts/PlusJakartaSans-Regular.ttf new file mode 100644 index 0000000000..f43b5f4356 Binary files /dev/null and b/api/src/backend/tasks/assets/fonts/PlusJakartaSans-Regular.ttf differ diff --git a/api/src/backend/tasks/assets/img/prowler_logo.png b/api/src/backend/tasks/assets/img/prowler_logo.png new file mode 100644 index 0000000000..b9bdc4c088 Binary files /dev/null and b/api/src/backend/tasks/assets/img/prowler_logo.png differ diff --git a/api/src/backend/tasks/jobs/export.py b/api/src/backend/tasks/jobs/export.py index 441b7a95d4..a3aee7bfc9 100644 --- a/api/src/backend/tasks/jobs/export.py +++ b/api/src/backend/tasks/jobs/export.py @@ -20,12 +20,19 @@ from prowler.lib.outputs.asff.asff import ASFF from prowler.lib.outputs.compliance.aws_well_architected.aws_well_architected import ( AWSWellArchitected, ) +from prowler.lib.outputs.compliance.c5.c5_aws import AWSC5 +from prowler.lib.outputs.compliance.c5.c5_azure import AzureC5 +from prowler.lib.outputs.compliance.c5.c5_gcp import GCPC5 +from prowler.lib.outputs.compliance.ccc.ccc_aws import CCC_AWS +from prowler.lib.outputs.compliance.ccc.ccc_azure import CCC_Azure +from prowler.lib.outputs.compliance.ccc.ccc_gcp import CCC_GCP from prowler.lib.outputs.compliance.cis.cis_aws import AWSCIS from prowler.lib.outputs.compliance.cis.cis_azure import AzureCIS from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS from prowler.lib.outputs.compliance.cis.cis_github import GithubCIS from prowler.lib.outputs.compliance.cis.cis_kubernetes import KubernetesCIS from prowler.lib.outputs.compliance.cis.cis_m365 import M365CIS +from prowler.lib.outputs.compliance.cis.cis_oraclecloud import OracleCloudCIS from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS @@ -73,13 +80,17 @@ COMPLIANCE_CLASS_MAP = { (lambda name: name.startswith("iso27001_"), AWSISO27001), (lambda name: name.startswith("kisa"), AWSKISAISMSP), (lambda name: name == "prowler_threatscore_aws", ProwlerThreatScoreAWS), + (lambda name: name == "ccc_aws", CCC_AWS), + (lambda name: name.startswith("c5_"), AWSC5), ], "azure": [ (lambda name: name.startswith("cis_"), AzureCIS), (lambda name: name == "mitre_attack_azure", AzureMitreAttack), (lambda name: name.startswith("ens_"), AzureENS), (lambda name: name.startswith("iso27001_"), AzureISO27001), + (lambda name: name == "ccc_azure", CCC_Azure), (lambda name: name == "prowler_threatscore_azure", ProwlerThreatScoreAzure), + (lambda name: name == "c5_azure", AzureC5), ], "gcp": [ (lambda name: name.startswith("cis_"), GCPCIS), @@ -87,6 +98,8 @@ COMPLIANCE_CLASS_MAP = { (lambda name: name.startswith("ens_"), GCPENS), (lambda name: name.startswith("iso27001_"), GCPISO27001), (lambda name: name == "prowler_threatscore_gcp", ProwlerThreatScoreGCP), + (lambda name: name == "ccc_gcp", CCC_GCP), + (lambda name: name == "c5_gcp", GCPC5), ], "kubernetes": [ (lambda name: name.startswith("cis_"), KubernetesCIS), @@ -100,6 +113,13 @@ COMPLIANCE_CLASS_MAP = { "github": [ (lambda name: name.startswith("cis_"), GithubCIS), ], + "iac": [ + # IaC provider doesn't have specific compliance frameworks yet + # Trivy handles its own compliance checks + ], + "oraclecloud": [ + (lambda name: name.startswith("cis_"), OracleCloudCIS), + ], } @@ -175,18 +195,21 @@ def get_s3_client(): return s3_client -def _upload_to_s3(tenant_id: str, zip_path: str, scan_id: str) -> str | None: +def _upload_to_s3( + tenant_id: str, scan_id: str, local_path: str, relative_key: str +) -> str | None: """ - Upload the specified ZIP file to an S3 bucket. - If the S3 bucket environment variables are not configured, - the function returns None without performing an upload. + Upload a local artifact to an S3 bucket under the tenant/scan prefix. + Args: - tenant_id (str): The tenant identifier, used as part of the S3 key prefix. - zip_path (str): The local file system path to the ZIP file to be uploaded. - scan_id (str): The scan identifier, used as part of the S3 key prefix. + tenant_id (str): The tenant identifier used as the first segment of the S3 key. + scan_id (str): The scan identifier used as the second segment of the S3 key. + local_path (str): Filesystem path to the artifact to upload. + relative_key (str): Object key relative to `//`. + Returns: - str: The S3 URI of the uploaded file (e.g., "s3:///") if successful. - None: If the required environment variables for the S3 bucket are not set. + str | None: S3 URI of the uploaded artifact, or None if the upload is skipped. + Raises: botocore.exceptions.ClientError: If the upload attempt to S3 fails for any reason. """ @@ -194,34 +217,26 @@ def _upload_to_s3(tenant_id: str, zip_path: str, scan_id: str) -> str | None: if not bucket: return + if not relative_key: + return + + if not os.path.isfile(local_path): + return + try: s3 = get_s3_client() - # Upload the ZIP file (outputs) to the S3 bucket - zip_key = f"{tenant_id}/{scan_id}/{os.path.basename(zip_path)}" - s3.upload_file( - Filename=zip_path, - Bucket=bucket, - Key=zip_key, - ) + s3_key = f"{tenant_id}/{scan_id}/{relative_key}" + s3.upload_file(Filename=local_path, Bucket=bucket, Key=s3_key) - # Upload the compliance directory to the S3 bucket - compliance_dir = os.path.join(os.path.dirname(zip_path), "compliance") - for filename in os.listdir(compliance_dir): - local_path = os.path.join(compliance_dir, filename) - if not os.path.isfile(local_path): - continue - file_key = f"{tenant_id}/{scan_id}/compliance/{filename}" - s3.upload_file(Filename=local_path, Bucket=bucket, Key=file_key) - - return f"s3://{base.DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET}/{zip_key}" + return f"s3://{base.DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET}/{s3_key}" except (ClientError, NoCredentialsError, ParamValidationError, ValueError) as e: logger.error(f"S3 upload failed: {str(e)}") def _generate_output_directory( output_directory, prowler_provider: object, tenant_id: str, scan_id: str -) -> tuple[str, str]: +) -> tuple[str, str, str]: """ Generate a file system path for the output directory of a prowler scan. @@ -248,6 +263,7 @@ def _generate_output_directory( >>> _generate_output_directory("/tmp", "aws", "tenant-1234", "scan-5678") '/tmp/tenant-1234/aws/scan-5678/prowler-output-2023-02-15T12:34:56', '/tmp/tenant-1234/aws/scan-5678/compliance/prowler-output-2023-02-15T12:34:56' + '/tmp/tenant-1234/aws/scan-5678/threatscore/prowler-output-2023-02-15T12:34:56' """ # Sanitize the prowler provider name to ensure it is a valid directory name prowler_provider_sanitized = re.sub(r"[^\w\-]", "-", prowler_provider) @@ -268,4 +284,10 @@ def _generate_output_directory( ) os.makedirs("/".join(compliance_path.split("/")[:-1]), exist_ok=True) - return path, compliance_path + threatscore_path = ( + f"{output_directory}/{tenant_id}/{scan_id}/threatscore/prowler-output-" + f"{prowler_provider_sanitized}-{timestamp}" + ) + os.makedirs("/".join(threatscore_path.split("/")[:-1]), exist_ok=True) + + return path, compliance_path, threatscore_path diff --git a/api/src/backend/tasks/jobs/integrations.py b/api/src/backend/tasks/jobs/integrations.py index 06e7f4e7a5..71d2185deb 100644 --- a/api/src/backend/tasks/jobs/integrations.py +++ b/api/src/backend/tasks/jobs/integrations.py @@ -5,6 +5,7 @@ from celery.utils.log import get_task_logger from config.django.base import DJANGO_FINDINGS_BATCH_SIZE from tasks.utils import batched +from api.db_router import READ_REPLICA_ALIAS, MainRouter from api.db_utils import rls_transaction from api.models import Finding, Integration, Provider from api.utils import initialize_prowler_integration, initialize_prowler_provider @@ -178,7 +179,7 @@ def get_security_hub_client_from_integration( if the connection was successful and the SecurityHub client or connection object. """ # Get the provider associated with this integration - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): provider_relationship = integration.integrationproviderrelationship_set.first() if not provider_relationship: return Connection( @@ -207,7 +208,7 @@ def get_security_hub_client_from_integration( regions_status[region] = region in connection.enabled_regions # Save regions information in the integration configuration - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=MainRouter.default_db): integration.configuration["regions"] = regions_status integration.save() @@ -222,7 +223,7 @@ def get_security_hub_client_from_integration( return True, security_hub else: # Reset regions information if connection fails - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=MainRouter.default_db): integration.configuration["regions"] = {} integration.save() @@ -289,7 +290,7 @@ def upload_security_hub_integration( has_findings = False batch_number = 0 - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): qs = ( Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id) .order_by("uid") @@ -333,8 +334,11 @@ def upload_security_hub_integration( f"Security Hub connection failed for integration {integration.id}: " f"{security_hub.error}" ) - integration.connected = False - integration.save() + with rls_transaction( + tenant_id, using=MainRouter.default_db + ): + integration.connected = False + integration.save() break # Skip this integration security_hub_client = security_hub diff --git a/api/src/backend/tasks/jobs/lighthouse_providers.py b/api/src/backend/tasks/jobs/lighthouse_providers.py new file mode 100644 index 0000000000..53f9e4dd97 --- /dev/null +++ b/api/src/backend/tasks/jobs/lighthouse_providers.py @@ -0,0 +1,452 @@ +from typing import Dict + +import boto3 +import openai +from botocore.exceptions import BotoCoreError, ClientError +from celery.utils.log import get_task_logger + +from api.models import LighthouseProviderConfiguration, LighthouseProviderModels + +logger = get_task_logger(__name__) + + +def _extract_openai_api_key( + provider_cfg: LighthouseProviderConfiguration, +) -> str | None: + """ + Safely extract the OpenAI API key from a provider configuration. + + Args: + provider_cfg (LighthouseProviderConfiguration): The provider configuration instance + containing the credentials. + + Returns: + str | None: The API key string if present and valid, otherwise None. + """ + creds = provider_cfg.credentials_decoded + if not isinstance(creds, dict): + return None + api_key = creds.get("api_key") + if not isinstance(api_key, str) or not api_key: + return None + return api_key + + +def _extract_openai_compatible_params( + provider_cfg: LighthouseProviderConfiguration, +) -> Dict[str, str] | None: + """ + Extract base_url and api_key for OpenAI-compatible providers. + """ + creds = provider_cfg.credentials_decoded + base_url = provider_cfg.base_url + if not isinstance(creds, dict): + return None + api_key = creds.get("api_key") + if not isinstance(api_key, str) or not api_key: + return None + if not isinstance(base_url, str) or not base_url: + return None + return {"base_url": base_url, "api_key": api_key} + + +def _extract_bedrock_credentials( + provider_cfg: LighthouseProviderConfiguration, +) -> Dict[str, str] | None: + """ + Safely extract AWS Bedrock credentials from a provider configuration. + + Args: + provider_cfg (LighthouseProviderConfiguration): The provider configuration instance + containing the credentials. + + Returns: + Dict[str, str] | None: Dictionary with 'access_key_id', 'secret_access_key', and + 'region' if present and valid, otherwise None. + """ + creds = provider_cfg.credentials_decoded + if not isinstance(creds, dict): + return None + + access_key_id = creds.get("access_key_id") + secret_access_key = creds.get("secret_access_key") + region = creds.get("region") + + # Validate all required fields are present and are strings + if ( + not isinstance(access_key_id, str) + or not access_key_id + or not isinstance(secret_access_key, str) + or not secret_access_key + or not isinstance(region, str) + or not region + ): + return None + + return { + "access_key_id": access_key_id, + "secret_access_key": secret_access_key, + "region": region, + } + + +def check_lighthouse_provider_connection(provider_config_id: str) -> Dict: + """ + Validate a Lighthouse provider configuration by calling the provider API and + toggle its active state accordingly. + + Args: + provider_config_id: The primary key of the `LighthouseProviderConfiguration` + to validate. + + Returns: + dict: A result dictionary with the following keys: + - "connected" (bool): Whether the provider credentials are valid. + - "error" (str | None): The error message when not connected, otherwise None. + + Side Effects: + - Updates and persists `is_active` on the `LighthouseProviderConfiguration`. + + Raises: + LighthouseProviderConfiguration.DoesNotExist: If no configuration exists with the given ID. + """ + provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id) + + try: + if ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI + ): + api_key = _extract_openai_api_key(provider_cfg) + if not api_key: + provider_cfg.is_active = False + provider_cfg.save() + return {"connected": False, "error": "API key is invalid or missing"} + + # Test connection by listing models + client = openai.OpenAI(api_key=api_key) + _ = client.models.list() + + elif ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + bedrock_creds = _extract_bedrock_credentials(provider_cfg) + if not bedrock_creds: + provider_cfg.is_active = False + provider_cfg.save() + return { + "connected": False, + "error": "AWS credentials are invalid or missing", + } + + # Test connection by listing foundation models + bedrock_client = boto3.client( + "bedrock", + aws_access_key_id=bedrock_creds["access_key_id"], + aws_secret_access_key=bedrock_creds["secret_access_key"], + region_name=bedrock_creds["region"], + ) + _ = bedrock_client.list_foundation_models() + + elif ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE + ): + params = _extract_openai_compatible_params(provider_cfg) + if not params: + provider_cfg.is_active = False + provider_cfg.save() + return { + "connected": False, + "error": "Base URL or API key is invalid or missing", + } + + # Test connection using OpenAI SDK with custom base_url + # Note: base_url should include version (e.g., https://openrouter.ai/api/v1) + client = openai.OpenAI( + api_key=params["api_key"], + base_url=params["base_url"], + ) + _ = client.models.list() + + else: + return {"connected": False, "error": "Unsupported provider type"} + + # Connection successful + provider_cfg.is_active = True + provider_cfg.save() + return {"connected": True, "error": None} + + except Exception as e: + logger.warning( + "%s connection check failed: %s", provider_cfg.provider_type, str(e) + ) + provider_cfg.is_active = False + provider_cfg.save() + return {"connected": False, "error": str(e)} + + +def _fetch_openai_models(api_key: str) -> Dict[str, str]: + """ + Fetch available models from OpenAI API. + + Args: + api_key: OpenAI API key for authentication. + + Returns: + Dict mapping model_id to model_name. For OpenAI, both are the same + as the API doesn't provide separate display names. + + Raises: + Exception: If the API call fails. + """ + client = openai.OpenAI(api_key=api_key) + models = client.models.list() + # OpenAI uses model.id for both ID and display name + return {m.id: m.id for m in getattr(models, "data", [])} + + +def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, str]: + """ + Fetch available models from an OpenAI-compatible API using the OpenAI SDK. + + Returns a mapping of model_id -> model_name. Prefers the 'name' attribute + if available (e.g., from OpenRouter), otherwise falls back to 'id'. + + Note: base_url should include version (e.g., https://openrouter.ai/api/v1) + """ + client = openai.OpenAI(api_key=api_key, base_url=base_url) + models = client.models.list() + + available_models: Dict[str, str] = {} + for model in models.data: + model_id = model.id + # Prefer provider-supplied human-friendly name when available + name = getattr(model, "name", None) + if name: + available_models[model_id] = name + else: + available_models[model_id] = model_id + + return available_models + + +def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]: + """ + Fetch available models from AWS Bedrock with entitlement verification. + + This function: + 1. Lists foundation models with TEXT modality support + 2. Lists inference profiles with TEXT modality support + 3. Verifies user has entitlement access to each model + + Args: + bedrock_creds: Dictionary with 'access_key_id', 'secret_access_key', and 'region'. + + Returns: + Dict mapping model_id to model_name for all accessible models. + + Raises: + BotoCoreError, ClientError: If AWS API calls fail. + """ + bedrock_client = boto3.client( + "bedrock", + aws_access_key_id=bedrock_creds["access_key_id"], + aws_secret_access_key=bedrock_creds["secret_access_key"], + region_name=bedrock_creds["region"], + ) + + models_to_check: Dict[str, str] = {} + + # Step 1: Get foundation models with TEXT modality + foundation_response = bedrock_client.list_foundation_models() + model_summaries = foundation_response.get("modelSummaries", []) + + for model in model_summaries: + # Check if model supports TEXT input and output modality + input_modalities = model.get("inputModalities", []) + output_modalities = model.get("outputModalities", []) + + if "TEXT" not in input_modalities or "TEXT" not in output_modalities: + continue + + model_id = model.get("modelId") + if not model_id: + continue + + inference_types = model.get("inferenceTypesSupported", []) + + # Only include models with ON_DEMAND inference support + if "ON_DEMAND" in inference_types: + models_to_check[model_id] = model["modelName"] + + # Step 2: Get inference profiles + try: + inference_profiles_response = bedrock_client.list_inference_profiles() + inference_profiles = inference_profiles_response.get( + "inferenceProfileSummaries", [] + ) + + for profile in inference_profiles: + # Check if profile supports TEXT modality + input_modalities = profile.get("inputModalities", []) + output_modalities = profile.get("outputModalities", []) + + if "TEXT" not in input_modalities or "TEXT" not in output_modalities: + continue + + profile_id = profile.get("inferenceProfileId") + if profile_id: + models_to_check[profile_id] = profile["inferenceProfileName"] + + except (BotoCoreError, ClientError) as e: + logger.info( + "Could not fetch inference profiles in %s: %s", + bedrock_creds["region"], + str(e), + ) + + # Step 3: Verify entitlement availability for each model + available_models: Dict[str, str] = {} + + for model_id, model_name in models_to_check.items(): + try: + availability = bedrock_client.get_foundation_model_availability( + modelId=model_id + ) + + entitlement = availability.get("entitlementAvailability") + + # Only include models user has access to + if entitlement == "AVAILABLE": + available_models[model_id] = model_name + else: + logger.debug( + "Skipping model %s - entitlement status: %s", model_id, entitlement + ) + + except (BotoCoreError, ClientError) as e: + logger.debug( + "Could not check availability for model %s: %s", model_id, str(e) + ) + continue + + return available_models + + +def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict: + """ + Refresh the catalog of models for a Lighthouse provider configuration. + + Fetches the current list of models from the provider, upserts entries into + `LighthouseProviderModels`, and deletes stale entries no longer returned. + + Args: + provider_config_id: The primary key of the `LighthouseProviderConfiguration` + whose models should be refreshed. + + Returns: + dict: A result dictionary with the following keys on success: + - "created" (int): Number of new model rows created. + - "updated" (int): Number of existing model rows updated. + - "deleted" (int): Number of stale model rows removed. + If an error occurs, the dictionary will contain an "error" (str) field instead. + + Raises: + LighthouseProviderConfiguration.DoesNotExist: If no configuration exists with the given ID. + """ + provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id) + fetched_models: Dict[str, str] = {} + + # Fetch models from the appropriate provider + try: + if ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI + ): + api_key = _extract_openai_api_key(provider_cfg) + if not api_key: + return { + "created": 0, + "updated": 0, + "deleted": 0, + "error": "API key is invalid or missing", + } + fetched_models = _fetch_openai_models(api_key) + + elif ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + bedrock_creds = _extract_bedrock_credentials(provider_cfg) + if not bedrock_creds: + return { + "created": 0, + "updated": 0, + "deleted": 0, + "error": "AWS credentials are invalid or missing", + } + fetched_models = _fetch_bedrock_models(bedrock_creds) + + elif ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE + ): + params = _extract_openai_compatible_params(provider_cfg) + if not params: + return { + "created": 0, + "updated": 0, + "deleted": 0, + "error": "Base URL or API key is invalid or missing", + } + fetched_models = _fetch_openai_compatible_models( + params["base_url"], params["api_key"] + ) + + else: + return { + "created": 0, + "updated": 0, + "deleted": 0, + "error": "Unsupported provider type", + } + + except Exception as e: + logger.warning( + "Unexpected error refreshing %s models: %s", + provider_cfg.provider_type, + str(e), + ) + return {"created": 0, "updated": 0, "deleted": 0, "error": str(e)} + + # Upsert models into the catalog + created = 0 + updated = 0 + + for model_id, model_name in fetched_models.items(): + obj, was_created = LighthouseProviderModels.objects.update_or_create( + tenant_id=provider_cfg.tenant_id, + provider_configuration=provider_cfg, + model_id=model_id, + defaults={ + "model_name": model_name, + "default_parameters": {}, + }, + ) + if was_created: + created += 1 + else: + updated += 1 + + # Delete stale models not present anymore + deleted, _ = ( + LighthouseProviderModels.objects.filter( + tenant_id=provider_cfg.tenant_id, provider_configuration=provider_cfg + ) + .exclude(model_id__in=fetched_models.keys()) + .delete() + ) + + return {"created": created, "updated": updated, "deleted": deleted} diff --git a/api/src/backend/tasks/jobs/muting.py b/api/src/backend/tasks/jobs/muting.py new file mode 100644 index 0000000000..6ef4d127f5 --- /dev/null +++ b/api/src/backend/tasks/jobs/muting.py @@ -0,0 +1,64 @@ +from celery.utils.log import get_task_logger +from config.django.base import DJANGO_FINDINGS_BATCH_SIZE +from tasks.utils import batched + +from api.db_utils import rls_transaction +from api.models import Finding, MuteRule + +logger = get_task_logger(__name__) + + +def mute_historical_findings(tenant_id: str, mute_rule_id: str): + """ + Mute historical findings that match the given mute rule. + + This function processes findings in batches, updating their muted status + and adding the mute reason. + + Args: + tenant_id (str): The tenant ID for RLS context + mute_rule_id (str): The ID of the mute rule to apply + + Returns: + dict: Summary of the muting operation with findings_muted count + """ + findings_muted_count = 0 + + # Get the list of UIDs to mute and the reason + with rls_transaction(tenant_id): + mute_rule = MuteRule.objects.get(id=mute_rule_id, tenant_id=tenant_id) + finding_uids = mute_rule.finding_uids + mute_reason = mute_rule.reason + muted_at = mute_rule.inserted_at + + # Query findings that match the UIDs and are not already muted + with rls_transaction(tenant_id): + findings_to_mute = Finding.objects.filter( + tenant_id=tenant_id, uid__in=finding_uids, muted=False + ) + total_findings = findings_to_mute.count() + + logger.info( + f"Processing {total_findings} findings for mute rule {mute_rule_id}" + ) + + if total_findings > 0: + for batch, is_last in batched( + findings_to_mute.iterator(), DJANGO_FINDINGS_BATCH_SIZE + ): + batch_ids = [f.id for f in batch] + updated_count = Finding.all_objects.filter( + id__in=batch_ids, tenant_id=tenant_id + ).update( + muted=True, + muted_at=muted_at, + muted_reason=mute_reason, + ) + findings_muted_count += updated_count + + logger.info(f"Muted {findings_muted_count} findings for rule {mute_rule_id}") + + return { + "findings_muted": findings_muted_count, + "rule_id": mute_rule_id, + } diff --git a/api/src/backend/tasks/jobs/report.py b/api/src/backend/tasks/jobs/report.py new file mode 100644 index 0000000000..4136cd3748 --- /dev/null +++ b/api/src/backend/tasks/jobs/report.py @@ -0,0 +1,1280 @@ +import io +import os +from collections import defaultdict +from pathlib import Path +from shutil import rmtree + +import matplotlib.pyplot as plt +from celery.utils.log import get_task_logger +from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY +from reportlab.lib import colors +from reportlab.lib.enums import TA_CENTER +from reportlab.lib.pagesizes import letter +from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet +from reportlab.lib.units import inch +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont +from reportlab.pdfgen import canvas +from reportlab.platypus import ( + Image, + PageBreak, + Paragraph, + SimpleDocTemplate, + Spacer, + Table, + TableStyle, +) +from tasks.jobs.export import _generate_output_directory, _upload_to_s3 +from tasks.jobs.threatscore import compute_threatscore_metrics +from tasks.jobs.threatscore_utils import ( + _aggregate_requirement_statistics_from_database, + _calculate_requirements_data_from_statistics, +) +from tasks.utils import batched + +from api.db_router import READ_REPLICA_ALIAS +from api.db_utils import rls_transaction +from api.models import ( + Finding, + Provider, + ScanSummary, + StatusChoices, + ThreatScoreSnapshot, +) +from api.utils import initialize_prowler_provider +from prowler.lib.check.compliance_models import Compliance +from prowler.lib.outputs.finding import Finding as FindingOutput + +pdfmetrics.registerFont( + TTFont( + "PlusJakartaSans", + os.path.join( + os.path.dirname(__file__), "../assets/fonts/PlusJakartaSans-Regular.ttf" + ), + ) +) + +pdfmetrics.registerFont( + TTFont( + "FiraCode", + os.path.join(os.path.dirname(__file__), "../assets/fonts/FiraCode-Regular.ttf"), + ) +) + +logger = get_task_logger(__name__) + + +def _create_pdf_styles() -> dict[str, ParagraphStyle]: + """ + Create and return PDF paragraph styles used throughout the report. + + Returns: + dict[str, ParagraphStyle]: A dictionary containing the following styles: + - 'title': Title style with prowler green color + - 'h1': Heading 1 style with blue color and background + - 'h2': Heading 2 style with light blue color + - 'h3': Heading 3 style for sub-headings + - 'normal': Normal text style with left indent + - 'normal_center': Normal text style without indent + """ + styles = getSampleStyleSheet() + prowler_dark_green = colors.Color(0.1, 0.5, 0.2) + + title_style = ParagraphStyle( + "CustomTitle", + parent=styles["Title"], + fontSize=24, + textColor=prowler_dark_green, + spaceAfter=20, + fontName="PlusJakartaSans", + alignment=TA_CENTER, + ) + + h1 = ParagraphStyle( + "CustomH1", + parent=styles["Heading1"], + fontSize=18, + textColor=colors.Color(0.2, 0.4, 0.6), + spaceBefore=20, + spaceAfter=12, + fontName="PlusJakartaSans", + leftIndent=0, + borderWidth=2, + borderColor=colors.Color(0.2, 0.4, 0.6), + borderPadding=8, + backColor=colors.Color(0.95, 0.97, 1.0), + ) + + h2 = ParagraphStyle( + "CustomH2", + parent=styles["Heading2"], + fontSize=14, + textColor=colors.Color(0.3, 0.5, 0.7), + spaceBefore=15, + spaceAfter=8, + fontName="PlusJakartaSans", + leftIndent=10, + borderWidth=1, + borderColor=colors.Color(0.7, 0.8, 0.9), + borderPadding=5, + backColor=colors.Color(0.98, 0.99, 1.0), + ) + + h3 = ParagraphStyle( + "CustomH3", + parent=styles["Heading3"], + fontSize=12, + textColor=colors.Color(0.4, 0.6, 0.8), + spaceBefore=10, + spaceAfter=6, + fontName="PlusJakartaSans", + leftIndent=20, + ) + + normal = ParagraphStyle( + "CustomNormal", + parent=styles["Normal"], + fontSize=10, + textColor=colors.Color(0.2, 0.2, 0.2), + spaceBefore=4, + spaceAfter=4, + leftIndent=30, + fontName="PlusJakartaSans", + ) + + normal_center = ParagraphStyle( + "CustomNormalCenter", + parent=styles["Normal"], + fontSize=10, + textColor=colors.Color(0.2, 0.2, 0.2), + fontName="PlusJakartaSans", + ) + + return { + "title": title_style, + "h1": h1, + "h2": h2, + "h3": h3, + "normal": normal, + "normal_center": normal_center, + } + + +def _create_risk_component(risk_level: int, weight: int, score: int = 0) -> Table: + """ + Create a visual risk component table for the PDF report. + + Args: + risk_level (int): The risk level (0-5), where higher values indicate higher risk. + weight (int): The weight of the risk component. + score (int): The calculated score. Defaults to 0. + + Returns: + Table: A ReportLab Table object with colored cells representing risk, weight, and score. + """ + if risk_level >= 4: + risk_color = colors.Color(0.8, 0.2, 0.2) + elif risk_level >= 3: + risk_color = colors.Color(0.9, 0.6, 0.2) + elif risk_level >= 2: + risk_color = colors.Color(0.9, 0.9, 0.2) + else: + risk_color = colors.Color(0.2, 0.8, 0.2) + + if weight <= 50: + weight_color = colors.Color(0.2, 0.8, 0.2) + elif weight <= 100: + weight_color = colors.Color(0.9, 0.9, 0.2) + else: + weight_color = colors.Color(0.8, 0.2, 0.2) + + score_color = colors.Color(0.4, 0.4, 0.4) + + data = [ + [ + "Risk Level:", + str(risk_level), + "Weight:", + str(weight), + "Score:", + str(score), + ] + ] + + table = Table( + data, + colWidths=[ + 0.8 * inch, + 0.4 * inch, + 0.6 * inch, + 0.4 * inch, + 0.5 * inch, + 0.4 * inch, + ], + ) + + table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (0, 0), colors.Color(0.9, 0.9, 0.9)), + ("BACKGROUND", (1, 0), (1, 0), risk_color), + ("TEXTCOLOR", (1, 0), (1, 0), colors.white), + ("FONTNAME", (1, 0), (1, 0), "FiraCode"), + ("BACKGROUND", (2, 0), (2, 0), colors.Color(0.9, 0.9, 0.9)), + ("BACKGROUND", (3, 0), (3, 0), weight_color), + ("TEXTCOLOR", (3, 0), (3, 0), colors.white), + ("FONTNAME", (3, 0), (3, 0), "FiraCode"), + ("BACKGROUND", (4, 0), (4, 0), colors.Color(0.9, 0.9, 0.9)), + ("BACKGROUND", (5, 0), (5, 0), score_color), + ("TEXTCOLOR", (5, 0), (5, 0), colors.white), + ("FONTNAME", (5, 0), (5, 0), "FiraCode"), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("FONTSIZE", (0, 0), (-1, -1), 10), + ("GRID", (0, 0), (-1, -1), 0.5, colors.black), + ("LEFTPADDING", (0, 0), (-1, -1), 6), + ("RIGHTPADDING", (0, 0), (-1, -1), 6), + ("TOPPADDING", (0, 0), (-1, -1), 8), + ("BOTTOMPADDING", (0, 0), (-1, -1), 8), + ] + ) + ) + + return table + + +def _create_status_component(status: str) -> Table: + """ + Create a visual status component with colored background. + + Args: + status (str): The status value (e.g., "PASS", "FAIL", "MANUAL"). + + Returns: + Table: A ReportLab Table object displaying the status with appropriate color coding. + """ + if status.upper() == "PASS": + status_color = colors.Color(0.2, 0.8, 0.2) + elif status.upper() == "FAIL": + status_color = colors.Color(0.8, 0.2, 0.2) + else: + status_color = colors.Color(0.4, 0.4, 0.4) + + data = [["State:", status.upper()]] + + table = Table(data, colWidths=[0.6 * inch, 0.8 * inch]) + + table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (0, 0), colors.Color(0.9, 0.9, 0.9)), + ("FONTNAME", (0, 0), (0, 0), "PlusJakartaSans"), + ("BACKGROUND", (1, 0), (1, 0), status_color), + ("TEXTCOLOR", (1, 0), (1, 0), colors.white), + ("FONTNAME", (1, 0), (1, 0), "FiraCode"), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("FONTSIZE", (0, 0), (-1, -1), 12), + ("GRID", (0, 0), (-1, -1), 0.5, colors.black), + ("LEFTPADDING", (0, 0), (-1, -1), 8), + ("RIGHTPADDING", (0, 0), (-1, -1), 8), + ("TOPPADDING", (0, 0), (-1, -1), 10), + ("BOTTOMPADDING", (0, 0), (-1, -1), 10), + ] + ) + ) + + return table + + +def _create_section_score_chart( + requirements_list: list[dict], attributes_by_requirement_id: dict +) -> io.BytesIO: + """ + Create a bar chart showing compliance score by section using ThreatScore formula. + + Args: + requirements_list (list[dict]): List of requirement dictionaries with status and findings data. + attributes_by_requirement_id (dict): Mapping of requirement IDs to their attributes including risk level and weight. + + Returns: + io.BytesIO: A BytesIO buffer containing the chart image in PNG format. + """ + # Define expected sections + expected_sections = [ + "1. IAM", + "2. Attack Surface", + "3. Logging and Monitoring", + "4. Encryption", + ] + + # Initialize all expected sections with default values + sections_data = { + section: { + "numerator": 0, + "denominator": 0, + "has_findings": False, + } + for section in expected_sections + } + + # Collect data from requirements + for requirement in requirements_list: + requirement_id = requirement["id"] + requirement_attributes = attributes_by_requirement_id.get(requirement_id, {}) + + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if metadata: + m = metadata[0] + section = getattr(m, "Section", "Unknown") + + # Add section if not in expected list (for flexibility) + if section not in sections_data: + sections_data[section] = { + "numerator": 0, + "denominator": 0, + "has_findings": False, + } + + # Get findings data + passed_findings = requirement["attributes"].get("passed_findings", 0) + total_findings = requirement["attributes"].get("total_findings", 0) + + if total_findings > 0: + sections_data[section]["has_findings"] = True + risk_level = getattr(m, "LevelOfRisk", 0) + weight = getattr(m, "Weight", 0) + + # Calculate using ThreatScore formula from UI + rate_i = passed_findings / total_findings + rfac_i = 1 + 0.25 * risk_level + + sections_data[section]["numerator"] += ( + rate_i * total_findings * weight * rfac_i + ) + sections_data[section]["denominator"] += ( + total_findings * weight * rfac_i + ) + + section_names = [] + compliance_percentages = [] + + for section, data in sections_data.items(): + if data["has_findings"] and data["denominator"] > 0: + compliance_percentage = (data["numerator"] / data["denominator"]) * 100 + else: + compliance_percentage = 100 # No findings = 100% (PASS) + + section_names.append(section) + compliance_percentages.append(compliance_percentage) + + # Sort alphabetically by section name + sorted_data = sorted( + zip(section_names, compliance_percentages), + key=lambda x: x[0], + ) + section_names, compliance_percentages = ( + zip(*sorted_data) if sorted_data else ([], []) + ) + + fig, ax = plt.subplots(figsize=(12, 8)) + + colors_list = [] + for percentage in compliance_percentages: + if percentage >= 80: + color = "#4CAF50" + elif percentage >= 60: + color = "#8BC34A" + elif percentage >= 40: + color = "#FFEB3B" + elif percentage >= 20: + color = "#FF9800" + else: + color = "#F44336" + colors_list.append(color) + + bars = ax.bar(section_names, compliance_percentages, color=colors_list) + + ax.set_ylabel("Compliance Score (%)", fontsize=12) + ax.set_xlabel("Section", fontsize=12) + ax.set_ylim(0, 100) + + for bar, percentage in zip(bars, compliance_percentages): + height = bar.get_height() + ax.text( + bar.get_x() + bar.get_width() / 2.0, + height + 1, + f"{percentage:.1f}%", + ha="center", + va="bottom", + fontweight="bold", + ) + + plt.xticks(rotation=45, ha="right") + + ax.grid(True, alpha=0.3, axis="y") + + plt.tight_layout() + + buffer = io.BytesIO() + plt.savefig(buffer, format="png", dpi=300, bbox_inches="tight") + buffer.seek(0) + plt.close() + + return buffer + + +def _add_pdf_footer(canvas_obj: canvas.Canvas, doc: SimpleDocTemplate) -> None: + """ + Add footer with page number and branding to each page of the PDF. + + Args: + canvas_obj (canvas.Canvas): The ReportLab canvas object for drawing. + doc (SimpleDocTemplate): The document template containing page information. + """ + width, height = doc.pagesize + page_num_text = f"Page {doc.page}" + canvas_obj.setFont("PlusJakartaSans", 9) + canvas_obj.setFillColorRGB(0.4, 0.4, 0.4) + canvas_obj.drawString(30, 20, page_num_text) + powered_text = "Powered by Prowler" + text_width = canvas_obj.stringWidth(powered_text, "PlusJakartaSans", 9) + canvas_obj.drawString(width - text_width - 30, 20, powered_text) + + +def _load_findings_for_requirement_checks( + tenant_id: str, scan_id: str, check_ids: list[str], prowler_provider +) -> dict[str, list[FindingOutput]]: + """ + Load findings for specific check IDs on-demand. + + This function loads only the findings needed for a specific set of checks, + minimizing memory usage by avoiding loading all findings at once. This is used + when generating detailed findings tables for specific requirements in the PDF. + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): The ID of the scan to retrieve findings for. + check_ids (list[str]): List of check IDs to load findings for. + prowler_provider: The initialized Prowler provider instance. + + Returns: + dict[str, list[FindingOutput]]: Dictionary mapping check_id to list of FindingOutput objects. + + Example: + { + 'aws_iam_user_mfa_enabled': [FindingOutput(...), FindingOutput(...)], + 'aws_s3_bucket_public_access': [FindingOutput(...)] + } + """ + findings_by_check_id = defaultdict(list) + + if not check_ids: + return dict(findings_by_check_id) + + logger.info(f"Loading findings for {len(check_ids)} checks on-demand") + + findings_queryset = ( + Finding.all_objects.filter( + tenant_id=tenant_id, scan_id=scan_id, check_id__in=check_ids + ) + .order_by("uid") + .iterator() + ) + + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + for batch, is_last_batch in batched( + findings_queryset, DJANGO_FINDINGS_BATCH_SIZE + ): + for finding_model in batch: + finding_output = FindingOutput.transform_api_finding( + finding_model, prowler_provider + ) + findings_by_check_id[finding_output.check_id].append(finding_output) + + total_findings_loaded = sum( + len(findings) for findings in findings_by_check_id.values() + ) + logger.info( + f"Loaded {total_findings_loaded} findings for {len(findings_by_check_id)} checks" + ) + + return dict(findings_by_check_id) + + +def generate_threatscore_report( + tenant_id: str, + scan_id: str, + compliance_id: str, + output_path: str, + provider_id: str, + only_failed: bool = True, + min_risk_level: int = 4, +) -> None: + """ + Generate a PDF compliance report based on Prowler ThreatScore framework. + + This function creates a comprehensive PDF report containing: + - Compliance overview and metadata + - Section-by-section compliance scores with charts + - Overall ThreatScore calculation + - Critical failed requirements + - Detailed findings for each requirement + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): ID of the scan executed by Prowler. + compliance_id (str): ID of the compliance framework (e.g., "prowler_threatscore_aws"). + output_path (str): Output PDF file path (e.g., "/tmp/threatscore_report.pdf"). + provider_id (str): Provider ID for the scan. + only_failed (bool): If True, only requirements with status "FAIL" will be included + in the detailed requirements section. Defaults to True. + min_risk_level (int): Minimum risk level for critical failed requirements. Defaults to 4. + + Raises: + Exception: If any error occurs during PDF generation, it will be logged and re-raised. + """ + logger.info( + f"Generating the report for the scan {scan_id} with provider {provider_id}" + ) + try: + # Get PDF styles + pdf_styles = _create_pdf_styles() + title_style = pdf_styles["title"] + h1 = pdf_styles["h1"] + h2 = pdf_styles["h2"] + h3 = pdf_styles["h3"] + normal = pdf_styles["normal"] + normal_center = pdf_styles["normal_center"] + + # Get compliance and provider information + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + provider_obj = Provider.objects.get(id=provider_id) + prowler_provider = initialize_prowler_provider(provider_obj) + provider_type = provider_obj.provider + + frameworks_bulk = Compliance.get_bulk(provider_type) + compliance_obj = frameworks_bulk[compliance_id] + compliance_framework = getattr(compliance_obj, "Framework", "N/A") + compliance_version = getattr(compliance_obj, "Version", "N/A") + compliance_name = getattr(compliance_obj, "Name", "N/A") + compliance_description = getattr(compliance_obj, "Description", "") + + # Aggregate requirement statistics from database (memory-efficient) + logger.info(f"Aggregating requirement statistics for scan {scan_id}") + requirement_statistics_by_check_id = ( + _aggregate_requirement_statistics_from_database(tenant_id, scan_id) + ) + + # Calculate requirements data using aggregated statistics + attributes_by_requirement_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + compliance_obj, requirement_statistics_by_check_id + ) + ) + + # Initialize PDF document + doc = SimpleDocTemplate( + output_path, + pagesize=letter, + title=f"Prowler ThreatScore Report - {compliance_framework}", + author="Prowler", + subject=f"Compliance Report for {compliance_framework}", + creator="Prowler Engineering Team", + keywords=f"compliance,{compliance_framework},security,framework,prowler", + ) + + elements = [] + + # Add logo + img_path = os.path.join( + os.path.dirname(__file__), "../assets/img/prowler_logo.png" + ) + logo = Image( + img_path, + width=5 * inch, + height=1 * inch, + ) + elements.append(logo) + + elements.append(Spacer(1, 0.5 * inch)) + elements.append(Paragraph("Prowler ThreatScore Report", title_style)) + elements.append(Spacer(1, 0.5 * inch)) + + # Add compliance information table + info_data = [ + ["Framework:", compliance_framework], + ["ID:", compliance_id], + ["Name:", Paragraph(compliance_name, normal_center)], + ["Version:", compliance_version], + ["Scan ID:", scan_id], + ["Description:", Paragraph(compliance_description, normal_center)], + ] + info_table = Table(info_data, colWidths=[2 * inch, 4 * inch]) + info_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (0, 5), colors.Color(0.2, 0.4, 0.6)), + ("TEXTCOLOR", (0, 0), (0, 5), colors.white), + ("FONTNAME", (0, 0), (0, 5), "FiraCode"), + ("BACKGROUND", (1, 0), (1, 5), colors.Color(0.95, 0.97, 1.0)), + ("TEXTCOLOR", (1, 0), (1, 5), colors.Color(0.2, 0.2, 0.2)), + ("FONTNAME", (1, 0), (1, 5), "PlusJakartaSans"), + ("ALIGN", (0, 0), (-1, -1), "LEFT"), + ("VALIGN", (0, 0), (-1, -1), "TOP"), + ("FONTSIZE", (0, 0), (-1, -1), 11), + ("GRID", (0, 0), (-1, -1), 1, colors.Color(0.7, 0.8, 0.9)), + ("LEFTPADDING", (0, 0), (-1, -1), 10), + ("RIGHTPADDING", (0, 0), (-1, -1), 10), + ("TOPPADDING", (0, 0), (-1, -1), 8), + ("BOTTOMPADDING", (0, 0), (-1, -1), 8), + ] + ) + ) + + elements.append(info_table) + elements.append(PageBreak()) + + # Add compliance score chart + elements.append(Paragraph("Compliance Score by Sections", h1)) + elements.append(Spacer(1, 0.2 * inch)) + + chart_buffer = _create_section_score_chart( + requirements_list, attributes_by_requirement_id + ) + chart_image = Image(chart_buffer, width=7 * inch, height=5.5 * inch) + elements.append(chart_image) + + # Calculate overall ThreatScore using the same formula as the UI + numerator = 0 + denominator = 0 + has_findings = False + + for requirement in requirements_list: + requirement_id = requirement["id"] + requirement_attributes = attributes_by_requirement_id.get( + requirement_id, {} + ) + + # Get findings data + passed_findings = requirement["attributes"].get("passed_findings", 0) + total_findings = requirement["attributes"].get("total_findings", 0) + + # Skip if no findings (avoid division by zero) + if total_findings == 0: + continue + + has_findings = True + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if metadata and len(metadata) > 0: + m = metadata[0] + risk_level = getattr(m, "LevelOfRisk", 0) + weight = getattr(m, "Weight", 0) + + # Calculate using ThreatScore formula from UI + rate_i = passed_findings / total_findings + rfac_i = 1 + 0.25 * risk_level + + numerator += rate_i * total_findings * weight * rfac_i + denominator += total_findings * weight * rfac_i + + # Calculate ThreatScore (percentualScore) + # If no findings exist, consider it 100% (PASS) + if not has_findings: + overall_compliance = 100 + elif denominator > 0: + overall_compliance = (numerator / denominator) * 100 + else: + overall_compliance = 0 + + elements.append(Spacer(1, 0.3 * inch)) + + summary_data = [ + ["ThreatScore:", f"{overall_compliance:.2f}%"], + ] + + if overall_compliance >= 80: + compliance_color = colors.Color(0.2, 0.8, 0.2) + elif overall_compliance >= 60: + compliance_color = colors.Color(0.8, 0.8, 0.2) + else: + compliance_color = colors.Color(0.8, 0.2, 0.2) + + summary_table = Table(summary_data, colWidths=[2.5 * inch, 2 * inch]) + summary_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (0, 0), colors.Color(0.1, 0.3, 0.5)), + ("TEXTCOLOR", (0, 0), (0, 0), colors.white), + ("FONTNAME", (0, 0), (0, 0), "FiraCode"), + ("FONTSIZE", (0, 0), (0, 0), 12), + ("BACKGROUND", (1, 0), (1, 0), compliance_color), + ("TEXTCOLOR", (1, 0), (1, 0), colors.white), + ("FONTNAME", (1, 0), (1, 0), "FiraCode"), + ("FONTSIZE", (1, 0), (1, 0), 16), + ("ALIGN", (0, 0), (-1, -1), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("GRID", (0, 0), (-1, -1), 1.5, colors.Color(0.5, 0.6, 0.7)), + ("LEFTPADDING", (0, 0), (-1, -1), 12), + ("RIGHTPADDING", (0, 0), (-1, -1), 12), + ("TOPPADDING", (0, 0), (-1, -1), 10), + ("BOTTOMPADDING", (0, 0), (-1, -1), 10), + ] + ) + ) + + elements.append(summary_table) + elements.append(PageBreak()) + + # Add requirements index + elements.append(Paragraph("Requirements Index", h1)) + + sections = {} + for ( + requirement_id, + requirement_attributes, + ) in attributes_by_requirement_id.items(): + meta = requirement_attributes["attributes"]["req_attributes"][0] + section = getattr(meta, "Section", "N/A") + subsection = getattr(meta, "SubSection", "N/A") + title = getattr(meta, "Title", "N/A") + + if section not in sections: + sections[section] = {} + if subsection not in sections[section]: + sections[section][subsection] = [] + + sections[section][subsection].append({"id": requirement_id, "title": title}) + + section_num = 1 + for section_name, subsections in sections.items(): + elements.append(Paragraph(f"{section_num}. {section_name}", h2)) + + subsection_num = 1 + for subsection_name, requirements in subsections.items(): + elements.append(Paragraph(f"{subsection_name}", h3)) + + req_num = 1 + for req in requirements: + elements.append(Paragraph(f"{req['id']} - {req['title']}", normal)) + req_num += 1 + + subsection_num += 1 + + section_num += 1 + elements.append(Spacer(1, 0.1 * inch)) + + elements.append(PageBreak()) + + # Add critical failed requirements section + elements.append(Paragraph("Top Requirements by Level of Risk", h1)) + elements.append(Spacer(1, 0.1 * inch)) + elements.append( + Paragraph( + f"Critical Failed Requirements (Risk Level â‰Ĩ {min_risk_level})", h2 + ) + ) + elements.append(Spacer(1, 0.2 * inch)) + + critical_failed_requirements = [] + for requirement in requirements_list: + requirement_status = requirement["attributes"]["status"] + if requirement_status == StatusChoices.FAIL: + requirement_id = requirement["id"] + metadata = ( + attributes_by_requirement_id.get(requirement_id, {}) + .get("attributes", {}) + .get("req_attributes", [{}])[0] + ) + if metadata: + risk_level = getattr(metadata, "LevelOfRisk", 0) + weight = getattr(metadata, "Weight", 0) + + if risk_level >= min_risk_level: + critical_failed_requirements.append( + { + "requirement": requirement, + "attributes": attributes_by_requirement_id[ + requirement_id + ], + "risk_level": risk_level, + "weight": weight, + "metadata": metadata, + } + ) + + critical_failed_requirements.sort( + key=lambda x: (x["risk_level"], x["weight"]), reverse=True + ) + + if not critical_failed_requirements: + elements.append( + Paragraph( + "✅ No critical failed requirements found. Great job!", normal + ) + ) + else: + elements.append( + Paragraph( + f"Found {len(critical_failed_requirements)} critical failed requirements that require immediate attention:", + normal, + ) + ) + elements.append(Spacer(1, 0.5 * inch)) + + table_data = [["Risk", "Weight", "Requirement ID", "Title", "Section"]] + + for idx, critical_failed_requirement in enumerate( + critical_failed_requirements + ): + requirement_id = critical_failed_requirement["requirement"]["id"] + risk_level = critical_failed_requirement["risk_level"] + weight = critical_failed_requirement["weight"] + title = getattr(critical_failed_requirement["metadata"], "Title", "N/A") + section = getattr( + critical_failed_requirement["metadata"], "Section", "N/A" + ) + + if len(title) > 50: + title = title[:47] + "..." + + table_data.append( + [str(risk_level), str(weight), requirement_id, title, section] + ) + + critical_table = Table( + table_data, + colWidths=[0.7 * inch, 0.9 * inch, 1.3 * inch, 3.1 * inch, 1.5 * inch], + ) + + critical_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (-1, 0), colors.Color(0.8, 0.2, 0.2)), + ("TEXTCOLOR", (0, 0), (-1, 0), colors.white), + ("FONTNAME", (0, 0), (-1, 0), "FiraCode"), + ("FONTSIZE", (0, 0), (-1, 0), 10), + ("BACKGROUND", (0, 1), (0, -1), colors.Color(0.8, 0.2, 0.2)), + ("TEXTCOLOR", (0, 1), (0, -1), colors.white), + ("FONTNAME", (0, 1), (0, -1), "FiraCode"), + ("ALIGN", (0, 1), (0, -1), "CENTER"), + ("FONTSIZE", (0, 1), (0, -1), 12), + ("ALIGN", (1, 1), (1, -1), "CENTER"), + ("FONTNAME", (1, 1), (1, -1), "FiraCode"), + ("FONTNAME", (2, 1), (2, -1), "FiraCode"), + ("FONTSIZE", (2, 1), (2, -1), 9), + ("FONTNAME", (3, 1), (-1, -1), "PlusJakartaSans"), + ("FONTSIZE", (3, 1), (-1, -1), 8), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("GRID", (0, 0), (-1, -1), 1, colors.Color(0.7, 0.7, 0.7)), + ("LEFTPADDING", (0, 0), (-1, -1), 6), + ("RIGHTPADDING", (0, 0), (-1, -1), 6), + ("TOPPADDING", (0, 0), (-1, -1), 8), + ("BOTTOMPADDING", (0, 0), (-1, -1), 8), + ( + "BACKGROUND", + (1, 1), + (-1, -1), + colors.Color(0.98, 0.98, 0.98), + ), + ] + ) + ) + + for idx, critical_failed_requirement in enumerate( + critical_failed_requirements + ): + row_idx = idx + 1 + weight = critical_failed_requirement["weight"] + + if weight >= 150: + weight_color = colors.Color(0.8, 0.2, 0.2) + elif weight >= 100: + weight_color = colors.Color(0.9, 0.6, 0.2) + else: + weight_color = colors.Color(0.9, 0.9, 0.2) + + critical_table.setStyle( + TableStyle( + [ + ("BACKGROUND", (1, row_idx), (1, row_idx), weight_color), + ("TEXTCOLOR", (1, row_idx), (1, row_idx), colors.white), + ] + ) + ) + + elements.append(critical_table) + elements.append(Spacer(1, 0.2 * inch)) + + # Get styles for warning + styles = getSampleStyleSheet() + warning_text = """ + IMMEDIATE ACTION REQUIRED:
+ These requirements have the highest risk levels and have failed compliance checks. + Please prioritize addressing these issues to improve your security posture. + """ + + warning_style = ParagraphStyle( + "Warning", + parent=styles["Normal"], + fontSize=11, + textColor=colors.Color(0.8, 0.2, 0.2), + spaceBefore=10, + spaceAfter=10, + leftIndent=20, + rightIndent=20, + fontName="PlusJakartaSans", + backColor=colors.Color(1.0, 0.95, 0.95), + borderWidth=2, + borderColor=colors.Color(0.8, 0.2, 0.2), + borderPadding=10, + ) + + elements.append(Paragraph(warning_text, warning_style)) + + elements.append(PageBreak()) + + # Add detailed requirements section + def get_weight_for_requirement(requirement_dict): + requirement_id = requirement_dict["id"] + requirement_attributes = attributes_by_requirement_id.get( + requirement_id, {} + ) + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if metadata: + return getattr(metadata[0], "Weight", 0) + return 0 + + sorted_requirements = sorted( + requirements_list, key=get_weight_for_requirement, reverse=True + ) + + if only_failed: + sorted_requirements = [ + requirement + for requirement in sorted_requirements + if requirement["attributes"]["status"] == StatusChoices.FAIL + ] + + # Collect all check IDs for requirements that will be displayed + # This allows us to load only the findings we actually need (memory optimization) + check_ids_to_load = [] + for requirement in sorted_requirements: + requirement_id = requirement["id"] + requirement_attributes = attributes_by_requirement_id.get( + requirement_id, {} + ) + check_ids = requirement_attributes.get("attributes", {}).get("checks", []) + check_ids_to_load.extend(check_ids) + + # Load findings on-demand only for the checks that will be displayed + logger.info( + f"Loading findings on-demand for {len(sorted_requirements)} requirements" + ) + findings_by_check_id = _load_findings_for_requirement_checks( + tenant_id, scan_id, check_ids_to_load, prowler_provider + ) + + for requirement in sorted_requirements: + requirement_id = requirement["id"] + requirement_attributes = attributes_by_requirement_id.get( + requirement_id, {} + ) + requirement_description = requirement["attributes"]["description"] + requirement_status = requirement["attributes"]["status"] + + elements.append( + Paragraph( + f"{requirement_id}: {requirement_attributes.get('description', requirement_description)}", + h1, + ) + ) + + status_component = _create_status_component(requirement_status) + elements.append(status_component) + elements.append(Spacer(1, 0.1 * inch)) + + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if metadata and len(metadata) > 0: + m = metadata[0] + elements.append(Paragraph("Title: ", h3)) + elements.append(Paragraph(f"{getattr(m, 'Title', 'N/A')}", normal)) + elements.append(Paragraph("Section: ", h3)) + elements.append(Paragraph(f"{getattr(m, 'Section', 'N/A')}", normal)) + elements.append(Paragraph("SubSection: ", h3)) + elements.append(Paragraph(f"{getattr(m, 'SubSection', 'N/A')}", normal)) + elements.append(Paragraph("Description: ", h3)) + elements.append( + Paragraph(f"{getattr(m, 'AttributeDescription', 'N/A')}", normal) + ) + elements.append(Paragraph("Additional Information: ", h3)) + elements.append( + Paragraph(f"{getattr(m, 'AdditionalInformation', 'N/A')}", normal) + ) + elements.append(Spacer(1, 0.1 * inch)) + + risk_level = getattr(m, "LevelOfRisk", 0) + weight = getattr(m, "Weight", 0) + + if requirement_status == StatusChoices.PASS: + score = risk_level * weight + else: + score = 0 + + risk_component = _create_risk_component(risk_level, weight, score) + elements.append(risk_component) + elements.append(Spacer(1, 0.1 * inch)) + + # Get findings for this requirement's checks (loaded on-demand earlier) + requirement_check_ids = requirement_attributes.get("attributes", {}).get( + "checks", [] + ) + for check_id in requirement_check_ids: + elements.append(Paragraph(f"Check: {check_id}", h2)) + elements.append(Spacer(1, 0.1 * inch)) + + # Get findings for this check (already loaded on-demand) + check_findings = findings_by_check_id.get(check_id, []) + + if not check_findings: + elements.append( + Paragraph("- No information for this finding currently", normal) + ) + else: + findings_table_data = [ + [ + "Finding", + "Resource name", + "Severity", + "Status", + "Region", + ] + ] + for finding_output in check_findings: + check_metadata = getattr(finding_output, "metadata", {}) + finding_title = getattr( + check_metadata, + "CheckTitle", + getattr(finding_output, "check_id", ""), + ) + resource_name = getattr(finding_output, "resource_name", "") + if not resource_name: + resource_name = getattr(finding_output, "resource_uid", "") + severity = getattr(check_metadata, "Severity", "").capitalize() + finding_status = getattr(finding_output, "status", "").upper() + region = getattr(finding_output, "region", "global") + + findings_table_data.append( + [ + Paragraph(finding_title, normal_center), + Paragraph(resource_name, normal_center), + Paragraph(severity, normal_center), + Paragraph(finding_status, normal_center), + Paragraph(region, normal_center), + ] + ) + findings_table = Table( + findings_table_data, + colWidths=[ + 2.5 * inch, + 3 * inch, + 0.9 * inch, + 0.9 * inch, + 0.9 * inch, + ], + ) + findings_table.setStyle( + TableStyle( + [ + ( + "BACKGROUND", + (0, 0), + (-1, 0), + colors.Color(0.2, 0.4, 0.6), + ), + ("TEXTCOLOR", (0, 0), (-1, 0), colors.white), + ("FONTNAME", (0, 0), (-1, 0), "FiraCode"), + ("ALIGN", (0, 0), (0, 0), "CENTER"), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("FONTSIZE", (0, 0), (-1, -1), 9), + ( + "GRID", + (0, 0), + (-1, -1), + 0.1, + colors.Color(0.7, 0.8, 0.9), + ), + ("LEFTPADDING", (0, 0), (0, 0), 0), + ("RIGHTPADDING", (0, 0), (0, 0), 0), + ("TOPPADDING", (0, 0), (-1, -1), 4), + ("BOTTOMPADDING", (0, 0), (-1, -1), 4), + ] + ) + ) + elements.append(findings_table) + elements.append(Spacer(1, 0.1 * inch)) + + elements.append(PageBreak()) + + # Build the PDF + doc.build(elements, onFirstPage=_add_pdf_footer, onLaterPages=_add_pdf_footer) + except Exception as e: + logger.info( + f"Error building the document, line {e.__traceback__.tb_lineno} -- {e}" + ) + raise e + + +def generate_threatscore_report_job( + tenant_id: str, scan_id: str, provider_id: str +) -> dict[str, bool | str]: + """ + Job function to generate a threatscore report and upload it to S3. + + This function orchestrates the complete report generation workflow: + 1. Validates that the scan has findings + 2. Checks provider type compatibility + 3. Generates the output directory + 4. Calls generate_threatscore_report to create the PDF + 5. Computes and stores ThreatScore metrics snapshot + 6. Uploads the PDF to S3 + 7. Cleans up temporary files + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): The ID of the scan to generate a report for. + provider_id (str): The ID of the provider used in the scan. + + Returns: + dict[str, bool | str]: A dictionary containing: + - 'upload' (bool): True if the report was successfully uploaded to S3, False otherwise. + - 'error' (str): Error message if an exception occurred (only present on error). + """ + # Check if the scan has findings and get provider info + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + if not ScanSummary.objects.filter(scan_id=scan_id).exists(): + logger.info(f"No findings found for scan {scan_id}") + return {"upload": False} + + provider_obj = Provider.objects.get(id=provider_id) + provider_uid = provider_obj.uid + provider_type = provider_obj.provider + + if provider_type not in ["aws", "azure", "gcp", "m365"]: + logger.info( + f"Provider {provider_id} is not supported for threatscore report" + ) + return {"upload": False} + + # This compliance is hardcoded because is the only one that is available for the threatscore report + compliance_id = f"prowler_threatscore_{provider_type}" + logger.info( + f"Generating threatscore report for scan {scan_id} with compliance {compliance_id} inside the job" + ) + try: + logger.info("Generating the output directory") + out_dir, _, threatscore_path = _generate_output_directory( + DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id + ) + except Exception as e: + logger.error(f"Error generating output directory: {e}") + return {"error": str(e)} + + pdf_path = f"{threatscore_path}_threatscore_report.pdf" + logger.info(f"The path for the threatscore report is {pdf_path}") + generate_threatscore_report( + tenant_id=tenant_id, + scan_id=scan_id, + compliance_id=compliance_id, + output_path=pdf_path, + provider_id=provider_id, + only_failed=True, + min_risk_level=4, + ) + + # Compute and store ThreatScore metrics snapshot + logger.info(f"Computing ThreatScore metrics for scan {scan_id}") + try: + metrics = compute_threatscore_metrics( + tenant_id=tenant_id, + scan_id=scan_id, + provider_id=provider_id, + compliance_id=compliance_id, + min_risk_level=4, + ) + + # Create snapshot in database + with rls_transaction(tenant_id): + # Get previous snapshot for the same provider to calculate delta + previous_snapshot = ( + ThreatScoreSnapshot.objects.filter( + tenant_id=tenant_id, + provider_id=provider_id, + compliance_id=compliance_id, + ) + .order_by("-inserted_at") + .first() + ) + + # Calculate score delta (improvement) + score_delta = None + if previous_snapshot: + score_delta = metrics["overall_score"] - float( + previous_snapshot.overall_score + ) + + snapshot = ThreatScoreSnapshot.objects.create( + tenant_id=tenant_id, + scan_id=scan_id, + provider_id=provider_id, + compliance_id=compliance_id, + overall_score=metrics["overall_score"], + score_delta=score_delta, + section_scores=metrics["section_scores"], + critical_requirements=metrics["critical_requirements"], + total_requirements=metrics["total_requirements"], + passed_requirements=metrics["passed_requirements"], + failed_requirements=metrics["failed_requirements"], + manual_requirements=metrics["manual_requirements"], + total_findings=metrics["total_findings"], + passed_findings=metrics["passed_findings"], + failed_findings=metrics["failed_findings"], + ) + + delta_msg = ( + f" (delta: {score_delta:+.2f}%)" if score_delta is not None else "" + ) + logger.info( + f"ThreatScore snapshot created with ID {snapshot.id} " + f"(score: {snapshot.overall_score}%{delta_msg})" + ) + except Exception as e: + # Log error but don't fail the job if snapshot creation fails + logger.error(f"Error creating ThreatScore snapshot: {e}") + + upload_uri = _upload_to_s3( + tenant_id, + scan_id, + pdf_path, + f"threatscore/{Path(pdf_path).name}", + ) + if upload_uri: + try: + rmtree(Path(pdf_path).parent, ignore_errors=True) + except Exception as e: + logger.error(f"Error deleting output files: {e}") + final_location, did_upload = upload_uri, True + else: + final_location, did_upload = out_dir, False + + logger.info(f"Threatscore report outputs at {final_location}") + + return {"upload": did_upload} diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 63183d57db..ba06c9594c 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -1,8 +1,12 @@ +import csv +import io import json import time +import uuid from collections import defaultdict from copy import deepcopy from datetime import datetime, timezone +from typing import Any from celery.utils.log import get_task_logger from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS @@ -14,8 +18,11 @@ from api.compliance import ( PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE, generate_scan_compliance, ) +from api.db_router import READ_REPLICA_ALIAS, MainRouter from api.db_utils import ( - create_objects_in_batches, + POSTGRES_TENANT_VAR, + SET_CONFIG_QUERY, + psycopg_connection, rls_transaction, update_objects_in_batches, ) @@ -23,6 +30,7 @@ from api.exceptions import ProviderConnectionError from api.models import ( ComplianceRequirementOverview, Finding, + MuteRule, Processor, Provider, Resource, @@ -40,6 +48,28 @@ from prowler.lib.scan.scan import Scan as ProwlerScan logger = get_task_logger(__name__) +# Column order must match `ComplianceRequirementOverview` schema in +# `api/models.py`. Keep this list minimal but sufficient to populate all +# non-nullable fields plus the counters we care about. +COMPLIANCE_REQUIREMENT_COPY_COLUMNS = ( + "id", + "tenant_id", + "inserted_at", + "compliance_id", + "framework", + "version", + "description", + "region", + "requirement_id", + "requirement_status", + "passed_checks", + "failed_checks", + "total_checks", + "passed_findings", + "total_findings", + "scan_id", +) + def _create_finding_delta( last_status: FindingStatus | None | str, new_status: FindingStatus | None @@ -107,6 +137,124 @@ def _store_resources( return resource_instance, (resource_instance.uid, resource_instance.region) +def _copy_compliance_requirement_rows( + tenant_id: str, rows: list[dict[str, Any]] +) -> None: + """Stream compliance requirement rows into Postgres using COPY. + + We leverage the admin connection (when available) to bypass the COPY + RLS + restriction, writing only the fields required by + ``ComplianceRequirementOverview``. + + Args: + tenant_id: Target tenant UUID. + rows: List of row dictionaries prepared by + :func:`create_compliance_requirements`. + """ + + csv_buffer = io.StringIO() + writer = csv.writer(csv_buffer) + + datetime_now = datetime.now(tz=timezone.utc) + for row in rows: + writer.writerow( + [ + str(row.get("id")), + str(row.get("tenant_id")), + (row.get("inserted_at") or datetime_now).isoformat(), + row.get("compliance_id") or "", + row.get("framework") or "", + row.get("version") or "", + row.get("description") or "", + row.get("region") or "", + row.get("requirement_id") or "", + row.get("requirement_status") or "", + row.get("passed_checks", 0), + row.get("failed_checks", 0), + row.get("total_checks", 0), + row.get("passed_findings", 0), + row.get("total_findings", 0), + str(row.get("scan_id")), + ] + ) + + csv_buffer.seek(0) + copy_sql = ( + "COPY compliance_requirements_overviews (" + + ", ".join(COMPLIANCE_REQUIREMENT_COPY_COLUMNS) + + ") FROM STDIN WITH (FORMAT CSV, DELIMITER ',', QUOTE '\"', ESCAPE '\"', NULL '\\N')" + ) + + try: + with psycopg_connection(MainRouter.admin_db) as connection: + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.execute(SET_CONFIG_QUERY, [POSTGRES_TENANT_VAR, tenant_id]) + cursor.copy_expert(copy_sql, csv_buffer) + connection.commit() + except Exception: + connection.rollback() + raise + finally: + csv_buffer.close() + + +def _persist_compliance_requirement_rows( + tenant_id: str, rows: list[dict[str, Any]] +) -> None: + """Persist compliance requirement rows using COPY with ORM fallback. + + Args: + tenant_id: Target tenant UUID. + rows: Precomputed row dictionaries that reflect the compliance + overview state for a scan. + """ + if not rows: + return + + try: + _copy_compliance_requirement_rows(tenant_id, rows) + except Exception as error: + logger.exception( + "COPY bulk insert for compliance requirements failed; falling back to ORM bulk_create", + exc_info=error, + ) + fallback_objects = [ + ComplianceRequirementOverview( + id=row["id"], + tenant_id=row["tenant_id"], + inserted_at=row["inserted_at"], + compliance_id=row["compliance_id"], + framework=row["framework"], + version=row["version"], + description=row["description"], + region=row["region"], + requirement_id=row["requirement_id"], + requirement_status=row["requirement_status"], + passed_checks=row["passed_checks"], + failed_checks=row["failed_checks"], + total_checks=row["total_checks"], + passed_findings=row.get("passed_findings", 0), + total_findings=row.get("total_findings", 0), + scan_id=row["scan_id"], + ) + for row in rows + ] + with rls_transaction(tenant_id): + ComplianceRequirementOverview.objects.bulk_create( + fallback_objects, batch_size=500 + ) + + +def _normalized_compliance_key(framework: str | None, version: str | None) -> str: + """Return normalized identifier used to group compliance totals.""" + + normalized_framework = (framework or "").lower().replace("-", "").replace("_", "") + normalized_version = (version or "").lower().replace("-", "").replace("_", "") + return f"{normalized_framework}{normalized_version}" + + def perform_prowler_scan( tenant_id: str, scan_id: str, @@ -143,7 +291,7 @@ def perform_prowler_scan( scan_instance.save() # Find the mutelist processor if it exists - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): try: mutelist_processor = Processor.objects.get( tenant_id=tenant_id, processor_type=Processor.ProcessorChoices.MUTELIST @@ -154,6 +302,20 @@ def perform_prowler_scan( logger.error(f"Error processing mutelist rules: {e}") mutelist_processor = None + # Load enabled mute rules for this tenant + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + try: + active_mute_rules = MuteRule.objects.filter( + tenant_id=tenant_id, enabled=True + ).values_list("finding_uids", "reason") + + mute_rules_cache = {} + for finding_uids, reason in active_mute_rules: + for uid in finding_uids: + mute_rules_cache[uid] = reason + except Exception as e: + logger.error(f"Error loading mute rules: {e}") + mute_rules_cache = {} try: with rls_transaction(tenant_id): try: @@ -272,7 +434,7 @@ def perform_prowler_scan( unique_resources.add((resource_instance.uid, resource_instance.region)) # Process finding - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): finding_uid = finding.uid last_first_seen_at = None if finding_uid not in last_status_cache: @@ -302,9 +464,26 @@ def perform_prowler_scan( if not last_first_seen_at: last_first_seen_at = datetime.now(tz=timezone.utc) - # If the finding is muted at this time the reason must be the configured Mutelist - muted_reason = "Muted by mutelist" if finding.muted else None + # Determine if finding should be muted and why + # Priority: mutelist processor (highest) > manual mute rules + is_muted = False + muted_reason = None + # Check mutelist processor first (highest priority) + if finding.muted: + is_muted = True + muted_reason = "Muted by mutelist" + # If not muted by mutelist, check manual mute rules + elif finding_uid in mute_rules_cache: + is_muted = True + muted_reason = mute_rules_cache[finding_uid] + + # Increment failed_findings_count cache if the finding status is FAIL and not muted + if status == FindingStatus.FAIL and not is_muted: + resource_uid = finding.resource_uid + resource_failed_findings_cache[resource_uid] += 1 + + with rls_transaction(tenant_id): # Create the finding finding_instance = Finding.objects.create( tenant_id=tenant_id, @@ -319,17 +498,13 @@ def perform_prowler_scan( check_id=finding.check_id, scan=scan_instance, first_seen_at=last_first_seen_at, - muted=finding.muted, + muted=is_muted, + muted_at=datetime.now(tz=timezone.utc) if is_muted else None, muted_reason=muted_reason, compliance=finding.compliance, ) finding_instance.add_resources([resource_instance]) - # Increment failed_findings_count cache if the finding status is FAIL and not muted - if status == FindingStatus.FAIL and not finding.muted: - resource_uid = finding.resource_uid - resource_failed_findings_cache[resource_uid] += 1 - # Update scan resource summaries scan_resource_cache.add( ( @@ -439,7 +614,7 @@ def aggregate_findings(tenant_id: str, scan_id: str): - muted_new: Muted findings with a delta of 'new'. - muted_changed: Muted findings with a delta of 'changed'. """ - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): findings = Finding.objects.filter(tenant_id=tenant_id, scan_id=scan_id) aggregation = findings.values( @@ -582,11 +757,28 @@ def create_compliance_requirements(tenant_id: str, scan_id: str): ValidationError: If tenant_id is not a valid UUID. """ try: - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): scan_instance = Scan.objects.get(pk=scan_id) provider_instance = scan_instance.provider prowler_provider = return_prowler_provider(provider_instance) + compliance_template = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE.get( + provider_instance.provider, {} + ) + modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" + threatscore_requirements_by_check: dict[str, set[str]] = {} + threatscore_framework = compliance_template.get( + modeled_threatscore_compliance_id + ) + if threatscore_framework: + for requirement_id, requirement in threatscore_framework[ + "requirements" + ].items(): + for check_id in requirement["checks"]: + threatscore_requirements_by_check.setdefault(check_id, set()).add( + requirement_id + ) + # Get check status data by region from findings findings = ( Finding.all_objects.filter(scan_id=scan_id, muted=False) @@ -603,8 +795,7 @@ def create_compliance_requirements(tenant_id: str, scan_id: str): findings_count_by_compliance = {} check_status_by_region = {} - modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): for finding in findings: for resource in finding.small_resources: region = resource.region @@ -640,73 +831,76 @@ def create_compliance_requirements(tenant_id: str, scan_id: str): # If not available, use regions from findings regions = set(check_status_by_region.keys()) - # Get compliance template for the provider - compliance_template = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE[ - provider_instance.provider - ] - # Create compliance data by region compliance_overview_by_region = { region: deepcopy(compliance_template) for region in regions } - # Apply check statuses to compliance data - for region, check_status in check_status_by_region.items(): - compliance_data = compliance_overview_by_region.setdefault( - region, deepcopy(compliance_template) - ) - for check_name, status in check_status.items(): - generate_scan_compliance( - compliance_data, - provider_instance.provider, - check_name, - status, - ) + compliance_requirement_rows: list[dict[str, Any]] = [] - # Prepare compliance requirement objects - compliance_requirement_objects = [] - for region, compliance_data in compliance_overview_by_region.items(): - for compliance_id, compliance in compliance_data.items(): - modeled_framework = ( - compliance["framework"].lower().replace("-", "").replace("_", "") + # Skip if provider has no compliance frameworks + if compliance_template: + # Apply check statuses to compliance data + for region, check_status in check_status_by_region.items(): + compliance_data = compliance_overview_by_region.setdefault( + region, deepcopy(compliance_template) ) - modeled_version = ( - compliance["version"].lower().replace("-", "").replace("_", "") - ) - modeled_compliance_id = f"{modeled_framework}{modeled_version}" - # Create an overview record for each requirement within each compliance framework - for requirement_id, requirement in compliance["requirements"].items(): - compliance_requirement_objects.append( - ComplianceRequirementOverview( - tenant_id=tenant_id, - scan=scan_instance, - region=region, - compliance_id=compliance_id, - framework=compliance["framework"], - version=compliance["version"], - requirement_id=requirement_id, - description=requirement["description"], - passed_checks=requirement["checks_status"]["pass"], - failed_checks=requirement["checks_status"]["fail"], - total_checks=requirement["checks_status"]["total"], - requirement_status=requirement["status"], - passed_findings=findings_count_by_compliance.get(region, {}) - .get(modeled_compliance_id, {}) - .get(requirement_id, {}) - .get("pass", 0), - total_findings=findings_count_by_compliance.get(region, {}) - .get(modeled_compliance_id, {}) - .get(requirement_id, {}) - .get("total", 0), - ) + for check_name, status in check_status.items(): + generate_scan_compliance( + compliance_data, + provider_instance.provider, + check_name, + status, ) - # Bulk create requirement records - create_objects_in_batches( - tenant_id, ComplianceRequirementOverview, compliance_requirement_objects - ) + + # Prepare compliance requirement rows + utc_datetime_now = datetime.now(tz=timezone.utc) + for region, compliance_data in compliance_overview_by_region.items(): + for compliance_id, compliance in compliance_data.items(): + modeled_compliance_id = _normalized_compliance_key( + compliance["framework"], compliance["version"] + ) + # Create an overview record for each requirement within each compliance framework + for requirement_id, requirement in compliance[ + "requirements" + ].items(): + checks_status = requirement["checks_status"] + compliance_requirement_rows.append( + { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "inserted_at": utc_datetime_now, + "compliance_id": compliance_id, + "framework": compliance["framework"], + "version": compliance["version"] or "", + "description": requirement.get("description") or "", + "region": region, + "requirement_id": requirement_id, + "requirement_status": requirement["status"], + "passed_checks": checks_status["pass"], + "failed_checks": checks_status["fail"], + "total_checks": checks_status["total"], + "scan_id": scan_instance.id, + "passed_findings": findings_count_by_compliance.get( + region, {} + ) + .get(modeled_compliance_id, {}) + .get(requirement_id, {}) + .get("pass", 0), + "total_findings": findings_count_by_compliance.get( + region, {} + ) + .get(modeled_compliance_id, {}) + .get(requirement_id, {}) + .get("total", 0), + } + ) + + # Bulk create requirement records using PostgreSQL COPY + _persist_compliance_requirement_rows(tenant_id, compliance_requirement_rows) return { - "requirements_created": len(compliance_requirement_objects), + "requirements_created": len(compliance_requirement_rows), "regions_processed": list(regions), "compliance_frameworks": ( list(compliance_overview_by_region.get(list(regions)[0], {}).keys()) diff --git a/api/src/backend/tasks/jobs/threatscore.py b/api/src/backend/tasks/jobs/threatscore.py new file mode 100644 index 0000000000..414f2d20f2 --- /dev/null +++ b/api/src/backend/tasks/jobs/threatscore.py @@ -0,0 +1,214 @@ +from celery.utils.log import get_task_logger +from tasks.jobs.threatscore_utils import ( + _aggregate_requirement_statistics_from_database, + _calculate_requirements_data_from_statistics, +) + +from api.db_router import READ_REPLICA_ALIAS +from api.db_utils import rls_transaction +from api.models import Provider, StatusChoices +from prowler.lib.check.compliance_models import Compliance + +logger = get_task_logger(__name__) + + +def compute_threatscore_metrics( + tenant_id: str, + scan_id: str, + provider_id: str, + compliance_id: str, + min_risk_level: int = 4, +) -> dict: + """ + Compute ThreatScore metrics for a given scan. + + This function calculates all the metrics needed for a ThreatScore snapshot: + - Overall ThreatScore percentage + - Section-by-section scores + - Critical failed requirements (risk >= min_risk_level) + - Summary statistics (requirements and findings counts) + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): The ID of the scan to analyze. + provider_id (str): The ID of the provider used in the scan. + compliance_id (str): Compliance framework ID (e.g., "prowler_threatscore_aws"). + min_risk_level (int): Minimum risk level for critical requirements. Defaults to 4. + + Returns: + dict: A dictionary containing: + - overall_score (float): Overall ThreatScore percentage (0-100) + - section_scores (dict): Section name -> score percentage mapping + - critical_requirements (list): List of critical failed requirement dicts + - total_requirements (int): Total number of requirements + - passed_requirements (int): Number of PASS requirements + - failed_requirements (int): Number of FAIL requirements + - manual_requirements (int): Number of MANUAL requirements + - total_findings (int): Total findings count + - passed_findings (int): Passed findings count + - failed_findings (int): Failed findings count + + Example: + >>> metrics = compute_threatscore_metrics( + ... tenant_id="tenant-123", + ... scan_id="scan-456", + ... provider_id="provider-789", + ... compliance_id="prowler_threatscore_aws" + ... ) + >>> print(f"Overall ThreatScore: {metrics['overall_score']:.2f}%") + """ + # Get provider and compliance information + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + provider_obj = Provider.objects.get(id=provider_id) + provider_type = provider_obj.provider + + frameworks_bulk = Compliance.get_bulk(provider_type) + compliance_obj = frameworks_bulk[compliance_id] + + # Aggregate requirement statistics from database + requirement_statistics_by_check_id = ( + _aggregate_requirement_statistics_from_database(tenant_id, scan_id) + ) + + # Calculate requirements data using aggregated statistics + attributes_by_requirement_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + compliance_obj, requirement_statistics_by_check_id + ) + ) + + # Initialize metrics + overall_numerator = 0 + overall_denominator = 0 + overall_has_findings = False + + sections_data = {} + + total_requirements = len(requirements_list) + passed_requirements = 0 + failed_requirements = 0 + manual_requirements = 0 + total_findings = 0 + passed_findings = 0 + failed_findings = 0 + + critical_requirements_list = [] + + # Process each requirement + for requirement in requirements_list: + requirement_id = requirement["id"] + requirement_status = requirement["attributes"]["status"] + requirement_attributes = attributes_by_requirement_id.get(requirement_id, {}) + + # Count requirements by status + if requirement_status == StatusChoices.PASS: + passed_requirements += 1 + elif requirement_status == StatusChoices.FAIL: + failed_requirements += 1 + elif requirement_status == StatusChoices.MANUAL: + manual_requirements += 1 + + # Get findings data + req_passed_findings = requirement["attributes"].get("passed_findings", 0) + req_total_findings = requirement["attributes"].get("total_findings", 0) + + # Accumulate findings counts + total_findings += req_total_findings + passed_findings += req_passed_findings + failed_findings += req_total_findings - req_passed_findings + + # Skip requirements with no findings + if req_total_findings == 0: + continue + + overall_has_findings = True + + # Get requirement metadata + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if not metadata or len(metadata) == 0: + continue + + m = metadata[0] + risk_level = getattr(m, "LevelOfRisk", 0) + weight = getattr(m, "Weight", 0) + section = getattr(m, "Section", "Unknown") + + # Calculate ThreatScore components using formula from UI + rate_i = req_passed_findings / req_total_findings + rfac_i = 1 + 0.25 * risk_level + + # Update overall score + overall_numerator += rate_i * req_total_findings * weight * rfac_i + overall_denominator += req_total_findings * weight * rfac_i + + # Update section scores + if section not in sections_data: + sections_data[section] = { + "numerator": 0, + "denominator": 0, + "has_findings": False, + } + + sections_data[section]["has_findings"] = True + sections_data[section]["numerator"] += ( + rate_i * req_total_findings * weight * rfac_i + ) + sections_data[section]["denominator"] += req_total_findings * weight * rfac_i + + # Identify critical failed requirements + if requirement_status == StatusChoices.FAIL and risk_level >= min_risk_level: + critical_requirements_list.append( + { + "requirement_id": requirement_id, + "title": getattr(m, "Title", "N/A"), + "section": section, + "subsection": getattr(m, "SubSection", "N/A"), + "risk_level": risk_level, + "weight": weight, + "passed_findings": req_passed_findings, + "total_findings": req_total_findings, + "description": getattr(m, "AttributeDescription", "N/A"), + } + ) + + # Calculate overall ThreatScore + if not overall_has_findings: + overall_score = 100.0 + elif overall_denominator > 0: + overall_score = (overall_numerator / overall_denominator) * 100 + else: + overall_score = 0.0 + + # Calculate section scores + section_scores = {} + for section, data in sections_data.items(): + if data["has_findings"] and data["denominator"] > 0: + section_scores[section] = (data["numerator"] / data["denominator"]) * 100 + else: + section_scores[section] = 100.0 + + # Sort critical requirements by risk level (desc) and weight (desc) + critical_requirements_list.sort( + key=lambda x: (x["risk_level"], x["weight"]), reverse=True + ) + + logger.info( + f"ThreatScore computed: {overall_score:.2f}% " + f"({passed_requirements}/{total_requirements} requirements passed, " + f"{len(critical_requirements_list)} critical failures)" + ) + + return { + "overall_score": round(overall_score, 2), + "section_scores": {k: round(v, 2) for k, v in section_scores.items()}, + "critical_requirements": critical_requirements_list, + "total_requirements": total_requirements, + "passed_requirements": passed_requirements, + "failed_requirements": failed_requirements, + "manual_requirements": manual_requirements, + "total_findings": total_findings, + "passed_findings": passed_findings, + "failed_findings": failed_findings, + } diff --git a/api/src/backend/tasks/jobs/threatscore_utils.py b/api/src/backend/tasks/jobs/threatscore_utils.py new file mode 100644 index 0000000000..bc2cb009a2 --- /dev/null +++ b/api/src/backend/tasks/jobs/threatscore_utils.py @@ -0,0 +1,127 @@ +from celery.utils.log import get_task_logger +from django.db.models import Count, Q + +from api.db_router import READ_REPLICA_ALIAS +from api.db_utils import rls_transaction +from api.models import Finding, StatusChoices + +logger = get_task_logger(__name__) + + +def _aggregate_requirement_statistics_from_database( + tenant_id: str, scan_id: str +) -> dict[str, dict[str, int]]: + """ + Aggregate finding statistics by check_id using database aggregation. + + This function uses Django ORM aggregation to calculate pass/fail statistics + entirely in the database, avoiding the need to load findings into memory. + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): The ID of the scan to retrieve findings for. + + Returns: + dict[str, dict[str, int]]: Dictionary mapping check_id to statistics: + - 'passed' (int): Number of passed findings for this check + - 'total' (int): Total number of findings for this check + + Example: + { + 'aws_iam_user_mfa_enabled': {'passed': 10, 'total': 15}, + 'aws_s3_bucket_public_access': {'passed': 0, 'total': 5} + } + """ + requirement_statistics_by_check_id = {} + + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + aggregated_statistics_queryset = ( + Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id) + .values("check_id") + .annotate( + total_findings=Count("id"), + passed_findings=Count("id", filter=Q(status=StatusChoices.PASS)), + ) + ) + + for aggregated_stat in aggregated_statistics_queryset: + check_id = aggregated_stat["check_id"] + requirement_statistics_by_check_id[check_id] = { + "passed": aggregated_stat["passed_findings"], + "total": aggregated_stat["total_findings"], + } + + logger.info( + f"Aggregated statistics for {len(requirement_statistics_by_check_id)} unique checks" + ) + return requirement_statistics_by_check_id + + +def _calculate_requirements_data_from_statistics( + compliance_obj, requirement_statistics_by_check_id: dict[str, dict[str, int]] +) -> tuple[dict[str, dict], list[dict]]: + """ + Calculate requirement status and statistics using pre-aggregated database statistics. + + Args: + compliance_obj: The compliance framework object containing requirements. + requirement_statistics_by_check_id (dict[str, dict[str, int]]): Pre-aggregated statistics + mapping check_id to {'passed': int, 'total': int} counts. + + Returns: + tuple[dict[str, dict], list[dict]]: A tuple containing: + - attributes_by_requirement_id: Dictionary mapping requirement IDs to their attributes. + - requirements_list: List of requirement dictionaries with status and statistics. + """ + attributes_by_requirement_id = {} + requirements_list = [] + + compliance_framework = getattr(compliance_obj, "Framework", "N/A") + compliance_version = getattr(compliance_obj, "Version", "N/A") + + for requirement in compliance_obj.Requirements: + requirement_id = requirement.Id + requirement_description = getattr(requirement, "Description", "") + requirement_checks = getattr(requirement, "Checks", []) + requirement_attributes = getattr(requirement, "Attributes", []) + + attributes_by_requirement_id[requirement_id] = { + "attributes": { + "req_attributes": requirement_attributes, + "checks": requirement_checks, + }, + "description": requirement_description, + } + + total_passed_findings = 0 + total_findings_count = 0 + + for check_id in requirement_checks: + if check_id in requirement_statistics_by_check_id: + check_statistics = requirement_statistics_by_check_id[check_id] + total_findings_count += check_statistics["total"] + total_passed_findings += check_statistics["passed"] + + if total_findings_count > 0: + if total_passed_findings == total_findings_count: + requirement_status = StatusChoices.PASS + else: + requirement_status = StatusChoices.FAIL + else: + requirement_status = StatusChoices.MANUAL + + requirements_list.append( + { + "id": requirement_id, + "attributes": { + "framework": compliance_framework, + "version": compliance_version, + "status": requirement_status, + "description": requirement_description, + "passed_findings": total_passed_findings, + "total_findings": total_findings_count, + }, + } + ) + + return attributes_by_requirement_id, requirements_list diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index dbb3c3a00e..799427cd19 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -1,3 +1,4 @@ +import os from datetime import datetime, timedelta, timezone from pathlib import Path from shutil import rmtree @@ -26,6 +27,12 @@ from tasks.jobs.integrations import ( upload_s3_integration, upload_security_hub_integration, ) +from tasks.jobs.lighthouse_providers import ( + check_lighthouse_provider_connection, + refresh_lighthouse_provider_models, +) +from tasks.jobs.muting import mute_historical_findings +from tasks.jobs.report import generate_threatscore_report_job from tasks.jobs.scan import ( aggregate_findings, create_compliance_requirements, @@ -34,6 +41,7 @@ from tasks.jobs.scan import ( from tasks.utils import batched, get_next_execution_datetime from api.compliance import get_compliance_frameworks +from api.db_router import READ_REPLICA_ALIAS from api.db_utils import rls_transaction from api.decorators import set_tenant from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices @@ -63,10 +71,15 @@ def _perform_scan_complete_tasks(tenant_id: str, scan_id: str, provider_id: str) generate_outputs_task.si( scan_id=scan_id, provider_id=provider_id, tenant_id=tenant_id ), - check_integrations_task.si( - tenant_id=tenant_id, - provider_id=provider_id, - scan_id=scan_id, + group( + generate_threatscore_report_task.si( + tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id + ), + check_integrations_task.si( + tenant_id=tenant_id, + provider_id=provider_id, + scan_id=scan_id, + ), ), ).apply_async() @@ -303,7 +316,7 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str): frameworks_bulk = Compliance.get_bulk(provider_type) frameworks_avail = get_compliance_frameworks(provider_type) - out_dir, comp_dir = _generate_output_directory( + out_dir, comp_dir, _ = _generate_output_directory( DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id ) @@ -343,70 +356,90 @@ def generate_outputs_task(scan_id: str, provider_id: str, tenant_id: str): .order_by("uid") .iterator() ) - for batch, is_last in batched(qs, DJANGO_FINDINGS_BATCH_SIZE): - fos = [FindingOutput.transform_api_finding(f, prowler_provider) for f in batch] + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + for batch, is_last in batched(qs, DJANGO_FINDINGS_BATCH_SIZE): + fos = [ + FindingOutput.transform_api_finding(f, prowler_provider) for f in batch + ] - # Outputs - for mode, cfg in OUTPUT_FORMATS_MAPPING.items(): - # Skip ASFF generation if not needed - if mode == "json-asff" and not generate_asff: - continue + # Outputs + for mode, cfg in OUTPUT_FORMATS_MAPPING.items(): + # Skip ASFF generation if not needed + if mode == "json-asff" and not generate_asff: + continue - cls = cfg["class"] - suffix = cfg["suffix"] - extra = cfg.get("kwargs", {}).copy() - if mode == "html": - extra.update(provider=prowler_provider, stats=scan_summary) + cls = cfg["class"] + suffix = cfg["suffix"] + extra = cfg.get("kwargs", {}).copy() + if mode == "html": + extra.update(provider=prowler_provider, stats=scan_summary) - writer, initialization = get_writer( - output_writers, - cls, - lambda cls=cls, fos=fos, suffix=suffix: cls( - findings=fos, - file_path=out_dir, - file_extension=suffix, - from_cli=False, - ), - is_last, - ) - if not initialization: - writer.transform(fos) - writer.batch_write_data_to_file(**extra) - writer._data.clear() + writer, initialization = get_writer( + output_writers, + cls, + lambda cls=cls, fos=fos, suffix=suffix: cls( + findings=fos, + file_path=out_dir, + file_extension=suffix, + from_cli=False, + ), + is_last, + ) + if not initialization: + writer.transform(fos) + writer.batch_write_data_to_file(**extra) + writer._data.clear() - # Compliance CSVs - for name in frameworks_avail: - compliance_obj = frameworks_bulk[name] + # Compliance CSVs + for name in frameworks_avail: + compliance_obj = frameworks_bulk[name] - klass = GenericCompliance - for condition, cls in COMPLIANCE_CLASS_MAP.get(provider_type, []): - if condition(name): - klass = cls - break + klass = GenericCompliance + for condition, cls in COMPLIANCE_CLASS_MAP.get(provider_type, []): + if condition(name): + klass = cls + break - filename = f"{comp_dir}_{name}.csv" + filename = f"{comp_dir}_{name}.csv" - writer, initialization = get_writer( - compliance_writers, - name, - lambda klass=klass, fos=fos: klass( - findings=fos, - compliance=compliance_obj, - file_path=filename, - from_cli=False, - ), - is_last, - ) - if not initialization: - writer.transform(fos, compliance_obj, name) - writer.batch_write_data_to_file() - writer._data.clear() + writer, initialization = get_writer( + compliance_writers, + name, + lambda klass=klass, fos=fos: klass( + findings=fos, + compliance=compliance_obj, + file_path=filename, + from_cli=False, + ), + is_last, + ) + if not initialization: + writer.transform(fos, compliance_obj, name) + writer.batch_write_data_to_file() + writer._data.clear() compressed = _compress_output_files(out_dir) - upload_uri = _upload_to_s3(tenant_id, compressed, scan_id) + + upload_uri = _upload_to_s3( + tenant_id, + scan_id, + compressed, + os.path.basename(compressed), + ) + + compliance_dir_path = Path(comp_dir).parent + if compliance_dir_path.exists(): + for artifact_path in sorted(compliance_dir_path.iterdir()): + if artifact_path.is_file(): + _upload_to_s3( + tenant_id, + scan_id, + str(artifact_path), + f"compliance/{artifact_path.name}", + ) # S3 integrations (need output_directory) - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): s3_integrations = Integration.objects.filter( integrationproviderrelationship__provider_id=provider_id, integration_type=Integration.IntegrationChoices.AMAZON_S3, @@ -496,6 +529,24 @@ def check_lighthouse_connection_task(lighthouse_config_id: str, tenant_id: str = return check_lighthouse_connection(lighthouse_config_id=lighthouse_config_id) +@shared_task(base=RLSTask, name="lighthouse-provider-connection-check") +@set_tenant +def check_lighthouse_provider_connection_task( + provider_config_id: str, tenant_id: str | None = None +) -> dict: + """Task wrapper to validate provider credentials and set is_active.""" + return check_lighthouse_provider_connection(provider_config_id=provider_config_id) + + +@shared_task(base=RLSTask, name="lighthouse-provider-models-refresh") +@set_tenant +def refresh_lighthouse_provider_models_task( + provider_config_id: str, tenant_id: str | None = None +) -> dict: + """Task wrapper to refresh provider models catalog for the given configuration.""" + return refresh_lighthouse_provider_models(provider_config_id=provider_config_id) + + @shared_task(name="integration-check") def check_integrations_task(tenant_id: str, provider_id: str, scan_id: str = None): """ @@ -613,3 +664,43 @@ def jira_integration_task( return send_findings_to_jira( tenant_id, integration_id, project_key, issue_type, finding_ids ) + + +@shared_task( + base=RLSTask, + name="scan-threatscore-report", + queue="scan-reports", +) +def generate_threatscore_report_task(tenant_id: str, scan_id: str, provider_id: str): + """ + Task to generate a threatscore report for a given scan. + Args: + tenant_id (str): The tenant identifier. + scan_id (str): The scan identifier. + provider_id (str): The provider identifier. + """ + return generate_threatscore_report_job( + tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id + ) + + +@shared_task(name="findings-mute-historical") +def mute_historical_findings_task(tenant_id: str, mute_rule_id: str): + """ + Background task to mute all historical findings matching a mute rule. + + This task processes findings in batches to avoid memory issues with large datasets. + It updates the Finding.muted, Finding.muted_at, and Finding.muted_reason fields + for all findings whose UID is in the mute rule's finding_uids list. + + Args: + tenant_id (str): The tenant ID for RLS context. + mute_rule_id (str): The primary key of the MuteRule to apply. + + Returns: + dict: A dictionary containing: + - 'findings_muted' (int): Total number of findings muted. + - 'rule_id' (str): The mute rule ID. + - 'status' (str): Final status ('completed'). + """ + return mute_historical_findings(tenant_id, mute_rule_id) diff --git a/api/src/backend/tasks/tests/test_export.py b/api/src/backend/tasks/tests/test_export.py index c10f20774b..f1e7120989 100644 --- a/api/src/backend/tasks/tests/test_export.py +++ b/api/src/backend/tasks/tests/test_export.py @@ -72,17 +72,26 @@ class TestOutputs: client_mock = MagicMock() mock_get_client.return_value = client_mock - result = _upload_to_s3("tenant-id", str(zip_path), "scan-id") + result = _upload_to_s3( + "tenant-id", + "scan-id", + str(zip_path), + "outputs.zip", + ) expected_uri = "s3://test-bucket/tenant-id/scan-id/outputs.zip" assert result == expected_uri - assert client_mock.upload_file.call_count == 2 + client_mock.upload_file.assert_called_once_with( + Filename=str(zip_path), + Bucket="test-bucket", + Key="tenant-id/scan-id/outputs.zip", + ) @patch("tasks.jobs.export.get_s3_client") @patch("tasks.jobs.export.base") def test_upload_to_s3_missing_bucket(self, mock_base, mock_get_client): mock_base.DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET = "" - result = _upload_to_s3("tenant", "/tmp/fake.zip", "scan") + result = _upload_to_s3("tenant", "scan", "/tmp/fake.zip", "fake.zip") assert result is None @patch("tasks.jobs.export.get_s3_client") @@ -101,11 +110,15 @@ class TestOutputs: client_mock = MagicMock() mock_get_client.return_value = client_mock - result = _upload_to_s3("tenant", str(zip_path), "scan") + result = _upload_to_s3( + "tenant", + "scan", + str(compliance_dir / "subdir"), + "compliance/subdir", + ) - expected_uri = "s3://test-bucket/tenant/scan/results.zip" - assert result == expected_uri - client_mock.upload_file.assert_called_once() + assert result is None + client_mock.upload_file.assert_not_called() @patch( "tasks.jobs.export.get_s3_client", @@ -126,7 +139,12 @@ class TestOutputs: compliance_dir.mkdir() (compliance_dir / "report.csv").write_text("csv") - _upload_to_s3("tenant", str(zip_path), "scan") + _upload_to_s3( + "tenant", + "scan", + str(zip_path), + "zipfile.zip", + ) mock_logger.assert_called() @patch("tasks.jobs.export.rls_transaction") @@ -150,15 +168,17 @@ class TestOutputs: provider = "aws" expected_timestamp = "20230615103045" - path, compliance = _generate_output_directory( + path, compliance, threatscore = _generate_output_directory( base_dir, provider, tenant_id, scan_id ) assert os.path.isdir(os.path.dirname(path)) assert os.path.isdir(os.path.dirname(compliance)) + assert os.path.isdir(os.path.dirname(threatscore)) assert path.endswith(f"{provider}-{expected_timestamp}") assert compliance.endswith(f"{provider}-{expected_timestamp}") + assert threatscore.endswith(f"{provider}-{expected_timestamp}") @patch("tasks.jobs.export.rls_transaction") @patch("tasks.jobs.export.Scan") @@ -181,12 +201,14 @@ class TestOutputs: provider = "aws/test@check" expected_timestamp = "20230615103045" - path, compliance = _generate_output_directory( + path, compliance, threatscore = _generate_output_directory( base_dir, provider, tenant_id, scan_id ) assert os.path.isdir(os.path.dirname(path)) assert os.path.isdir(os.path.dirname(compliance)) + assert os.path.isdir(os.path.dirname(threatscore)) assert path.endswith(f"aws-test-check-{expected_timestamp}") assert compliance.endswith(f"aws-test-check-{expected_timestamp}") + assert threatscore.endswith(f"aws-test-check-{expected_timestamp}") diff --git a/api/src/backend/tasks/tests/test_integrations.py b/api/src/backend/tasks/tests/test_integrations.py index b8e06c9e7b..75f07677a0 100644 --- a/api/src/backend/tasks/tests/test_integrations.py +++ b/api/src/backend/tasks/tests/test_integrations.py @@ -9,6 +9,7 @@ from tasks.jobs.integrations import ( upload_security_hub_integration, ) +from api.db_router import READ_REPLICA_ALIAS, MainRouter from api.models import Integration from api.utils import prowler_integration_connection_test from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection @@ -880,7 +881,8 @@ class TestSecurityHubIntegrationUploads: # Verify RLS transaction was used correctly # Should be called twice: once for getting provider info, once for resetting regions assert mock_rls.call_count == 2 - mock_rls.assert_any_call(tenant_id) + mock_rls.assert_any_call(tenant_id, using=READ_REPLICA_ALIAS) + mock_rls.assert_any_call(tenant_id, using=MainRouter.default_db) # Verify test_connection was called with integration credentials (not provider's) mock_test_connection.assert_called_once_with( diff --git a/api/src/backend/tasks/tests/test_muting.py b/api/src/backend/tasks/tests/test_muting.py new file mode 100644 index 0000000000..d8ae310f2e --- /dev/null +++ b/api/src/backend/tasks/tests/test_muting.py @@ -0,0 +1,532 @@ +from datetime import datetime, timezone +from uuid import uuid4 + +import pytest +from django.core.exceptions import ObjectDoesNotExist +from tasks.jobs.muting import mute_historical_findings + +from api.models import Finding, MuteRule +from prowler.lib.check.models import Severity +from prowler.lib.outputs.finding import Status + + +@pytest.mark.django_db +class TestMuteHistoricalFindings: + """ + Test suite for the mute_historical_findings function. + + This class tests the batch processing of findings to update their muted status + based on MuteRule criteria. + """ + + @pytest.fixture(scope="function") + def test_user(self, create_test_user): + """Create a test user for mute rule creation.""" + return create_test_user + + @pytest.fixture(scope="function") + def mute_rule_with_findings(self, tenants_fixture, findings_fixture, test_user): + """ + Create a mute rule that targets the first finding in the fixture. + """ + tenant = tenants_fixture[0] + finding = findings_fixture[0] + + mute_rule = MuteRule.objects.create( + tenant_id=tenant.id, + name="Test Mute Rule", + reason="Testing mute functionality", + enabled=True, + created_by=test_user, + finding_uids=[finding.uid], + ) + + return mute_rule + + @pytest.fixture(scope="function") + def mute_rule_multiple_findings(self, scans_fixture, test_user): + """ + Create multiple unmuted findings and a mute rule targeting all of them. + """ + scan = scans_fixture[0] + tenant_id = scan.tenant_id + + # Create 5 unmuted findings + finding_uids = [] + for i in range(5): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"test_finding_uid_mute_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Test status {i}", + impact=Severity.high, + severity=Severity.high, + raw_result={ + "status": Status.FAIL, + "impact": Severity.high, + "severity": Severity.high, + }, + check_id=f"test_check_id_{i}", + check_metadata={ + "CheckId": f"test_check_id_{i}", + "Description": f"Test description {i}", + }, + muted=False, + ) + finding_uids.append(finding.uid) + + # Create mute rule targeting all findings + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Multiple Findings Mute Rule", + reason="Testing batch muting", + enabled=True, + created_by=test_user, + finding_uids=finding_uids, + ) + + return mute_rule, finding_uids + + @pytest.fixture(scope="function") + def mute_rule_already_muted(self, findings_fixture, test_user): + """ + Create a mute rule that targets an already-muted finding. + """ + tenant_id = findings_fixture[1].tenant_id + already_muted_finding = findings_fixture[1] + + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Already Muted Rule", + reason="Testing already muted findings", + enabled=True, + created_by=test_user, + finding_uids=[already_muted_finding.uid], + ) + + return mute_rule + + @pytest.fixture(scope="function") + def mute_rule_mixed_findings(self, scans_fixture, test_user): + """ + Create a mute rule with a mix of muted and unmuted findings. + """ + scan = scans_fixture[0] + tenant_id = scan.tenant_id + + # Create 3 unmuted findings + unmuted_uids = [] + for i in range(3): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"unmuted_finding_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Unmuted status {i}", + impact=Severity.medium, + severity=Severity.medium, + raw_result={ + "status": Status.FAIL, + "impact": Severity.medium, + "severity": Severity.medium, + }, + check_id=f"unmuted_check_{i}", + check_metadata={ + "CheckId": f"unmuted_check_{i}", + "Description": f"Unmuted description {i}", + }, + muted=False, + ) + unmuted_uids.append(finding.uid) + + # Create 2 already muted findings + muted_uids = [] + for i in range(2): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"muted_finding_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Muted status {i}", + impact=Severity.low, + severity=Severity.low, + raw_result={ + "status": Status.FAIL, + "impact": Severity.low, + "severity": Severity.low, + }, + check_id=f"muted_check_{i}", + check_metadata={ + "CheckId": f"muted_check_{i}", + "Description": f"Muted description {i}", + }, + muted=True, + muted_at=datetime.now(timezone.utc), + muted_reason="Already muted", + ) + muted_uids.append(finding.uid) + + # Create mute rule targeting all findings + all_uids = unmuted_uids + muted_uids + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Mixed Findings Rule", + reason="Testing mixed muted/unmuted findings", + enabled=True, + created_by=test_user, + finding_uids=all_uids, + ) + + return mute_rule, unmuted_uids, muted_uids + + @pytest.fixture(scope="function") + def mute_rule_batch_test(self, scans_fixture, test_user): + """ + Create enough findings to test batch processing (>1000 for default batch size). + """ + scan = scans_fixture[0] + tenant_id = scan.tenant_id + + # Create 1500 findings to exceed default batch size of 1000 + finding_uids = [] + for i in range(1500): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"batch_test_finding_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Batch test status {i}", + impact=Severity.critical, + severity=Severity.critical, + raw_result={ + "status": Status.FAIL, + "impact": Severity.critical, + "severity": Severity.critical, + }, + check_id=f"batch_test_check_{i}", + check_metadata={ + "CheckId": f"batch_test_check_{i}", + "Description": f"Batch test description {i}", + }, + muted=False, + ) + finding_uids.append(finding.uid) + + # Create mute rule targeting all findings + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Batch Processing Rule", + reason="Testing batch processing functionality", + enabled=True, + created_by=test_user, + finding_uids=finding_uids, + ) + + return mute_rule, finding_uids + + def test_mute_historical_findings_single_finding( + self, mute_rule_with_findings, findings_fixture + ): + """ + Test muting a single historical finding. + """ + mute_rule = mute_rule_with_findings + tenant_id = str(mute_rule.tenant_id) + finding = findings_fixture[0] + + # Ensure the finding is not muted before execution + finding.refresh_from_db() + assert finding.muted is False + assert finding.muted_at is None + assert finding.muted_reason is None + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify return value + assert result["findings_muted"] == 1 + assert result["rule_id"] == str(mute_rule.id) + + # Verify the finding was muted + finding.refresh_from_db() + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + def test_mute_historical_findings_multiple_findings( + self, mute_rule_multiple_findings + ): + """ + Test muting multiple historical findings. + """ + mute_rule, finding_uids = mute_rule_multiple_findings + tenant_id = str(mute_rule.tenant_id) + + # Verify all findings are unmuted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids) + assert findings.count() == 5 + for finding in findings: + assert finding.muted is False + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify return value + assert result["findings_muted"] == 5 + assert result["rule_id"] == str(mute_rule.id) + + # Verify all findings were muted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids) + for finding in findings: + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + def test_mute_historical_findings_already_muted( + self, mute_rule_already_muted, findings_fixture + ): + """ + Test that already-muted findings are not counted or updated. + """ + mute_rule = mute_rule_already_muted + tenant_id = str(mute_rule.tenant_id) + finding = findings_fixture[1] + + # Verify the finding is already muted + finding.refresh_from_db() + assert finding.muted is True + original_muted_at = finding.muted_at + original_muted_reason = finding.muted_reason + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify no findings were muted + assert result["findings_muted"] == 0 + assert result["rule_id"] == str(mute_rule.id) + + # Verify the finding's mute status did not change + finding.refresh_from_db() + assert finding.muted is True + assert finding.muted_at == original_muted_at + assert finding.muted_reason == original_muted_reason + + def test_mute_historical_findings_mixed_status(self, mute_rule_mixed_findings): + """ + Test muting when some findings are already muted and others are not. + """ + mute_rule, unmuted_uids, muted_uids = mute_rule_mixed_findings + tenant_id = str(mute_rule.tenant_id) + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify only unmuted findings were counted + assert result["findings_muted"] == 3 + assert result["rule_id"] == str(mute_rule.id) + + # Verify unmuted findings are now muted + unmuted_findings = Finding.objects.filter( + tenant_id=tenant_id, uid__in=unmuted_uids + ) + for finding in unmuted_findings: + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + # Verify already-muted findings remained unchanged + already_muted_findings = Finding.objects.filter( + tenant_id=tenant_id, uid__in=muted_uids + ) + for finding in already_muted_findings: + assert finding.muted is True + assert finding.muted_reason == "Already muted" + + def test_mute_historical_findings_nonexistent_rule(self, tenants_fixture): + """ + Test that a nonexistent mute rule raises ObjectDoesNotExist. + """ + tenant_id = str(tenants_fixture[0].id) + nonexistent_rule_id = str(uuid4()) + + with pytest.raises(ObjectDoesNotExist): + mute_historical_findings(tenant_id, nonexistent_rule_id) + + def test_mute_historical_findings_no_matching_findings( + self, tenants_fixture, test_user + ): + """ + Test muting when no findings match the rule's UIDs. + """ + tenant_id = str(tenants_fixture[0].id) + + # Create a mute rule with non-existent finding UIDs + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test No Match Rule", + reason="Testing no matching findings", + enabled=True, + created_by=test_user, + finding_uids=[ + "nonexistent_uid_1", + "nonexistent_uid_2", + "nonexistent_uid_3", + ], + ) + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify no findings were muted + assert result["findings_muted"] == 0 + assert result["rule_id"] == str(mute_rule.id) + + def test_mute_historical_findings_batch_processing(self, mute_rule_batch_test): + """ + Test that large numbers of findings are processed in batches correctly. + """ + mute_rule, finding_uids = mute_rule_batch_test + tenant_id = str(mute_rule.tenant_id) + + # Verify all findings exist and are unmuted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids) + assert findings.count() == 1500 + for finding in findings: + assert finding.muted is False + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify return value + assert result["findings_muted"] == 1500 + assert result["rule_id"] == str(mute_rule.id) + + # Verify all findings were muted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids) + for finding in findings: + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + def test_mute_historical_findings_preserves_muted_at_timestamp( + self, mute_rule_with_findings, findings_fixture + ): + """ + Test that muted_at is set to the rule's inserted_at, not the current time. + """ + mute_rule = mute_rule_with_findings + tenant_id = str(mute_rule.tenant_id) + finding = findings_fixture[0] + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify the finding was muted + assert result["findings_muted"] == 1 + + # Verify muted_at matches the rule's inserted_at timestamp + finding.refresh_from_db() + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_at is not None + + def test_mute_historical_findings_partial_match(self, scans_fixture, test_user): + """ + Test muting when only some of the rule's UIDs exist as findings. + """ + scan = scans_fixture[0] + tenant_id = str(scan.tenant_id) + + # Create 3 findings + existing_uids = [] + for i in range(3): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"partial_match_finding_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Partial match status {i}", + impact=Severity.high, + severity=Severity.high, + raw_result={ + "status": Status.FAIL, + "impact": Severity.high, + "severity": Severity.high, + }, + check_id=f"partial_match_check_{i}", + check_metadata={ + "CheckId": f"partial_match_check_{i}", + "Description": f"Partial match description {i}", + }, + muted=False, + ) + existing_uids.append(finding.uid) + + # Create a mute rule with both existing and non-existing UIDs + all_uids = existing_uids + [ + "nonexistent_uid_1", + "nonexistent_uid_2", + ] + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Partial Match Rule", + reason="Testing partial matching", + enabled=True, + created_by=test_user, + finding_uids=all_uids, + ) + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify only existing findings were muted + assert result["findings_muted"] == 3 + assert result["rule_id"] == str(mute_rule.id) + + # Verify the existing findings were muted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=existing_uids) + assert findings.count() == 3 + for finding in findings: + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + def test_mute_historical_findings_empty_uids(self, tenants_fixture, test_user): + """ + Test muting when the rule has an empty finding_uids array. + """ + tenant_id = str(tenants_fixture[0].id) + + # Create a mute rule with empty finding_uids + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Empty UIDs Rule", + reason="Testing empty UIDs", + enabled=True, + created_by=test_user, + finding_uids=[], + ) + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify no findings were muted + assert result["findings_muted"] == 0 + assert result["rule_id"] == str(mute_rule.id) + + def test_mute_historical_findings_return_format(self, mute_rule_with_findings): + """ + Test that the return value has the correct format and fields. + """ + mute_rule = mute_rule_with_findings + tenant_id = str(mute_rule.tenant_id) + + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify return value structure + assert isinstance(result, dict) + assert "findings_muted" in result + assert "rule_id" in result + assert isinstance(result["findings_muted"], int) + assert isinstance(result["rule_id"], str) + assert result["rule_id"] == str(mute_rule.id) diff --git a/api/src/backend/tasks/tests/test_report.py b/api/src/backend/tasks/tests/test_report.py new file mode 100644 index 0000000000..91d3b1145d --- /dev/null +++ b/api/src/backend/tasks/tests/test_report.py @@ -0,0 +1,1108 @@ +import uuid +from datetime import timedelta +from decimal import Decimal +from pathlib import Path +from unittest.mock import MagicMock, patch + +import matplotlib +import pytest +from django.utils import timezone +from freezegun import freeze_time +from tasks.jobs.report import ( + _load_findings_for_requirement_checks, + generate_threatscore_report, + generate_threatscore_report_job, +) +from tasks.jobs.threatscore_utils import ( + _aggregate_requirement_statistics_from_database, + _calculate_requirements_data_from_statistics, +) +from tasks.tasks import generate_threatscore_report_task + +from api.models import Finding, Scan, StateChoices, StatusChoices, ThreatScoreSnapshot +from prowler.lib.check.models import Severity + +matplotlib.use("Agg") # Use non-interactive backend for tests + + +@pytest.mark.django_db +class TestGenerateThreatscoreReport: + def setup_method(self): + self.scan_id = str(uuid.uuid4()) + self.provider_id = str(uuid.uuid4()) + self.tenant_id = str(uuid.uuid4()) + + def test_no_findings_returns_early(self): + with patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter: + mock_filter.return_value.exists.return_value = False + + result = generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + assert result == {"upload": False} + mock_filter.assert_called_once_with(scan_id=self.scan_id) + + @patch("tasks.jobs.report.ThreatScoreSnapshot.objects.create") + @patch("tasks.jobs.report.rmtree") + @patch("tasks.jobs.report._upload_to_s3") + @patch("tasks.jobs.report.generate_threatscore_report") + @patch("tasks.jobs.report._generate_output_directory") + @patch("tasks.jobs.report.Provider.objects.get") + @patch("tasks.jobs.report.ScanSummary.objects.filter") + def test_generate_threatscore_report_happy_path( + self, + mock_scan_summary_filter, + mock_provider_get, + mock_generate_output_directory, + mock_generate_report, + mock_upload, + mock_rmtree, + mock_snapshot_create, + ): + mock_scan_summary_filter.return_value.exists.return_value = True + + mock_provider = MagicMock() + mock_provider.uid = "provider-uid" + mock_provider.provider = "aws" + mock_provider_get.return_value = mock_provider + + mock_generate_output_directory.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + mock_upload.return_value = "s3://bucket/threatscore_report.pdf" + + result = generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + assert result == {"upload": True} + mock_generate_report.assert_called_once_with( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + compliance_id="prowler_threatscore_aws", + output_path="/tmp/threatscore_path_threatscore_report.pdf", + provider_id=self.provider_id, + only_failed=True, + min_risk_level=4, + ) + mock_upload.assert_called_once_with( + self.tenant_id, + self.scan_id, + "/tmp/threatscore_path_threatscore_report.pdf", + "threatscore/threatscore_path_threatscore_report.pdf", + ) + mock_rmtree.assert_called_once_with( + Path("/tmp/threatscore_path_threatscore_report.pdf").parent, + ignore_errors=True, + ) + mock_snapshot_create.assert_called_once() + + @patch("tasks.jobs.report.ThreatScoreSnapshot.objects.create") + def test_generate_threatscore_report_fails_upload(self, mock_snapshot_create): + with ( + patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter, + patch("tasks.jobs.report.Provider.objects.get") as mock_provider_get, + patch("tasks.jobs.report._generate_output_directory") as mock_gen_dir, + patch("tasks.jobs.report.generate_threatscore_report"), + patch("tasks.jobs.report._upload_to_s3", return_value=None), + ): + mock_filter.return_value.exists.return_value = True + + # Mock provider + mock_provider = MagicMock() + mock_provider.uid = "aws-provider-uid" + mock_provider.provider = "aws" + mock_provider_get.return_value = mock_provider + + mock_gen_dir.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + result = generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + assert result == {"upload": False} + mock_snapshot_create.assert_called_once() + + @patch("tasks.jobs.report.ThreatScoreSnapshot.objects.create") + def test_generate_threatscore_report_logs_rmtree_exception( + self, mock_snapshot_create, caplog + ): + with ( + patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter, + patch("tasks.jobs.report.Provider.objects.get") as mock_provider_get, + patch("tasks.jobs.report._generate_output_directory") as mock_gen_dir, + patch("tasks.jobs.report.generate_threatscore_report"), + patch( + "tasks.jobs.report._upload_to_s3", return_value="s3://bucket/report.pdf" + ), + patch( + "tasks.jobs.report.rmtree", side_effect=Exception("Test deletion error") + ), + ): + mock_filter.return_value.exists.return_value = True + + # Mock provider + mock_provider = MagicMock() + mock_provider.uid = "aws-provider-uid" + mock_provider.provider = "aws" + mock_provider_get.return_value = mock_provider + + mock_gen_dir.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + with caplog.at_level("ERROR"): + generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + assert "Error deleting output files" in caplog.text + mock_snapshot_create.assert_called_once() + + @patch("tasks.jobs.report.ThreatScoreSnapshot.objects.create") + def test_generate_threatscore_report_azure_provider(self, mock_snapshot_create): + with ( + patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter, + patch("tasks.jobs.report.Provider.objects.get") as mock_provider_get, + patch("tasks.jobs.report._generate_output_directory") as mock_gen_dir, + patch("tasks.jobs.report.generate_threatscore_report") as mock_generate, + patch( + "tasks.jobs.report._upload_to_s3", return_value="s3://bucket/report.pdf" + ), + patch("tasks.jobs.report.rmtree"), + ): + mock_filter.return_value.exists.return_value = True + + mock_provider = MagicMock() + mock_provider.uid = "azure-provider-uid" + mock_provider.provider = "azure" + mock_provider_get.return_value = mock_provider + + mock_gen_dir.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + generate_threatscore_report_job( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + mock_generate.assert_called_once_with( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + compliance_id="prowler_threatscore_azure", + output_path="/tmp/threatscore_path_threatscore_report.pdf", + provider_id=self.provider_id, + only_failed=True, + min_risk_level=4, + ) + mock_snapshot_create.assert_called_once() + + @patch("tasks.jobs.report.rmtree") + @patch( + "tasks.jobs.report._upload_to_s3", + return_value="s3://bucket/threatscore/threatscore_report.pdf", + ) + @patch("tasks.jobs.report.generate_threatscore_report") + @patch("tasks.jobs.report._generate_output_directory") + @patch("tasks.jobs.report.ScanSummary.objects.filter") + @patch("tasks.jobs.report.compute_threatscore_metrics") + @pytest.mark.django_db + @freeze_time("2025-01-10T12:00:00Z") + def test_generate_threatscore_report_persists_snapshot_and_delta( + self, + mock_compute_metrics, + mock_scan_summary_filter, + mock_generate_output_directory, + mock_generate_report, + mock_upload, + mock_rmtree, + tenants_fixture, + providers_fixture, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + + scan_previous = Scan.objects.create( + tenant=tenant, + provider=provider, + name="previous-threatscore-scan", + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + started_at=timezone.now() - timedelta(hours=4), + completed_at=timezone.now() - timedelta(hours=3), + ) + ThreatScoreSnapshot.objects.create( + tenant=tenant, + scan=scan_previous, + provider=provider, + compliance_id="prowler_threatscore_aws", + overall_score=Decimal("70.00"), + score_delta=None, + section_scores={"1. IAM": "65.00"}, + critical_requirements=[], + total_requirements=50, + passed_requirements=35, + failed_requirements=15, + manual_requirements=0, + total_findings=40, + passed_findings=25, + failed_findings=15, + ) + + scan_current = Scan.objects.create( + tenant=tenant, + provider=provider, + name="current-threatscore-scan", + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + started_at=timezone.now() - timedelta(hours=2), + completed_at=timezone.now() - timedelta(hours=1), + ) + + mock_scan_summary_filter.return_value.exists.return_value = True + mock_generate_output_directory.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + metrics = { + "overall_score": 85.5, + "score_delta": 10.0, + "section_scores": {"1. IAM": 82.3, "2. Attack Surface": 60.0}, + "critical_requirements": [ + { + "requirement_id": "req_new", + "title": "New high-risk requirement", + "section": "1. IAM", + "subsection": "Root Account", + "risk_level": 5, + "weight": 150, + "passed_findings": 7, + "total_findings": 10, + "description": "Critical requirement description", + } + ], + "total_requirements": 140, + "passed_requirements": 100, + "failed_requirements": 40, + "manual_requirements": 0, + "total_findings": 200, + "passed_findings": 150, + "failed_findings": 50, + } + mock_compute_metrics.return_value = metrics + + result = generate_threatscore_report_job( + tenant_id=str(tenant.id), + scan_id=str(scan_current.id), + provider_id=str(provider.id), + ) + + assert result == {"upload": True} + mock_compute_metrics.assert_called_once_with( + tenant_id=str(tenant.id), + scan_id=str(scan_current.id), + provider_id=str(provider.id), + compliance_id="prowler_threatscore_aws", + min_risk_level=4, + ) + mock_generate_report.assert_called_once() + mock_upload.assert_called_once() + mock_rmtree.assert_called_once() + + snapshots = ThreatScoreSnapshot.objects.filter( + tenant=tenant, provider=provider + ).order_by("inserted_at") + assert snapshots.count() == 2 + + new_snapshot = ThreatScoreSnapshot.objects.get(scan=scan_current) + assert new_snapshot.compliance_id == "prowler_threatscore_aws" + assert Decimal(new_snapshot.overall_score) == Decimal("85.50") + assert Decimal(new_snapshot.score_delta) == Decimal("15.50") + assert new_snapshot.section_scores == metrics["section_scores"] + assert new_snapshot.critical_requirements == metrics["critical_requirements"] + assert new_snapshot.total_requirements == metrics["total_requirements"] + assert new_snapshot.total_findings == metrics["total_findings"] + + +@pytest.mark.django_db +class TestAggregateRequirementStatistics: + """Test suite for _aggregate_requirement_statistics_from_database function.""" + + def test_aggregates_findings_correctly(self, tenants_fixture, scans_fixture): + """Verify correct pass/total counts per check are aggregated from database.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + # Create findings with different check_ids and statuses + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-1", + check_id="check_1", + status=StatusChoices.PASS, + severity=Severity.high, + impact=Severity.high, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-2", + check_id="check_1", + status=StatusChoices.FAIL, + severity=Severity.high, + impact=Severity.high, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-3", + check_id="check_2", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + assert result == { + "check_1": {"passed": 1, "total": 2}, + "check_2": {"passed": 1, "total": 1}, + } + + def test_handles_empty_scan(self, tenants_fixture, scans_fixture): + """Return empty dict when no findings exist for the scan.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + assert result == {} + + def test_multiple_findings_same_check(self, tenants_fixture, scans_fixture): + """Aggregate multiple findings for same check_id correctly.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + # Create 5 findings for same check, 3 passed + for i in range(3): + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid=f"finding-pass-{i}", + check_id="check_same", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + for i in range(2): + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid=f"finding-fail-{i}", + check_id="check_same", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + assert result == {"check_same": {"passed": 3, "total": 5}} + + def test_only_failed_findings(self, tenants_fixture, scans_fixture): + """Correctly count when all findings are FAIL status.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-fail-1", + check_id="check_fail", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-fail-2", + check_id="check_fail", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + assert result == {"check_fail": {"passed": 0, "total": 2}} + + def test_mixed_statuses(self, tenants_fixture, scans_fixture): + """Test with PASS, FAIL, and MANUAL statuses mixed.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-pass", + check_id="check_mixed", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-fail", + check_id="check_mixed", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-manual", + check_id="check_mixed", + status=StatusChoices.MANUAL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + result = _aggregate_requirement_statistics_from_database( + str(tenant.id), str(scan.id) + ) + + # Only PASS status is counted as passed + assert result == {"check_mixed": {"passed": 1, "total": 3}} + + +@pytest.mark.django_db +class TestLoadFindingsForChecks: + """Test suite for _load_findings_for_requirement_checks function.""" + + def test_loads_only_requested_checks( + self, tenants_fixture, scans_fixture, providers_fixture + ): + """Verify only findings for specified check_ids are loaded.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + providers_fixture[0] + + # Create findings with different check_ids + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-1", + check_id="check_requested", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-2", + check_id="check_not_requested", + status=StatusChoices.FAIL, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + mock_provider = MagicMock() + + with patch( + "tasks.jobs.report.FindingOutput.transform_api_finding" + ) as mock_transform: + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_requested" + mock_transform.return_value = mock_finding_output + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), ["check_requested"], mock_provider + ) + + # Only one finding should be loaded + assert "check_requested" in result + assert "check_not_requested" not in result + assert len(result["check_requested"]) == 1 + assert mock_transform.call_count == 1 + + def test_empty_check_ids_returns_empty( + self, tenants_fixture, scans_fixture, providers_fixture + ): + """Return empty dict when check_ids list is empty.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + mock_provider = MagicMock() + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), [], mock_provider + ) + + assert result == {} + + def test_groups_by_check_id( + self, tenants_fixture, scans_fixture, providers_fixture + ): + """Multiple findings for same check are grouped correctly.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + # Create multiple findings for same check + for i in range(3): + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid=f"finding-{i}", + check_id="check_group", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + mock_provider = MagicMock() + + with patch( + "tasks.jobs.report.FindingOutput.transform_api_finding" + ) as mock_transform: + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_group" + mock_transform.return_value = mock_finding_output + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), ["check_group"], mock_provider + ) + + assert len(result["check_group"]) == 3 + + def test_transforms_to_finding_output( + self, tenants_fixture, scans_fixture, providers_fixture + ): + """Findings are transformed using FindingOutput.transform_api_finding.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid="finding-transform", + check_id="check_transform", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + mock_provider = MagicMock() + + with patch( + "tasks.jobs.report.FindingOutput.transform_api_finding" + ) as mock_transform: + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_transform" + mock_transform.return_value = mock_finding_output + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), ["check_transform"], mock_provider + ) + + # Verify transform was called + mock_transform.assert_called_once() + # Verify the transformed output is in the result + assert result["check_transform"][0] == mock_finding_output + + def test_batched_iteration(self, tenants_fixture, scans_fixture, providers_fixture): + """Works correctly with multiple batches of findings.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + # Create enough findings to ensure batching (assuming batch size > 1) + for i in range(10): + Finding.objects.create( + tenant_id=tenant.id, + scan=scan, + uid=f"finding-batch-{i}", + check_id="check_batch", + status=StatusChoices.PASS, + severity=Severity.medium, + impact=Severity.medium, + check_metadata={}, + raw_result={}, + ) + + mock_provider = MagicMock() + + with patch( + "tasks.jobs.report.FindingOutput.transform_api_finding" + ) as mock_transform: + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_batch" + mock_transform.return_value = mock_finding_output + + result = _load_findings_for_requirement_checks( + str(tenant.id), str(scan.id), ["check_batch"], mock_provider + ) + + # All 10 findings should be loaded regardless of batching + assert len(result["check_batch"]) == 10 + assert mock_transform.call_count == 10 + + +@pytest.mark.django_db +class TestCalculateRequirementsData: + """Test suite for _calculate_requirements_data_from_statistics function.""" + + def test_requirement_status_all_pass(self): + """Status is PASS when all findings for requirement checks pass.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_requirement = MagicMock() + mock_requirement.Id = "req_1" + mock_requirement.Description = "Test requirement" + mock_requirement.Checks = ["check_1", "check_2"] + mock_requirement.Attributes = [MagicMock()] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = { + "check_1": {"passed": 5, "total": 5}, + "check_2": {"passed": 3, "total": 3}, + } + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + assert len(requirements_list) == 1 + assert requirements_list[0]["attributes"]["status"] == StatusChoices.PASS + assert requirements_list[0]["attributes"]["passed_findings"] == 8 + assert requirements_list[0]["attributes"]["total_findings"] == 8 + + def test_requirement_status_some_fail(self): + """Status is FAIL when some findings fail.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_requirement = MagicMock() + mock_requirement.Id = "req_2" + mock_requirement.Description = "Test requirement with failures" + mock_requirement.Checks = ["check_3"] + mock_requirement.Attributes = [MagicMock()] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = { + "check_3": {"passed": 2, "total": 5}, + } + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + assert len(requirements_list) == 1 + assert requirements_list[0]["attributes"]["status"] == StatusChoices.FAIL + assert requirements_list[0]["attributes"]["passed_findings"] == 2 + assert requirements_list[0]["attributes"]["total_findings"] == 5 + + def test_requirement_status_no_findings(self): + """Status is MANUAL when no findings exist for requirement.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_requirement = MagicMock() + mock_requirement.Id = "req_3" + mock_requirement.Description = "Manual requirement" + mock_requirement.Checks = ["check_nonexistent"] + mock_requirement.Attributes = [MagicMock()] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = {} + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + assert len(requirements_list) == 1 + assert requirements_list[0]["attributes"]["status"] == StatusChoices.MANUAL + assert requirements_list[0]["attributes"]["passed_findings"] == 0 + assert requirements_list[0]["attributes"]["total_findings"] == 0 + + def test_aggregates_multiple_checks(self): + """Correctly sum stats across multiple checks in requirement.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_requirement = MagicMock() + mock_requirement.Id = "req_4" + mock_requirement.Description = "Multi-check requirement" + mock_requirement.Checks = ["check_a", "check_b", "check_c"] + mock_requirement.Attributes = [MagicMock()] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = { + "check_a": {"passed": 10, "total": 15}, + "check_b": {"passed": 5, "total": 10}, + "check_c": {"passed": 0, "total": 5}, + } + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + assert len(requirements_list) == 1 + # 10 + 5 + 0 = 15 passed + assert requirements_list[0]["attributes"]["passed_findings"] == 15 + # 15 + 10 + 5 = 30 total + assert requirements_list[0]["attributes"]["total_findings"] == 30 + # Not all passed, so should be FAIL + assert requirements_list[0]["attributes"]["status"] == StatusChoices.FAIL + + def test_returns_correct_structure(self): + """Verify tuple structure and dict keys are correct.""" + mock_compliance = MagicMock() + mock_compliance.Framework = "TestFramework" + mock_compliance.Version = "1.0" + + mock_attribute = MagicMock() + mock_requirement = MagicMock() + mock_requirement.Id = "req_5" + mock_requirement.Description = "Structure test" + mock_requirement.Checks = ["check_struct"] + mock_requirement.Attributes = [mock_attribute] + + mock_compliance.Requirements = [mock_requirement] + + requirement_statistics = {"check_struct": {"passed": 1, "total": 1}} + + attributes_by_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + mock_compliance, requirement_statistics + ) + ) + + # Verify attributes_by_id structure + assert "req_5" in attributes_by_id + assert "attributes" in attributes_by_id["req_5"] + assert "description" in attributes_by_id["req_5"] + assert "req_attributes" in attributes_by_id["req_5"]["attributes"] + assert "checks" in attributes_by_id["req_5"]["attributes"] + + # Verify requirements_list structure + assert len(requirements_list) == 1 + req = requirements_list[0] + assert "id" in req + assert "attributes" in req + assert "framework" in req["attributes"] + assert "version" in req["attributes"] + assert "status" in req["attributes"] + assert "description" in req["attributes"] + assert "passed_findings" in req["attributes"] + assert "total_findings" in req["attributes"] + + +@pytest.mark.django_db +class TestGenerateThreatscoreReportFunction: + def setup_method(self): + self.scan_id = str(uuid.uuid4()) + self.provider_id = str(uuid.uuid4()) + self.tenant_id = str(uuid.uuid4()) + self.compliance_id = "prowler_threatscore_aws" + self.output_path = "/tmp/test_threatscore_report.pdf" + + @patch("tasks.jobs.report.initialize_prowler_provider") + @patch("tasks.jobs.report.Provider.objects.get") + @patch("tasks.jobs.report.Compliance.get_bulk") + @patch("tasks.jobs.report._aggregate_requirement_statistics_from_database") + @patch("tasks.jobs.report._calculate_requirements_data_from_statistics") + @patch("tasks.jobs.report._load_findings_for_requirement_checks") + @patch("tasks.jobs.report.SimpleDocTemplate") + @patch("tasks.jobs.report.Image") + @patch("tasks.jobs.report.Spacer") + @patch("tasks.jobs.report.Paragraph") + @patch("tasks.jobs.report.PageBreak") + @patch("tasks.jobs.report.Table") + @patch("tasks.jobs.report.TableStyle") + @patch("tasks.jobs.report.plt.subplots") + @patch("tasks.jobs.report.plt.savefig") + @patch("tasks.jobs.report.io.BytesIO") + def test_generate_threatscore_report_success( + self, + mock_bytesio, + mock_savefig, + mock_subplots, + mock_table_style, + mock_table, + mock_page_break, + mock_paragraph, + mock_spacer, + mock_image, + mock_doc_template, + mock_load_findings, + mock_calculate_requirements, + mock_aggregate_statistics, + mock_compliance_get_bulk, + mock_provider_get, + mock_initialize_provider, + ): + """Test the updated generate_threatscore_report using new memory-efficient architecture.""" + mock_provider = MagicMock() + mock_provider.provider = "aws" + mock_provider_get.return_value = mock_provider + + prowler_provider = MagicMock() + mock_initialize_provider.return_value = prowler_provider + + # Mock compliance object with requirements + mock_compliance_obj = MagicMock() + mock_compliance_obj.Framework = "ProwlerThreatScore" + mock_compliance_obj.Version = "1.0" + mock_compliance_obj.Description = "Test Description" + + # Configure requirement with properly set numeric attributes for chart generation + mock_requirement = MagicMock() + mock_requirement.Id = "req_1" + mock_requirement.Description = "Test requirement" + mock_requirement.Checks = ["check_1"] + + # Create a properly configured attribute mock with numeric values + mock_requirement_attr = MagicMock() + mock_requirement_attr.Section = "1. IAM" + mock_requirement_attr.SubSection = "1.1 Identity" + mock_requirement_attr.Title = "Test Requirement Title" + mock_requirement_attr.LevelOfRisk = 3 + mock_requirement_attr.Weight = 100 + mock_requirement_attr.AttributeDescription = "Test requirement description" + mock_requirement_attr.AdditionalInformation = "Additional test information" + + mock_requirement.Attributes = [mock_requirement_attr] + mock_compliance_obj.Requirements = [mock_requirement] + + mock_compliance_get_bulk.return_value = { + self.compliance_id: mock_compliance_obj + } + + # Mock the aggregated statistics from database + mock_aggregate_statistics.return_value = {"check_1": {"passed": 5, "total": 10}} + + # Mock the calculated requirements data with properly configured attributes + mock_attributes_by_id = { + "req_1": { + "attributes": { + "req_attributes": [mock_requirement_attr], + "checks": ["check_1"], + }, + "description": "Test requirement", + } + } + mock_requirements_list = [ + { + "id": "req_1", + "attributes": { + "framework": "ProwlerThreatScore", + "version": "1.0", + "status": StatusChoices.FAIL, + "description": "Test requirement", + "passed_findings": 5, + "total_findings": 10, + }, + } + ] + mock_calculate_requirements.return_value = ( + mock_attributes_by_id, + mock_requirements_list, + ) + + # Mock the on-demand loaded findings + mock_finding_output = MagicMock() + mock_finding_output.check_id = "check_1" + mock_finding_output.status = "FAIL" + mock_finding_output.metadata = MagicMock() + mock_finding_output.metadata.CheckTitle = "Test Check" + mock_finding_output.metadata.Severity = "HIGH" + mock_finding_output.resource_name = "test-resource" + mock_finding_output.region = "us-east-1" + + mock_load_findings.return_value = {"check_1": [mock_finding_output]} + + # Mock PDF generation components + mock_doc = MagicMock() + mock_doc_template.return_value = mock_doc + + mock_fig, mock_ax = MagicMock(), MagicMock() + mock_subplots.return_value = (mock_fig, mock_ax) + mock_buffer = MagicMock() + mock_bytesio.return_value = mock_buffer + + mock_image.return_value = MagicMock() + mock_spacer.return_value = MagicMock() + mock_paragraph.return_value = MagicMock() + mock_page_break.return_value = MagicMock() + mock_table.return_value = MagicMock() + mock_table_style.return_value = MagicMock() + + # Execute the function + generate_threatscore_report( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + compliance_id=self.compliance_id, + output_path=self.output_path, + provider_id=self.provider_id, + only_failed=True, + min_risk_level=4, + ) + + # Verify the new workflow was followed + mock_provider_get.assert_called_once_with(id=self.provider_id) + mock_initialize_provider.assert_called_once_with(mock_provider) + mock_compliance_get_bulk.assert_called_once_with("aws") + + # Verify the new functions were called in correct order with correct parameters + mock_aggregate_statistics.assert_called_once_with(self.tenant_id, self.scan_id) + mock_calculate_requirements.assert_called_once_with( + mock_compliance_obj, {"check_1": {"passed": 5, "total": 10}} + ) + mock_load_findings.assert_called_once_with( + self.tenant_id, self.scan_id, ["check_1"], prowler_provider + ) + + # Verify PDF was built + mock_doc_template.assert_called_once() + mock_doc.build.assert_called_once() + + @patch("tasks.jobs.report.initialize_prowler_provider") + @patch("tasks.jobs.report.Provider.objects.get") + @patch("tasks.jobs.report.Compliance.get_bulk") + @patch("tasks.jobs.report.Finding.all_objects.filter") + def test_generate_threatscore_report_exception_handling( + self, + mock_finding_filter, + mock_compliance_get_bulk, + mock_provider_get, + mock_initialize_provider, + ): + mock_provider_get.side_effect = Exception("Provider not found") + + with pytest.raises(Exception, match="Provider not found"): + generate_threatscore_report( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + compliance_id=self.compliance_id, + output_path=self.output_path, + provider_id=self.provider_id, + only_failed=True, + min_risk_level=4, + ) + + +@pytest.mark.django_db +class TestGenerateThreatscoreReportTask: + def setup_method(self): + self.scan_id = str(uuid.uuid4()) + self.provider_id = str(uuid.uuid4()) + self.tenant_id = str(uuid.uuid4()) + + @patch("tasks.tasks.generate_threatscore_report_job") + def test_generate_threatscore_report_task_calls_job(self, mock_generate_job): + mock_generate_job.return_value = {"upload": True} + + result = generate_threatscore_report_task( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + assert result == {"upload": True} + mock_generate_job.assert_called_once_with( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) + + @patch("tasks.tasks.generate_threatscore_report_job") + def test_generate_threatscore_report_task_handles_job_exception( + self, mock_generate_job + ): + mock_generate_job.side_effect = Exception("Job failed") + + with pytest.raises(Exception, match="Job failed"): + generate_threatscore_report_task( + tenant_id=self.tenant_id, + scan_id=self.scan_id, + provider_id=self.provider_id, + ) diff --git a/api/src/backend/tasks/tests/test_scan.py b/api/src/backend/tasks/tests/test_scan.py index dcbec12956..e1c938f9f1 100644 --- a/api/src/backend/tasks/tests/test_scan.py +++ b/api/src/backend/tasks/tests/test_scan.py @@ -1,19 +1,32 @@ +import csv import json import uuid -from datetime import datetime +from datetime import datetime, timezone +from io import StringIO from unittest.mock import MagicMock, patch import pytest from tasks.jobs.scan import ( + _copy_compliance_requirement_rows, _create_finding_delta, + _persist_compliance_requirement_rows, _store_resources, create_compliance_requirements, perform_prowler_scan, ) from tasks.utils import CustomEncoder +from api.db_router import MainRouter from api.exceptions import ProviderConnectionError -from api.models import Finding, Provider, Resource, Scan, StateChoices, StatusChoices +from api.models import ( + Finding, + MuteRule, + Provider, + Resource, + Scan, + StateChoices, + StatusChoices, +) from prowler.lib.check.models import Severity @@ -734,6 +747,561 @@ class TestPerformScan: # Assert that failed_findings_count was reset to 0 during the scan assert resource.failed_findings_count == 0 + def test_perform_prowler_scan_with_active_mute_rules( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test active MuteRule mutes findings with correct reason""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create active MuteRule with specific finding UIDs + mute_rule_reason = "Accepted risk - production exception" + finding_uid_1 = "finding_to_mute_1" + finding_uid_2 = "finding_to_mute_2" + + MuteRule.objects.create( + tenant_id=tenant_id, + name="Production Exception Rule", + reason=mute_rule_reason, + enabled=True, + finding_uids=[finding_uid_1, finding_uid_2], + ) + + # Mock findings: one FAIL and one PASS, both should be muted + muted_fail_finding = MagicMock() + muted_fail_finding.uid = finding_uid_1 + muted_fail_finding.status = StatusChoices.FAIL + muted_fail_finding.status_extended = "muted fail" + muted_fail_finding.severity = Severity.high + muted_fail_finding.check_id = "muted_fail_check" + muted_fail_finding.get_metadata.return_value = {"key": "value"} + muted_fail_finding.resource_uid = "resource_uid_1" + muted_fail_finding.resource_name = "resource_1" + muted_fail_finding.region = "us-east-1" + muted_fail_finding.service_name = "ec2" + muted_fail_finding.resource_type = "instance" + muted_fail_finding.resource_tags = {} + muted_fail_finding.muted = False + muted_fail_finding.raw = {} + muted_fail_finding.resource_metadata = {} + muted_fail_finding.resource_details = {} + muted_fail_finding.partition = "aws" + muted_fail_finding.compliance = {} + + muted_pass_finding = MagicMock() + muted_pass_finding.uid = finding_uid_2 + muted_pass_finding.status = StatusChoices.PASS + muted_pass_finding.status_extended = "muted pass" + muted_pass_finding.severity = Severity.medium + muted_pass_finding.check_id = "muted_pass_check" + muted_pass_finding.get_metadata.return_value = {"key": "value"} + muted_pass_finding.resource_uid = "resource_uid_2" + muted_pass_finding.resource_name = "resource_2" + muted_pass_finding.region = "us-east-1" + muted_pass_finding.service_name = "s3" + muted_pass_finding.resource_type = "bucket" + muted_pass_finding.resource_tags = {} + muted_pass_finding.muted = False + muted_pass_finding.raw = {} + muted_pass_finding.resource_metadata = {} + muted_pass_finding.resource_details = {} + muted_pass_finding.partition = "aws" + muted_pass_finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [ + (100, [muted_fail_finding, muted_pass_finding]) + ] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify findings are muted with correct reason + fail_finding_db = Finding.objects.get(uid=finding_uid_1) + pass_finding_db = Finding.objects.get(uid=finding_uid_2) + + assert fail_finding_db.muted + assert fail_finding_db.muted_reason == mute_rule_reason + assert fail_finding_db.muted_at is not None + + assert pass_finding_db.muted + assert pass_finding_db.muted_reason == mute_rule_reason + assert pass_finding_db.muted_at is not None + + # Verify failed_findings_count is 0 for muted FAIL finding + resource_1 = Resource.objects.get(uid="resource_uid_1") + assert resource_1.failed_findings_count == 0 + + def test_perform_prowler_scan_with_inactive_mute_rules( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test inactive MuteRule does not mute findings""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create inactive MuteRule + finding_uid = "finding_inactive_rule" + MuteRule.objects.create( + tenant_id=tenant_id, + name="Inactive Rule", + reason="Should not apply", + enabled=False, + finding_uids=[finding_uid], + ) + + # Mock FAIL finding + fail_finding = MagicMock() + fail_finding.uid = finding_uid + fail_finding.status = StatusChoices.FAIL + fail_finding.status_extended = "test fail" + fail_finding.severity = Severity.high + fail_finding.check_id = "fail_check" + fail_finding.get_metadata.return_value = {"key": "value"} + fail_finding.resource_uid = "resource_uid_inactive" + fail_finding.resource_name = "resource_inactive" + fail_finding.region = "us-east-1" + fail_finding.service_name = "ec2" + fail_finding.resource_type = "instance" + fail_finding.resource_tags = {} + fail_finding.muted = False + fail_finding.raw = {} + fail_finding.resource_metadata = {} + fail_finding.resource_details = {} + fail_finding.partition = "aws" + fail_finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, [fail_finding])] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify finding is NOT muted + finding_db = Finding.objects.get(uid=finding_uid) + assert not finding_db.muted + assert finding_db.muted_reason is None + assert finding_db.muted_at is None + + # Verify failed_findings_count increments for FAIL finding + resource = Resource.objects.get(uid="resource_uid_inactive") + assert resource.failed_findings_count == 1 + + def test_perform_prowler_scan_mutelist_overrides_mute_rules( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test mutelist processor takes precedence over MuteRule""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create active MuteRule + finding_uid = "finding_both_rules" + MuteRule.objects.create( + tenant_id=tenant_id, + name="Manual Mute Rule", + reason="Muted by manual rule", + enabled=True, + finding_uids=[finding_uid], + ) + + # Mock finding with mutelist processor muted=True + muted_finding = MagicMock() + muted_finding.uid = finding_uid + muted_finding.status = StatusChoices.FAIL + muted_finding.status_extended = "test" + muted_finding.severity = Severity.high + muted_finding.check_id = "test_check" + muted_finding.get_metadata.return_value = {"key": "value"} + muted_finding.resource_uid = "resource_both" + muted_finding.resource_name = "resource_both" + muted_finding.region = "us-east-1" + muted_finding.service_name = "ec2" + muted_finding.resource_type = "instance" + muted_finding.resource_tags = {} + muted_finding.muted = True + muted_finding.raw = {} + muted_finding.resource_metadata = {} + muted_finding.resource_details = {} + muted_finding.partition = "aws" + muted_finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, [muted_finding])] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify mutelist reason takes precedence + finding_db = Finding.objects.get(uid=finding_uid) + assert finding_db.muted + assert finding_db.muted_reason == "Muted by mutelist" + assert finding_db.muted_at is not None + + # Verify failed_findings_count is 0 + resource = Resource.objects.get(uid="resource_both") + assert resource.failed_findings_count == 0 + + def test_perform_prowler_scan_mute_rules_multiple_findings( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test MuteRule with multiple finding UIDs mutes all findings""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create MuteRule with multiple finding UIDs + mute_rule_reason = "Bulk exception for dev environment" + finding_uids = [ + "bulk_finding_1", + "bulk_finding_2", + "bulk_finding_3", + "bulk_finding_4", + ] + MuteRule.objects.create( + tenant_id=tenant_id, + name="Bulk Mute Rule", + reason=mute_rule_reason, + enabled=True, + finding_uids=finding_uids, + ) + + # Mock multiple findings with mixed statuses + findings = [] + for i, uid in enumerate(finding_uids): + finding = MagicMock() + finding.uid = uid + finding.status = ( + StatusChoices.FAIL if i % 2 == 0 else StatusChoices.PASS + ) + finding.status_extended = f"test {i}" + finding.severity = Severity.medium + finding.check_id = f"check_{i}" + finding.get_metadata.return_value = {"key": f"value_{i}"} + finding.resource_uid = f"resource_bulk_{i}" + finding.resource_name = f"resource_{i}" + finding.region = "us-west-2" + finding.service_name = "lambda" + finding.resource_type = "function" + finding.resource_tags = {} + finding.muted = False + finding.raw = {} + finding.resource_metadata = {} + finding.resource_details = {} + finding.partition = "aws" + finding.compliance = {} + findings.append(finding) + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, findings)] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-west-2"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify all findings are muted with same reason + for uid in finding_uids: + finding_db = Finding.objects.get(uid=uid) + assert finding_db.muted + assert finding_db.muted_reason == mute_rule_reason + assert finding_db.muted_at is not None + + # Verify all resources have failed_findings_count = 0 + for i in range(len(finding_uids)): + resource = Resource.objects.get(uid=f"resource_bulk_{i}") + assert resource.failed_findings_count == 0 + + def test_perform_prowler_scan_mute_rules_error_handling( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test scan continues when MuteRule loading fails""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + patch("api.models.MuteRule.objects.filter") as mock_mute_rule_filter, + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Mock MuteRule.objects.filter to raise exception + mock_mute_rule_filter.side_effect = Exception("Database error") + + # Mock finding + finding = MagicMock() + finding.uid = "finding_error_handling" + finding.status = StatusChoices.FAIL + finding.status_extended = "test" + finding.severity = Severity.high + finding.check_id = "test_check" + finding.get_metadata.return_value = {"key": "value"} + finding.resource_uid = "resource_error" + finding.resource_name = "resource_error" + finding.region = "us-east-1" + finding.service_name = "ec2" + finding.resource_type = "instance" + finding.resource_tags = {} + finding.muted = False + finding.raw = {} + finding.resource_metadata = {} + finding.resource_details = {} + finding.partition = "aws" + finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, [finding])] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test - should not raise + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify scan completed successfully + scan.refresh_from_db() + assert scan.state == StateChoices.COMPLETED + + # Verify finding is not muted (mute_rules_cache was empty dict) + finding_db = Finding.objects.get(uid="finding_error_handling") + assert not finding_db.muted + assert finding_db.muted_reason is None + + # Verify failed_findings_count increments + resource = Resource.objects.get(uid="resource_error") + assert resource.failed_findings_count == 1 + + def test_perform_prowler_scan_muted_at_timestamp( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test muted_at timestamp is set correctly for muted findings""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create active MuteRule + finding_uid = "finding_timestamp_test" + MuteRule.objects.create( + tenant_id=tenant_id, + name="Timestamp Test Rule", + reason="Testing timestamp", + enabled=True, + finding_uids=[finding_uid], + ) + + # Mock finding + finding = MagicMock() + finding.uid = finding_uid + finding.status = StatusChoices.FAIL + finding.status_extended = "test" + finding.severity = Severity.high + finding.check_id = "test_check" + finding.get_metadata.return_value = {"key": "value"} + finding.resource_uid = "resource_timestamp" + finding.resource_name = "resource_timestamp" + finding.region = "us-east-1" + finding.service_name = "ec2" + finding.resource_type = "instance" + finding.resource_tags = {} + finding.muted = False + finding.raw = {} + finding.resource_metadata = {} + finding.resource_details = {} + finding.partition = "aws" + finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, [finding])] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Capture time before and after scan + before_scan = datetime.now(timezone.utc) + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + after_scan = datetime.now(timezone.utc) + + # Verify muted_at is within the scan time window + finding_db = Finding.objects.get(uid=finding_uid) + assert finding_db.muted + assert finding_db.muted_at is not None + assert before_scan <= finding_db.muted_at <= after_scan + # TODO Add tests for aggregations @@ -1045,3 +1613,773 @@ class TestCreateComplianceRequirements: assert "requirements_created" in result assert result["requirements_created"] >= 0 + + +class TestComplianceRequirementCopy: + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_streams_csv( + self, mock_psycopg_connection, settings + ): + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + captured = {} + + def copy_side_effect(sql, file_obj): + captured["sql"] = sql + captured["data"] = file_obj.read() + + cursor.copy_expert.side_effect = copy_side_effect + + row = { + "id": uuid.uuid4(), + "tenant_id": str(uuid.uuid4()), + "compliance_id": "cisa_aws", + "framework": "CISA", + "version": None, + "description": "desc", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": uuid.uuid4(), + } + + with patch.object(MainRouter, "admin_db", "admin"): + _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + + mock_psycopg_connection.assert_called_once_with("admin") + connection.cursor.assert_called_once() + cursor.execute.assert_called_once() + cursor.copy_expert.assert_called_once() + + csv_rows = list(csv.reader(StringIO(captured["data"]))) + assert csv_rows[0][0] == str(row["id"]) + assert csv_rows[0][5] == "" + assert csv_rows[0][-1] == str(row["scan_id"]) + + @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + @patch( + "tasks.jobs.scan._copy_compliance_requirement_rows", + side_effect=Exception("copy failed"), + ) + def test_persist_compliance_requirement_rows_fallback( + self, mock_copy, mock_rls_transaction, mock_bulk_create + ): + inserted_at = datetime.now(timezone.utc) + row = { + "id": uuid.uuid4(), + "tenant_id": str(uuid.uuid4()), + "inserted_at": inserted_at, + "compliance_id": "cisa_aws", + "framework": "CISA", + "version": "1.0", + "description": "desc", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": uuid.uuid4(), + } + + tenant_id = row["tenant_id"] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + _persist_compliance_requirement_rows(tenant_id, [row]) + + mock_copy.assert_called_once_with(tenant_id, [row]) + mock_rls_transaction.assert_called_once_with(tenant_id) + mock_bulk_create.assert_called_once() + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + assert len(objects) == 1 + fallback = objects[0] + assert fallback.version == row["version"] + assert fallback.compliance_id == row["compliance_id"] + + @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + @patch("tasks.jobs.scan._copy_compliance_requirement_rows") + def test_persist_compliance_requirement_rows_no_rows( + self, mock_copy, mock_rls_transaction, mock_bulk_create + ): + _persist_compliance_requirement_rows(str(uuid.uuid4()), []) + + mock_copy.assert_not_called() + mock_rls_transaction.assert_not_called() + mock_bulk_create.assert_not_called() + + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_multiple_rows( + self, mock_psycopg_connection, settings + ): + """Test COPY with multiple rows to ensure batch processing works correctly.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + captured = {} + + def copy_side_effect(sql, file_obj): + captured["sql"] = sql + captured["data"] = file_obj.read() + + cursor.copy_expert.side_effect = copy_side_effect + + tenant_id = str(uuid.uuid4()) + scan_id = uuid.uuid4() + inserted_at = datetime.now(timezone.utc) + + rows = [ + { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "inserted_at": inserted_at, + "compliance_id": "cisa_aws", + "framework": "CISA", + "version": "1.0", + "description": "First requirement", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 5, + "failed_checks": 0, + "total_checks": 5, + "scan_id": scan_id, + }, + { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "inserted_at": inserted_at, + "compliance_id": "cisa_aws", + "framework": "CISA", + "version": "1.0", + "description": "Second requirement", + "region": "us-west-2", + "requirement_id": "req-2", + "requirement_status": "FAIL", + "passed_checks": 3, + "failed_checks": 2, + "total_checks": 5, + "scan_id": scan_id, + }, + { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "inserted_at": inserted_at, + "compliance_id": "aws_foundational_security_aws", + "framework": "AWS-Foundational-Security-Best-Practices", + "version": "2.0", + "description": "Third requirement", + "region": "eu-west-1", + "requirement_id": "req-3", + "requirement_status": "MANUAL", + "passed_checks": 0, + "failed_checks": 0, + "total_checks": 3, + "scan_id": scan_id, + }, + ] + + with patch.object(MainRouter, "admin_db", "admin"): + _copy_compliance_requirement_rows(tenant_id, rows) + + mock_psycopg_connection.assert_called_once_with("admin") + connection.cursor.assert_called_once() + cursor.execute.assert_called_once() + cursor.copy_expert.assert_called_once() + + csv_rows = list(csv.reader(StringIO(captured["data"]))) + assert len(csv_rows) == 3 + + # Validate first row + assert csv_rows[0][0] == str(rows[0]["id"]) + assert csv_rows[0][1] == tenant_id + assert csv_rows[0][3] == "cisa_aws" + assert csv_rows[0][4] == "CISA" + assert csv_rows[0][6] == "First requirement" + assert csv_rows[0][7] == "us-east-1" + assert csv_rows[0][10] == "5" + assert csv_rows[0][11] == "0" + assert csv_rows[0][12] == "5" + + # Validate second row + assert csv_rows[1][0] == str(rows[1]["id"]) + assert csv_rows[1][7] == "us-west-2" + assert csv_rows[1][9] == "FAIL" + assert csv_rows[1][10] == "3" + assert csv_rows[1][11] == "2" + + # Validate third row + assert csv_rows[2][0] == str(rows[2]["id"]) + assert csv_rows[2][3] == "aws_foundational_security_aws" + assert csv_rows[2][5] == "2.0" + assert csv_rows[2][9] == "MANUAL" + + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_null_values( + self, mock_psycopg_connection, settings + ): + """Test COPY handles NULL/None values correctly in nullable fields.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + captured = {} + + def copy_side_effect(sql, file_obj): + captured["sql"] = sql + captured["data"] = file_obj.read() + + cursor.copy_expert.side_effect = copy_side_effect + + # Row with all nullable fields set to None/empty + row = { + "id": uuid.uuid4(), + "tenant_id": str(uuid.uuid4()), + "compliance_id": "test_framework", + "framework": "Test", + "version": None, # nullable + "description": None, # nullable + "region": "", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 0, + "failed_checks": 0, + "total_checks": 0, + "scan_id": uuid.uuid4(), + } + + with patch.object(MainRouter, "admin_db", "admin"): + _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + + csv_rows = list(csv.reader(StringIO(captured["data"]))) + assert len(csv_rows) == 1 + + # Validate that None values are converted to empty strings in CSV + assert csv_rows[0][5] == "" # version + assert csv_rows[0][6] == "" # description + + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_special_characters( + self, mock_psycopg_connection, settings + ): + """Test COPY correctly escapes special characters in CSV.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + captured = {} + + def copy_side_effect(sql, file_obj): + captured["sql"] = sql + captured["data"] = file_obj.read() + + cursor.copy_expert.side_effect = copy_side_effect + + # Row with special characters that need escaping + row = { + "id": uuid.uuid4(), + "tenant_id": str(uuid.uuid4()), + "compliance_id": 'framework"with"quotes', + "framework": "Framework,with,commas", + "version": "1.0", + "description": 'Description with "quotes", commas, and\nnewlines', + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": uuid.uuid4(), + } + + with patch.object(MainRouter, "admin_db", "admin"): + _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + + # Verify CSV was generated (csv module handles escaping automatically) + csv_rows = list(csv.reader(StringIO(captured["data"]))) + assert len(csv_rows) == 1 + + # Verify special characters are preserved after CSV parsing + assert csv_rows[0][3] == 'framework"with"quotes' + assert csv_rows[0][4] == "Framework,with,commas" + assert "quotes" in csv_rows[0][6] + assert "commas" in csv_rows[0][6] + + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_missing_inserted_at( + self, mock_psycopg_connection, settings + ): + """Test COPY uses current datetime when inserted_at is missing.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + captured = {} + + def copy_side_effect(sql, file_obj): + captured["sql"] = sql + captured["data"] = file_obj.read() + + cursor.copy_expert.side_effect = copy_side_effect + + # Row without inserted_at field + row = { + "id": uuid.uuid4(), + "tenant_id": str(uuid.uuid4()), + "compliance_id": "test_framework", + "framework": "Test", + "version": "1.0", + "description": "desc", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": uuid.uuid4(), + # Note: inserted_at is intentionally missing + } + + before_call = datetime.now(timezone.utc) + with patch.object(MainRouter, "admin_db", "admin"): + _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + after_call = datetime.now(timezone.utc) + + csv_rows = list(csv.reader(StringIO(captured["data"]))) + assert len(csv_rows) == 1 + + # Verify inserted_at was auto-generated and is a valid ISO datetime + inserted_at_str = csv_rows[0][2] + inserted_at = datetime.fromisoformat(inserted_at_str) + assert before_call <= inserted_at <= after_call + + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_transaction_rollback_on_copy_error( + self, mock_psycopg_connection, settings + ): + """Test transaction is rolled back when copy_expert fails.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + # Simulate copy_expert failure + cursor.copy_expert.side_effect = Exception("COPY command failed") + + row = { + "id": uuid.uuid4(), + "tenant_id": str(uuid.uuid4()), + "compliance_id": "test", + "framework": "Test", + "version": "1.0", + "description": "desc", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": uuid.uuid4(), + } + + with patch.object(MainRouter, "admin_db", "admin"): + with pytest.raises(Exception, match="COPY command failed"): + _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + + # Verify rollback was called + connection.rollback.assert_called_once() + connection.commit.assert_not_called() + + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_transaction_rollback_on_set_config_error( + self, mock_psycopg_connection, settings + ): + """Test transaction is rolled back when SET_CONFIG fails.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + # Simulate cursor.execute failure + cursor.execute.side_effect = Exception("SET prowler.tenant_id failed") + + row = { + "id": uuid.uuid4(), + "tenant_id": str(uuid.uuid4()), + "compliance_id": "test", + "framework": "Test", + "version": "1.0", + "description": "desc", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": uuid.uuid4(), + } + + with patch.object(MainRouter, "admin_db", "admin"): + with pytest.raises(Exception, match="SET prowler.tenant_id failed"): + _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + + # Verify rollback was called + connection.rollback.assert_called_once() + connection.commit.assert_not_called() + + @patch("tasks.jobs.scan.psycopg_connection") + def test_copy_compliance_requirement_rows_commit_on_success( + self, mock_psycopg_connection, settings + ): + """Test transaction is committed on successful COPY.""" + settings.DATABASES.setdefault("admin", settings.DATABASES["default"]) + + connection = MagicMock() + cursor = MagicMock() + cursor_context = MagicMock() + cursor_context.__enter__.return_value = cursor + cursor_context.__exit__.return_value = False + connection.cursor.return_value = cursor_context + connection.__enter__.return_value = connection + connection.__exit__.return_value = False + + context_manager = MagicMock() + context_manager.__enter__.return_value = connection + context_manager.__exit__.return_value = False + mock_psycopg_connection.return_value = context_manager + + cursor.copy_expert.return_value = None # Success + + row = { + "id": uuid.uuid4(), + "tenant_id": str(uuid.uuid4()), + "compliance_id": "test", + "framework": "Test", + "version": "1.0", + "description": "desc", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": uuid.uuid4(), + } + + with patch.object(MainRouter, "admin_db", "admin"): + _copy_compliance_requirement_rows(str(row["tenant_id"]), [row]) + + # Verify commit was called and rollback was not + connection.commit.assert_called_once() + connection.rollback.assert_not_called() + # Verify autocommit was disabled + assert connection.autocommit is False + + @patch("tasks.jobs.scan._copy_compliance_requirement_rows") + def test_persist_compliance_requirement_rows_success(self, mock_copy): + """Test successful COPY path without fallback to ORM.""" + mock_copy.return_value = None # Success, no exception + + tenant_id = str(uuid.uuid4()) + rows = [ + { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "inserted_at": datetime.now(timezone.utc), + "compliance_id": "test", + "framework": "Test", + "version": "1.0", + "description": "desc", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": uuid.uuid4(), + } + ] + + _persist_compliance_requirement_rows(tenant_id, rows) + + # Verify COPY was called + mock_copy.assert_called_once_with(tenant_id, rows) + + @patch("tasks.jobs.scan.logger") + @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + @patch( + "tasks.jobs.scan._copy_compliance_requirement_rows", + side_effect=Exception("COPY failed"), + ) + def test_persist_compliance_requirement_rows_fallback_logging( + self, mock_copy, mock_rls_transaction, mock_bulk_create, mock_logger + ): + """Test logger.exception is called when COPY fails and fallback occurs.""" + tenant_id = str(uuid.uuid4()) + row = { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "inserted_at": datetime.now(timezone.utc), + "compliance_id": "test", + "framework": "Test", + "version": "1.0", + "description": "desc", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 1, + "failed_checks": 0, + "total_checks": 1, + "scan_id": uuid.uuid4(), + } + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + _persist_compliance_requirement_rows(tenant_id, [row]) + + # Verify logger.exception was called + mock_logger.exception.assert_called_once() + args, kwargs = mock_logger.exception.call_args + assert "COPY bulk insert" in args[0] + assert "falling back to ORM" in args[0] + assert kwargs.get("exc_info") is not None + + @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + @patch( + "tasks.jobs.scan._copy_compliance_requirement_rows", + side_effect=Exception("copy failed"), + ) + def test_persist_compliance_requirement_rows_fallback_multiple_rows( + self, mock_copy, mock_rls_transaction, mock_bulk_create + ): + """Test ORM fallback with multiple rows.""" + tenant_id = str(uuid.uuid4()) + scan_id = uuid.uuid4() + inserted_at = datetime.now(timezone.utc) + + rows = [ + { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "inserted_at": inserted_at, + "compliance_id": "cisa_aws", + "framework": "CISA", + "version": "1.0", + "description": "First requirement", + "region": "us-east-1", + "requirement_id": "req-1", + "requirement_status": "PASS", + "passed_checks": 5, + "failed_checks": 0, + "total_checks": 5, + "scan_id": scan_id, + }, + { + "id": uuid.uuid4(), + "tenant_id": tenant_id, + "inserted_at": inserted_at, + "compliance_id": "cisa_aws", + "framework": "CISA", + "version": "1.0", + "description": "Second requirement", + "region": "us-west-2", + "requirement_id": "req-2", + "requirement_status": "FAIL", + "passed_checks": 2, + "failed_checks": 3, + "total_checks": 5, + "scan_id": scan_id, + }, + ] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + _persist_compliance_requirement_rows(tenant_id, rows) + + mock_copy.assert_called_once_with(tenant_id, rows) + mock_rls_transaction.assert_called_once_with(tenant_id) + mock_bulk_create.assert_called_once() + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + assert len(objects) == 2 + assert kwargs["batch_size"] == 500 + + # Validate first object + assert objects[0].id == rows[0]["id"] + assert objects[0].tenant_id == rows[0]["tenant_id"] + assert objects[0].compliance_id == rows[0]["compliance_id"] + assert objects[0].framework == rows[0]["framework"] + assert objects[0].region == rows[0]["region"] + assert objects[0].passed_checks == 5 + assert objects[0].failed_checks == 0 + + # Validate second object + assert objects[1].id == rows[1]["id"] + assert objects[1].requirement_id == rows[1]["requirement_id"] + assert objects[1].requirement_status == rows[1]["requirement_status"] + assert objects[1].passed_checks == 2 + assert objects[1].failed_checks == 3 + + @patch("tasks.jobs.scan.ComplianceRequirementOverview.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + @patch( + "tasks.jobs.scan._copy_compliance_requirement_rows", + side_effect=Exception("copy failed"), + ) + def test_persist_compliance_requirement_rows_fallback_all_fields( + self, mock_copy, mock_rls_transaction, mock_bulk_create + ): + """Test ORM fallback correctly maps all fields from row dict to model.""" + tenant_id = str(uuid.uuid4()) + row_id = uuid.uuid4() + scan_id = uuid.uuid4() + inserted_at = datetime.now(timezone.utc) + + row = { + "id": row_id, + "tenant_id": tenant_id, + "inserted_at": inserted_at, + "compliance_id": "aws_foundational_security_aws", + "framework": "AWS-Foundational-Security-Best-Practices", + "version": "2.0", + "description": "Ensure MFA is enabled", + "region": "eu-west-1", + "requirement_id": "iam.1", + "requirement_status": "FAIL", + "passed_checks": 10, + "failed_checks": 5, + "total_checks": 15, + "scan_id": scan_id, + } + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + _persist_compliance_requirement_rows(tenant_id, [row]) + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + assert len(objects) == 1 + + obj = objects[0] + # Validate ALL fields are correctly mapped + assert obj.id == row_id + assert obj.tenant_id == tenant_id + assert obj.inserted_at == inserted_at + assert obj.compliance_id == "aws_foundational_security_aws" + assert obj.framework == "AWS-Foundational-Security-Best-Practices" + assert obj.version == "2.0" + assert obj.description == "Ensure MFA is enabled" + assert obj.region == "eu-west-1" + assert obj.requirement_id == "iam.1" + assert obj.requirement_status == "FAIL" + assert obj.passed_checks == 10 + assert obj.failed_checks == 5 + assert obj.total_checks == 15 + assert obj.scan_id == scan_id diff --git a/api/src/backend/tasks/tests/test_tasks.py b/api/src/backend/tasks/tests/test_tasks.py index b4262027f6..b2d8b7bc2b 100644 --- a/api/src/backend/tasks/tests/test_tasks.py +++ b/api/src/backend/tasks/tests/test_tasks.py @@ -1,16 +1,24 @@ import uuid from unittest.mock import MagicMock, patch +import openai import pytest +from botocore.exceptions import ClientError from tasks.tasks import ( _perform_scan_complete_tasks, check_integrations_task, + check_lighthouse_provider_connection_task, generate_outputs_task, + refresh_lighthouse_provider_models_task, s3_integration_task, security_hub_integration_task, ) -from api.models import Integration +from api.models import ( + Integration, + LighthouseProviderConfiguration, + LighthouseProviderModels, +) # TODO Move this to outputs/reports jobs @@ -98,7 +106,11 @@ class TestGenerateOutputs: ), patch( "tasks.tasks._generate_output_directory", - return_value=("out-dir", "comp-dir"), + return_value=( + "/tmp/test/out-dir", + "/tmp/test/comp-dir", + "/tmp/test/threat-dir", + ), ), patch("tasks.tasks.Scan.all_objects.filter") as mock_scan_update, patch("tasks.tasks.rmtree"), @@ -126,7 +138,8 @@ class TestGenerateOutputs: patch("tasks.tasks.get_compliance_frameworks"), patch("tasks.tasks.Finding.all_objects.filter") as mock_findings, patch( - "tasks.tasks._generate_output_directory", return_value=("out", "comp") + "tasks.tasks._generate_output_directory", + return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"), ), patch("tasks.tasks.FindingOutput._transform_findings_stats"), patch("tasks.tasks.FindingOutput.transform_api_finding"), @@ -168,15 +181,35 @@ class TestGenerateOutputs: mock_finding_output = MagicMock() mock_finding_output.compliance = {"cis": ["requirement-1", "requirement-2"]} + html_writer_mock = MagicMock() + html_writer_mock._data = [] + html_writer_mock.close_file = False + html_writer_mock.transform = MagicMock() + html_writer_mock.batch_write_data_to_file = MagicMock() + + compliance_writer_mock = MagicMock() + compliance_writer_mock._data = [] + compliance_writer_mock.close_file = False + compliance_writer_mock.transform = MagicMock() + compliance_writer_mock.batch_write_data_to_file = MagicMock() + + # Create a mock class that returns our mock instance when called + mock_compliance_class = MagicMock(return_value=compliance_writer_mock) + + mock_provider = MagicMock() + mock_provider.provider = "aws" + mock_provider.uid = "test-provider-uid" + with ( patch("tasks.tasks.ScanSummary.objects.filter") as mock_filter, - patch("tasks.tasks.Provider.objects.get"), + patch("tasks.tasks.Provider.objects.get", return_value=mock_provider), patch("tasks.tasks.initialize_prowler_provider"), patch("tasks.tasks.Compliance.get_bulk", return_value={"cis": MagicMock()}), patch("tasks.tasks.get_compliance_frameworks", return_value=["cis"]), patch("tasks.tasks.Finding.all_objects.filter") as mock_findings, patch( - "tasks.tasks._generate_output_directory", return_value=("out", "comp") + "tasks.tasks._generate_output_directory", + return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"), ), patch( "tasks.tasks.FindingOutput._transform_findings_stats", @@ -190,6 +223,20 @@ class TestGenerateOutputs: patch("tasks.tasks._upload_to_s3", return_value="s3://bucket/f.zip"), patch("tasks.tasks.Scan.all_objects.filter"), patch("tasks.tasks.rmtree"), + patch( + "tasks.tasks.OUTPUT_FORMATS_MAPPING", + { + "html": { + "class": lambda *args, **kwargs: html_writer_mock, + "suffix": ".html", + "kwargs": {}, + } + }, + ), + patch( + "tasks.tasks.COMPLIANCE_CLASS_MAP", + {"aws": [(lambda x: True, mock_compliance_class)]}, + ), ): mock_filter.return_value.exists.return_value = True mock_findings.return_value.order_by.return_value.iterator.return_value = [ @@ -197,29 +244,12 @@ class TestGenerateOutputs: True, ] - html_writer_mock = MagicMock() - with ( - patch( - "tasks.tasks.OUTPUT_FORMATS_MAPPING", - { - "html": { - "class": lambda *args, **kwargs: html_writer_mock, - "suffix": ".html", - "kwargs": {}, - } - }, - ), - patch( - "tasks.tasks.COMPLIANCE_CLASS_MAP", - {"aws": [(lambda x: True, MagicMock())]}, - ), - ): - generate_outputs_task( - scan_id=self.scan_id, - provider_id=self.provider_id, - tenant_id=self.tenant_id, - ) - html_writer_mock.batch_write_data_to_file.assert_called_once() + generate_outputs_task( + scan_id=self.scan_id, + provider_id=self.provider_id, + tenant_id=self.tenant_id, + ) + html_writer_mock.batch_write_data_to_file.assert_called_once() def test_transform_called_only_on_second_batch(self): raw1 = MagicMock() @@ -256,7 +286,11 @@ class TestGenerateOutputs: ), patch( "tasks.tasks._generate_output_directory", - return_value=("outdir", "compdir"), + return_value=( + "/tmp/test/outdir", + "/tmp/test/compdir", + "/tmp/test/threatdir", + ), ), patch("tasks.tasks._compress_output_files", return_value="outdir.zip"), patch("tasks.tasks._upload_to_s3", return_value="s3://bucket/outdir.zip"), @@ -303,12 +337,14 @@ class TestGenerateOutputs: def __init__(self, *args, **kwargs): self.transform_calls = [] self._data = [] + self.close_file = False writer_instances.append(self) def transform(self, fos, comp_obj, name): self.transform_calls.append((fos, comp_obj, name)) def batch_write_data_to_file(self): + # Mock implementation - do nothing pass two_batches = [ @@ -329,7 +365,11 @@ class TestGenerateOutputs: patch("tasks.tasks.get_compliance_frameworks", return_value=["cis"]), patch( "tasks.tasks._generate_output_directory", - return_value=("outdir", "compdir"), + return_value=( + "/tmp/test/outdir", + "/tmp/test/compdir", + "/tmp/test/threatdir", + ), ), patch("tasks.tasks.FindingOutput._transform_findings_stats"), patch( @@ -368,15 +408,35 @@ class TestGenerateOutputs: mock_finding_output = MagicMock() mock_finding_output.compliance = {"cis": ["requirement-1", "requirement-2"]} + json_writer_mock = MagicMock() + json_writer_mock._data = [] + json_writer_mock.close_file = False + json_writer_mock.transform = MagicMock() + json_writer_mock.batch_write_data_to_file = MagicMock() + + compliance_writer_mock = MagicMock() + compliance_writer_mock._data = [] + compliance_writer_mock.close_file = False + compliance_writer_mock.transform = MagicMock() + compliance_writer_mock.batch_write_data_to_file = MagicMock() + + # Create a mock class that returns our mock instance when called + mock_compliance_class = MagicMock(return_value=compliance_writer_mock) + + mock_provider = MagicMock() + mock_provider.provider = "aws" + mock_provider.uid = "test-provider-uid" + with ( patch("tasks.tasks.ScanSummary.objects.filter") as mock_filter, - patch("tasks.tasks.Provider.objects.get"), + patch("tasks.tasks.Provider.objects.get", return_value=mock_provider), patch("tasks.tasks.initialize_prowler_provider"), patch("tasks.tasks.Compliance.get_bulk", return_value={"cis": MagicMock()}), patch("tasks.tasks.get_compliance_frameworks", return_value=["cis"]), patch("tasks.tasks.Finding.all_objects.filter") as mock_findings, patch( - "tasks.tasks._generate_output_directory", return_value=("out", "comp") + "tasks.tasks._generate_output_directory", + return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"), ), patch( "tasks.tasks.FindingOutput._transform_findings_stats", @@ -390,6 +450,20 @@ class TestGenerateOutputs: patch("tasks.tasks._upload_to_s3", return_value="s3://bucket/file.zip"), patch("tasks.tasks.Scan.all_objects.filter"), patch("tasks.tasks.rmtree", side_effect=Exception("Test deletion error")), + patch( + "tasks.tasks.OUTPUT_FORMATS_MAPPING", + { + "json": { + "class": lambda *args, **kwargs: json_writer_mock, + "suffix": ".json", + "kwargs": {}, + } + }, + ), + patch( + "tasks.tasks.COMPLIANCE_CLASS_MAP", + {"aws": [(lambda x: True, mock_compliance_class)]}, + ), ): mock_filter.return_value.exists.return_value = True mock_findings.return_value.order_by.return_value.iterator.return_value = [ @@ -397,29 +471,13 @@ class TestGenerateOutputs: True, ] - with ( - patch( - "tasks.tasks.OUTPUT_FORMATS_MAPPING", - { - "json": { - "class": lambda *args, **kwargs: MagicMock(), - "suffix": ".json", - "kwargs": {}, - } - }, - ), - patch( - "tasks.tasks.COMPLIANCE_CLASS_MAP", - {"aws": [(lambda x: True, MagicMock())]}, - ), - ): - with caplog.at_level("ERROR"): - generate_outputs_task( - scan_id=self.scan_id, - provider_id=self.provider_id, - tenant_id=self.tenant_id, - ) - assert "Error deleting output files" in caplog.text + with caplog.at_level("ERROR"): + generate_outputs_task( + scan_id=self.scan_id, + provider_id=self.provider_id, + tenant_id=self.tenant_id, + ) + assert "Error deleting output files" in caplog.text @patch("tasks.tasks.rls_transaction") @patch("tasks.tasks.Integration.objects.filter") @@ -435,7 +493,8 @@ class TestGenerateOutputs: patch("tasks.tasks.get_compliance_frameworks", return_value=[]), patch("tasks.tasks.Finding.all_objects.filter") as mock_findings, patch( - "tasks.tasks._generate_output_directory", return_value=("out", "comp") + "tasks.tasks._generate_output_directory", + return_value=("/tmp/test/out", "/tmp/test/comp", "/tmp/test/threat"), ), patch("tasks.tasks.FindingOutput._transform_findings_stats"), patch("tasks.tasks.FindingOutput.transform_api_finding"), @@ -476,8 +535,15 @@ class TestScanCompleteTasks: @patch("tasks.tasks.create_compliance_requirements_task.apply_async") @patch("tasks.tasks.perform_scan_summary_task.si") @patch("tasks.tasks.generate_outputs_task.si") + @patch("tasks.tasks.generate_threatscore_report_task.si") + @patch("tasks.tasks.check_integrations_task.si") def test_scan_complete_tasks( - self, mock_outputs_task, mock_scan_summary_task, mock_compliance_tasks + self, + mock_check_integrations_task, + mock_threatscore_task, + mock_outputs_task, + mock_scan_summary_task, + mock_compliance_tasks, ): _perform_scan_complete_tasks("tenant-id", "scan-id", "provider-id") mock_compliance_tasks.assert_called_once_with( @@ -492,6 +558,16 @@ class TestScanCompleteTasks: provider_id="provider-id", tenant_id="tenant-id", ) + mock_threatscore_task.assert_called_once_with( + tenant_id="tenant-id", + scan_id="scan-id", + provider_id="provider-id", + ) + mock_check_integrations_task.assert_called_once_with( + tenant_id="tenant-id", + provider_id="provider-id", + scan_id="scan-id", + ) @pytest.mark.django_db @@ -662,7 +738,7 @@ class TestCheckIntegrationsTask: mock_initialize_provider.return_value = MagicMock() mock_compliance_bulk.return_value = {} mock_get_frameworks.return_value = [] - mock_generate_dir.return_value = ("out-dir", "comp-dir") + mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir") mock_transform_stats.return_value = {"stats": "data"} # Mock findings @@ -787,7 +863,7 @@ class TestCheckIntegrationsTask: mock_initialize_provider.return_value = MagicMock() mock_compliance_bulk.return_value = {} mock_get_frameworks.return_value = [] - mock_generate_dir.return_value = ("out-dir", "comp-dir") + mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir") mock_transform_stats.return_value = {"stats": "data"} # Mock findings @@ -903,7 +979,7 @@ class TestCheckIntegrationsTask: mock_initialize_provider.return_value = MagicMock() mock_compliance_bulk.return_value = {} mock_get_frameworks.return_value = [] - mock_generate_dir.return_value = ("out-dir", "comp-dir") + mock_generate_dir.return_value = ("out-dir", "comp-dir", "threat-dir") mock_transform_stats.return_value = {"stats": "data"} # Mock findings @@ -1029,3 +1105,363 @@ class TestCheckIntegrationsTask: assert result is False mock_upload.assert_called_once_with(self.tenant_id, self.provider_id, scan_id) + + +@pytest.mark.django_db +class TestCheckLighthouseProviderConnectionTask: + def setup_method(self): + self.tenant_id = str(uuid.uuid4()) + + @pytest.mark.parametrize( + "provider_type,credentials,base_url,expected_result", + [ + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + {"api_key": "sk-test123"}, + None, + {"connected": True, "error": None}, + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + {"api_key": "sk-test123"}, + "https://openrouter.ai/api/v1", + {"connected": True, "error": None}, + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + { + "access_key_id": "AKIA123", + "secret_access_key": "secret", + "region": "us-east-1", + }, + None, + {"connected": True, "error": None}, + ), + ], + ) + def test_check_connection_success_all_providers( + self, tenants_fixture, provider_type, credentials, base_url, expected_result + ): + """Test successful connection check for all provider types.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=provider_type, + base_url=base_url, + is_active=False, + ) + provider_cfg.credentials_decoded = credentials + provider_cfg.save() + + # Mock the appropriate API calls + with ( + patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai, + patch("tasks.jobs.lighthouse_providers.boto3.client") as mock_boto3, + ): + mock_client = MagicMock() + mock_client.models.list.return_value = MagicMock() + mock_client.list_foundation_models.return_value = {} + mock_openai.return_value = mock_client + mock_boto3.return_value = mock_client + + # Execute + result = check_lighthouse_provider_connection_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result == expected_result + provider_cfg.refresh_from_db() + assert provider_cfg.is_active is True + + @pytest.mark.parametrize( + "provider_type,credentials,base_url,exception_to_raise", + [ + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + {"api_key": "sk-invalid"}, + None, + openai.AuthenticationError( + "Invalid API key", response=MagicMock(), body=None + ), + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + {"api_key": "sk-invalid"}, + "https://openrouter.ai/api/v1", + openai.APIConnectionError(request=MagicMock()), + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + { + "access_key_id": "AKIA123", + "secret_access_key": "secret", + "region": "us-east-1", + }, + None, + ClientError( + {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, + "list_foundation_models", + ), + ), + ], + ) + def test_check_connection_api_failure( + self, + tenants_fixture, + provider_type, + credentials, + base_url, + exception_to_raise, + ): + """Test connection check when API calls fail.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=provider_type, + base_url=base_url, + is_active=True, + ) + provider_cfg.credentials_decoded = credentials + provider_cfg.save() + + # Mock the API to raise exception + with ( + patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai, + patch("tasks.jobs.lighthouse_providers.boto3.client") as mock_boto3, + ): + mock_client = MagicMock() + if ( + provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + mock_client.list_foundation_models.side_effect = exception_to_raise + mock_boto3.return_value = mock_client + else: + mock_client.models.list.side_effect = exception_to_raise + mock_openai.return_value = mock_client + + # Execute + result = check_lighthouse_provider_connection_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result["connected"] is False + assert result["error"] is not None + provider_cfg.refresh_from_db() + assert provider_cfg.is_active is False + + def test_check_connection_updates_active_status(self, tenants_fixture): + """Test that connection check toggles is_active from True to False on failure.""" + # Create provider with is_active=True + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + base_url=None, + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "sk-test123"} + provider_cfg.save() + + # Mock API to fail + with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai: + mock_client = MagicMock() + mock_client.models.list.side_effect = openai.AuthenticationError( + "Invalid", response=MagicMock(), body=None + ) + mock_openai.return_value = mock_client + + # Execute + result = check_lighthouse_provider_connection_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert status changed + assert result["connected"] is False + provider_cfg.refresh_from_db() + assert provider_cfg.is_active is False + + def test_check_connection_provider_does_not_exist(self, tenants_fixture): + """Test that checking non-existent provider raises DoesNotExist.""" + non_existent_id = str(uuid.uuid4()) + + with pytest.raises(LighthouseProviderConfiguration.DoesNotExist): + check_lighthouse_provider_connection_task( + provider_config_id=non_existent_id, + tenant_id=str(tenants_fixture[0].id), + ) + + +@pytest.mark.django_db +class TestRefreshLighthouseProviderModelsTask: + def setup_method(self): + self.tenant_id = str(uuid.uuid4()) + + @pytest.mark.parametrize( + "provider_type,credentials,base_url,mock_models,expected_count", + [ + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + {"api_key": "sk-test123"}, + None, + {"gpt-5": "gpt-5", "gpt-4o": "gpt-4o"}, + 2, + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + {"api_key": "sk-test123"}, + "https://openrouter.ai/api/v1", + {"model-1": "Model One", "model-2": "Model Two"}, + 2, + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + { + "access_key_id": "AKIA123", + "secret_access_key": "secret", + "region": "us-east-1", + }, + None, + {"openai.gpt-oss-120b-1:0": "gpt-oss-120b"}, + 1, + ), + ], + ) + def test_refresh_models_create_new( + self, + tenants_fixture, + provider_type, + credentials, + base_url, + mock_models, + expected_count, + ): + """Test creating new models for all provider types.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=provider_type, + base_url=base_url, + is_active=True, + ) + provider_cfg.credentials_decoded = credentials + provider_cfg.save() + + # Mock the fetch functions + with ( + patch( + "tasks.jobs.lighthouse_providers._fetch_openai_models", + return_value=mock_models, + ), + patch( + "tasks.jobs.lighthouse_providers._fetch_openai_compatible_models", + return_value=mock_models, + ), + patch( + "tasks.jobs.lighthouse_providers._fetch_bedrock_models", + return_value=mock_models, + ), + ): + # Execute + result = refresh_lighthouse_provider_models_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result["created"] == expected_count + assert result["updated"] == 0 + assert result["deleted"] == 0 + assert ( + LighthouseProviderModels.objects.filter( + provider_configuration=provider_cfg + ).count() + == expected_count + ) + + def test_refresh_models_mixed_operations(self, tenants_fixture): + """Test mixed create, update, and delete operations.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + base_url=None, + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "sk-test123"} + provider_cfg.save() + + # Create 2 existing models (A, B) + LighthouseProviderModels.objects.create( + tenant_id=tenants_fixture[0].id, + provider_configuration=provider_cfg, + model_id="model-a", + model_name="Model A", + ) + LighthouseProviderModels.objects.create( + tenant_id=tenants_fixture[0].id, + provider_configuration=provider_cfg, + model_id="model-b", + model_name="Model B", + ) + + # Mock API to return models B (existing), C (new) - A will be deleted + mock_models = {"model-b": "Model B", "model-c": "Model C"} + with patch( + "tasks.jobs.lighthouse_providers._fetch_openai_models", + return_value=mock_models, + ): + # Execute + result = refresh_lighthouse_provider_models_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result["created"] == 1 # model-c created + assert result["updated"] == 1 # model-b updated + assert result["deleted"] == 1 # model-a deleted + + # Verify only B and C exist + remaining_models = LighthouseProviderModels.objects.filter( + provider_configuration=provider_cfg + ) + assert remaining_models.count() == 2 + assert set(remaining_models.values_list("model_id", flat=True)) == { + "model-b", + "model-c", + } + + def test_refresh_models_api_exception(self, tenants_fixture): + """Test refresh when API raises an exception.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + base_url=None, + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "sk-test123"} + provider_cfg.save() + + # Mock fetch to raise exception + with patch( + "tasks.jobs.lighthouse_providers._fetch_openai_models", + side_effect=openai.APIError("API Error", request=MagicMock(), body=None), + ): + # Execute + result = refresh_lighthouse_provider_models_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result["created"] == 0 + assert result["updated"] == 0 + assert result["deleted"] == 0 + assert "error" in result + assert result["error"] is not None diff --git a/dashboard/__main__.py b/dashboard/__main__.py index dac536ccdf..304e4afdf2 100644 --- a/dashboard/__main__.py +++ b/dashboard/__main__.py @@ -35,7 +35,8 @@ dashboard = dash.Dash( # Logo prowler_logo = html.Img( - src="https://prowler.com/wp-content/uploads/logo-dashboard.png", alt="Prowler Logo" + src="https://cdn.prod.website-files.com/68c4ec3f9fb7b154fbcb6e36/68ffb46d40ed7faa37a592a5_prowler-logo.png", + alt="Prowler Logo", ) menu_icons = { diff --git a/dashboard/compliance/c5_aws.py b/dashboard/compliance/c5_aws.py new file mode 100644 index 0000000000..8baac9a9a5 --- /dev/null +++ b/dashboard/compliance/c5_aws.py @@ -0,0 +1,43 @@ +import warnings + +from dashboard.common_methods import get_section_containers_3_levels + +warnings.filterwarnings("ignore") + + +def get_table(data): + data["REQUIREMENTS_DESCRIPTION"] = ( + data["REQUIREMENTS_ID"] + " - " + data["REQUIREMENTS_DESCRIPTION"] + ) + + data["REQUIREMENTS_DESCRIPTION"] = data["REQUIREMENTS_DESCRIPTION"].apply( + lambda x: x[:150] + "..." if len(str(x)) > 150 else x + ) + + data["REQUIREMENTS_ATTRIBUTES_SECTION"] = data[ + "REQUIREMENTS_ATTRIBUTES_SECTION" + ].apply(lambda x: x[:80] + "..." if len(str(x)) > 80 else x) + + data["REQUIREMENTS_ATTRIBUTES_SUBSECTION"] = data[ + "REQUIREMENTS_ATTRIBUTES_SUBSECTION" + ].apply(lambda x: x[:150] + "..." if len(str(x)) > 150 else x) + + aux = data[ + [ + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ] + + return get_section_containers_3_levels( + aux, + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "REQUIREMENTS_DESCRIPTION", + ) diff --git a/dashboard/compliance/c5_azure.py b/dashboard/compliance/c5_azure.py new file mode 100644 index 0000000000..8baac9a9a5 --- /dev/null +++ b/dashboard/compliance/c5_azure.py @@ -0,0 +1,43 @@ +import warnings + +from dashboard.common_methods import get_section_containers_3_levels + +warnings.filterwarnings("ignore") + + +def get_table(data): + data["REQUIREMENTS_DESCRIPTION"] = ( + data["REQUIREMENTS_ID"] + " - " + data["REQUIREMENTS_DESCRIPTION"] + ) + + data["REQUIREMENTS_DESCRIPTION"] = data["REQUIREMENTS_DESCRIPTION"].apply( + lambda x: x[:150] + "..." if len(str(x)) > 150 else x + ) + + data["REQUIREMENTS_ATTRIBUTES_SECTION"] = data[ + "REQUIREMENTS_ATTRIBUTES_SECTION" + ].apply(lambda x: x[:80] + "..." if len(str(x)) > 80 else x) + + data["REQUIREMENTS_ATTRIBUTES_SUBSECTION"] = data[ + "REQUIREMENTS_ATTRIBUTES_SUBSECTION" + ].apply(lambda x: x[:150] + "..." if len(str(x)) > 150 else x) + + aux = data[ + [ + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ] + + return get_section_containers_3_levels( + aux, + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "REQUIREMENTS_DESCRIPTION", + ) diff --git a/dashboard/compliance/ccc_aws.py b/dashboard/compliance/ccc_aws.py new file mode 100644 index 0000000000..5344dada84 --- /dev/null +++ b/dashboard/compliance/ccc_aws.py @@ -0,0 +1,36 @@ +import warnings + +from dashboard.common_methods import get_section_containers_3_levels + +warnings.filterwarnings("ignore") + + +def get_table(data): + + data["REQUIREMENTS_ID"] = ( + data["REQUIREMENTS_ID"] + " - " + data["REQUIREMENTS_DESCRIPTION"] + ) + + data["REQUIREMENTS_ID"] = data["REQUIREMENTS_ID"].apply( + lambda x: x[:150] + "..." if len(str(x)) > 150 else x + ) + + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ] + + return get_section_containers_3_levels( + aux, + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "REQUIREMENTS_ID", + ) diff --git a/dashboard/compliance/ccc_azure.py b/dashboard/compliance/ccc_azure.py new file mode 100644 index 0000000000..5344dada84 --- /dev/null +++ b/dashboard/compliance/ccc_azure.py @@ -0,0 +1,36 @@ +import warnings + +from dashboard.common_methods import get_section_containers_3_levels + +warnings.filterwarnings("ignore") + + +def get_table(data): + + data["REQUIREMENTS_ID"] = ( + data["REQUIREMENTS_ID"] + " - " + data["REQUIREMENTS_DESCRIPTION"] + ) + + data["REQUIREMENTS_ID"] = data["REQUIREMENTS_ID"].apply( + lambda x: x[:150] + "..." if len(str(x)) > 150 else x + ) + + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ] + + return get_section_containers_3_levels( + aux, + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "REQUIREMENTS_ID", + ) diff --git a/dashboard/compliance/ccc_gcp.py b/dashboard/compliance/ccc_gcp.py new file mode 100644 index 0000000000..5344dada84 --- /dev/null +++ b/dashboard/compliance/ccc_gcp.py @@ -0,0 +1,36 @@ +import warnings + +from dashboard.common_methods import get_section_containers_3_levels + +warnings.filterwarnings("ignore") + + +def get_table(data): + + data["REQUIREMENTS_ID"] = ( + data["REQUIREMENTS_ID"] + " - " + data["REQUIREMENTS_DESCRIPTION"] + ) + + data["REQUIREMENTS_ID"] = data["REQUIREMENTS_ID"].apply( + lambda x: x[:150] + "..." if len(str(x)) > 150 else x + ) + + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ] + + return get_section_containers_3_levels( + aux, + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "REQUIREMENTS_ID", + ) diff --git a/dashboard/compliance/cis_3_0_oci.py b/dashboard/compliance/cis_3_0_oci.py new file mode 100644 index 0000000000..286a6cd12f --- /dev/null +++ b/dashboard/compliance/cis_3_0_oci.py @@ -0,0 +1,41 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + """ + Generate CIS OCI Foundations Benchmark v3.0 compliance table. + + Args: + data: DataFrame containing compliance check results with columns: + - REQUIREMENTS_ID: CIS requirement ID (e.g., "1.1", "2.1") + - REQUIREMENTS_DESCRIPTION: Description of the requirement + - REQUIREMENTS_ATTRIBUTES_SECTION: CIS section name + - CHECKID: Prowler check identifier + - STATUS: Check status (PASS/FAIL) + - REGION: OCI region + - TENANCYID: OCI tenancy OCID + - RESOURCEID: Resource OCID or identifier + + Returns: + Section containers organized by CIS sections for dashboard display + """ + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "TENANCYID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/fedramp_20x_ksi_low_aws.py b/dashboard/compliance/fedramp_20x_ksi_low_aws.py new file mode 100644 index 0000000000..5ca220301f --- /dev/null +++ b/dashboard/compliance/fedramp_20x_ksi_low_aws.py @@ -0,0 +1,46 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + # Shorten the long FedRAMP KSI descriptions for better display + ksi_short_names = { + "A secure cloud service offering will protect user data, control access, and apply zero trust principles": "Identity and Access Management", + "A secure cloud service offering will use cloud native architecture and design principles to enforce and enhance the Confidentiality, Integrity and Availability of the system": "Cloud Native Architecture", + "A secure cloud service provider will ensure that all system changes are properly documented and configuration baselines are updated accordingly": "Change Management", + "A secure cloud service provider will continuously educate their employees on cybersecurity measures, testing them regularly": "Cybersecurity Education", + "A secure cloud service offering will document, report, and analyze security incidents to ensure regulatory compliance and continuous security improvement": "Incident Reporting", + "A secure cloud service offering will monitor, log, and audit all important events, activity, and changes": "Monitoring, Logging, and Auditing", + "A secure cloud service offering will have intentional, organized, universal guidance for how every information resource, including personnel, is secured": "Policy and Inventory", + "A secure cloud service offering will define, maintain, and test incident response plan(s) and recovery capabilities to ensure minimal service disruption and data loss": "Recovery Planning", + "A secure cloud service offering will follow FedRAMP encryption policies, continuously verify information resource integrity, and restrict access to third-party information resources": "Service Configuration", + "A secure cloud service offering will understand, monitor, and manage supply chain risks from third-party information resources": "Third-Party Information Resources", + } + + # Replace long descriptions with short names - use contains for partial matching + if not aux.empty: + for long_desc, short_name in ksi_short_names.items(): + mask = aux["REQUIREMENTS_DESCRIPTION"].str.contains( + long_desc, na=False, regex=False + ) + aux.loc[mask, "REQUIREMENTS_DESCRIPTION"] = short_name + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/fedramp_20x_ksi_low_azure.py b/dashboard/compliance/fedramp_20x_ksi_low_azure.py new file mode 100644 index 0000000000..5ca220301f --- /dev/null +++ b/dashboard/compliance/fedramp_20x_ksi_low_azure.py @@ -0,0 +1,46 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + # Shorten the long FedRAMP KSI descriptions for better display + ksi_short_names = { + "A secure cloud service offering will protect user data, control access, and apply zero trust principles": "Identity and Access Management", + "A secure cloud service offering will use cloud native architecture and design principles to enforce and enhance the Confidentiality, Integrity and Availability of the system": "Cloud Native Architecture", + "A secure cloud service provider will ensure that all system changes are properly documented and configuration baselines are updated accordingly": "Change Management", + "A secure cloud service provider will continuously educate their employees on cybersecurity measures, testing them regularly": "Cybersecurity Education", + "A secure cloud service offering will document, report, and analyze security incidents to ensure regulatory compliance and continuous security improvement": "Incident Reporting", + "A secure cloud service offering will monitor, log, and audit all important events, activity, and changes": "Monitoring, Logging, and Auditing", + "A secure cloud service offering will have intentional, organized, universal guidance for how every information resource, including personnel, is secured": "Policy and Inventory", + "A secure cloud service offering will define, maintain, and test incident response plan(s) and recovery capabilities to ensure minimal service disruption and data loss": "Recovery Planning", + "A secure cloud service offering will follow FedRAMP encryption policies, continuously verify information resource integrity, and restrict access to third-party information resources": "Service Configuration", + "A secure cloud service offering will understand, monitor, and manage supply chain risks from third-party information resources": "Third-Party Information Resources", + } + + # Replace long descriptions with short names - use contains for partial matching + if not aux.empty: + for long_desc, short_name in ksi_short_names.items(): + mask = aux["REQUIREMENTS_DESCRIPTION"].str.contains( + long_desc, na=False, regex=False + ) + aux.loc[mask, "REQUIREMENTS_DESCRIPTION"] = short_name + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/fedramp_20x_ksi_low_gcp.py b/dashboard/compliance/fedramp_20x_ksi_low_gcp.py new file mode 100644 index 0000000000..5ca220301f --- /dev/null +++ b/dashboard/compliance/fedramp_20x_ksi_low_gcp.py @@ -0,0 +1,46 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + # Shorten the long FedRAMP KSI descriptions for better display + ksi_short_names = { + "A secure cloud service offering will protect user data, control access, and apply zero trust principles": "Identity and Access Management", + "A secure cloud service offering will use cloud native architecture and design principles to enforce and enhance the Confidentiality, Integrity and Availability of the system": "Cloud Native Architecture", + "A secure cloud service provider will ensure that all system changes are properly documented and configuration baselines are updated accordingly": "Change Management", + "A secure cloud service provider will continuously educate their employees on cybersecurity measures, testing them regularly": "Cybersecurity Education", + "A secure cloud service offering will document, report, and analyze security incidents to ensure regulatory compliance and continuous security improvement": "Incident Reporting", + "A secure cloud service offering will monitor, log, and audit all important events, activity, and changes": "Monitoring, Logging, and Auditing", + "A secure cloud service offering will have intentional, organized, universal guidance for how every information resource, including personnel, is secured": "Policy and Inventory", + "A secure cloud service offering will define, maintain, and test incident response plan(s) and recovery capabilities to ensure minimal service disruption and data loss": "Recovery Planning", + "A secure cloud service offering will follow FedRAMP encryption policies, continuously verify information resource integrity, and restrict access to third-party information resources": "Service Configuration", + "A secure cloud service offering will understand, monitor, and manage supply chain risks from third-party information resources": "Third-Party Information Resources", + } + + # Replace long descriptions with short names - use contains for partial matching + if not aux.empty: + for long_desc, short_name in ksi_short_names.items(): + mask = aux["REQUIREMENTS_DESCRIPTION"].str.contains( + long_desc, na=False, regex=False + ) + aux.loc[mask, "REQUIREMENTS_DESCRIPTION"] = short_name + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/hipaa_gcp.py b/dashboard/compliance/hipaa_gcp.py new file mode 100644 index 0000000000..b0a8eb6582 --- /dev/null +++ b/dashboard/compliance/hipaa_gcp.py @@ -0,0 +1,25 @@ +import warnings + +from dashboard.common_methods import get_section_containers_format3 + +warnings.filterwarnings("ignore") + + +def get_table(data): + + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_DESCRIPTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_format3( + aux, "REQUIREMENTS_ATTRIBUTES_SECTION", "REQUIREMENTS_ID" + ) diff --git a/dashboard/compliance/nist_csf_2_0_aws.py b/dashboard/compliance/nist_csf_2_0_aws.py new file mode 100644 index 0000000000..07b956b104 --- /dev/null +++ b/dashboard/compliance/nist_csf_2_0_aws.py @@ -0,0 +1,24 @@ +import warnings + +from dashboard.common_methods import get_section_containers_format3 + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_DESCRIPTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_format3( + aux, "REQUIREMENTS_ATTRIBUTES_SECTION", "REQUIREMENTS_ID" + ) diff --git a/docs/.mintlifyignore b/docs/.mintlifyignore new file mode 100644 index 0000000000..2681ed7d5c --- /dev/null +++ b/docs/.mintlifyignore @@ -0,0 +1,4 @@ +.idea/ +.git/ +.claude/ +AGENTS.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..79917aeed1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,45 @@ +# Prowler Documentation + +This repository contains the Prowler Open Source documentation powered by [Mintlify](https://mintlify.com). + +## Documentation Structure + +- **Getting Started**: Overview, installation, and basic usage guides +- **User Guide**: Comprehensive guides for Prowler App, CLI, providers, and compliance +- **Developer Guide**: Technical documentation for developers contributing to Prowler + +## Local Development + +Install the [Mintlify CLI](https://www.npmjs.com/package/mint) to preview documentation changes locally: + +```bash +npm i -g mint +``` + +Run the following command at the root of your documentation (where `mint.json` is located): + +```bash +mint dev +``` + +View your local preview at `http://localhost:3000`. + +## Publishing Changes + +Changes pushed to the main branch are automatically deployed to production through Mintlify's GitHub integration. + +## Documentation Guidelines + +When contributing to the documentation, please follow the Prowler documentation style guide located in the `.claude` directory. + +## Troubleshooting + +- If your dev environment isn't running: Run `mint update` to ensure you have the most recent version of the CLI. +- If a page loads as a 404: Make sure you are running in a folder with a valid `mint.json` file and that the page path is correctly listed in the navigation. + +## Resources + +- [Prowler GitHub Repository](https://github.com/prowler-cloud/prowler) +- [Prowler Documentation](https://docs.prowler.com/) +- [Mintlify Documentation](https://mintlify.com/docs) +- [Mintlify Community](https://mintlify.com/community) diff --git a/docs/basic-usage/prowler-app.md b/docs/basic-usage/prowler-app.md deleted file mode 100644 index 0bea33007b..0000000000 --- a/docs/basic-usage/prowler-app.md +++ /dev/null @@ -1,61 +0,0 @@ -## Access Prowler App - -After [installation](../installation/prowler-app.md), navigate to [http://localhost:3000](http://localhost:3000) and sign up with email and password. - -Sign Up Button -Sign Up - -???+ note "User creation and default tenant behavior" - - When creating a new user, the behavior depends on whether an invitation is provided: - - - **Without an invitation**: - - - A new tenant is automatically created. - - The new user is assigned to this tenant. - - A set of **RBAC admin permissions** is generated and assigned to the user for the newly-created tenant. - - - **With an invitation**: The user is added to the specified tenant with the permissions defined in the invitation. - - This mechanism ensures that the first user in a newly created tenant has administrative permissions within that tenant. - -## Log In - -Access Prowler App by logging in with **email and password**. - -Log In - -## Add Cloud Provider - -Configure a cloud provider for scanning: - -1. Navigate to `Settings > Cloud Providers` and click `Add Account`. -2. Select the cloud provider. -3. Enter the provider's identifier (Optional: Add an alias): - - **AWS**: Account ID - - **GCP**: Project ID - - **Azure**: Subscription ID - - **Kubernetes**: Cluster ID - - **M365**: Domain ID -4. Follow the guided instructions to add and authenticate your credentials. - -## Start a Scan - -Once credentials are successfully added and validated, Prowler initiates a scan of your cloud environment. - -Click `Go to Scans` to monitor progress. - -## View Results - -Review findings during scan execution in the following sections: - -- **Overview** – Provides a high-level summary of your scans. - Overview - -- **Compliance** – Displays compliance insights based on security frameworks. - Compliance - -> For detailed usage instructions, refer to the [Prowler App Guide](../tutorials/prowler-app.md). - -???+ note - Prowler will automatically scan all configured providers every **24 hours**, ensuring your cloud environment stays continuously monitored. diff --git a/docs/contact.md b/docs/contact.mdx similarity index 93% rename from docs/contact.md rename to docs/contact.mdx index 94c8adf0bb..3f898b9049 100644 --- a/docs/contact.md +++ b/docs/contact.mdx @@ -1,4 +1,6 @@ -# Contact Us +--- +title: 'Contact Us' +--- For technical support or any type of inquiries, you are very welcome to: diff --git a/docs/developer-guide/aws-details.md b/docs/developer-guide/aws-details.mdx similarity index 86% rename from docs/developer-guide/aws-details.md rename to docs/developer-guide/aws-details.mdx index 9fa182d689..70f94f4006 100644 --- a/docs/developer-guide/aws-details.md +++ b/docs/developer-guide/aws-details.mdx @@ -1,12 +1,14 @@ -# AWS Provider +--- +title: 'AWS Provider' +--- In this page you can find all the details about [Amazon Web Services (AWS)](https://aws.amazon.com/) provider implementation in Prowler. -By default, Prowler will audit just one account and organization settings per scan. To configure it, follow the [AWS getting started guide](../tutorials/aws/getting-started-aws.md). +By default, Prowler will audit just one account and organization settings per scan. To configure it, follow the [AWS getting started guide](/user-guide/providers/aws/getting-started-aws). ## AWS Provider Classes Architecture -The AWS provider implementation follows the general [Provider structure](./provider.md). This section focuses on the AWS-specific implementation, highlighting how the generic provider concepts are realized for AWS in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](./provider.md). In next subsection you can find a list of the main classes of the AWS provider. +The AWS provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the AWS-specific implementation, highlighting how the generic provider concepts are realized for AWS in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). In next subsection you can find a list of the main classes of the AWS provider. ### `AwsProvider` (Main Class) @@ -33,7 +35,7 @@ The AWS provider implementation follows the general [Provider structure](./provi ### `AWSService` (Service Base Class) - **Location:** [`prowler/providers/aws/lib/service/service.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/aws/lib/service/service.py) -- **Purpose:** Abstract base class that all AWS service-specific classes inherit from. This implements the generic service pattern (described in [service page](./services.md#service-base-class)) specifically for AWS. +- **Purpose:** Abstract base class that all AWS service-specific classes inherit from. This implements the generic service pattern (described in [service page](/developer-guide/services#service-base-class)) specifically for AWS. - **Key AWS Responsibilities:** - Receives an `AwsProvider` instance to access session, identity, and configuration. - Manages clients for all services by regions. @@ -52,12 +54,12 @@ The AWS provider implementation follows the general [Provider structure](./provi ## Specific Patterns in AWS Services -The generic service pattern is described in [service page](./services.md#service-structure-and-initialisation). You can find all the right now implemented services in the following locations: +The generic service pattern is described in [service page](/developer-guide/services#service-structure-and-initialisation). You can find all the right now implemented services in the following locations: - Directly in the code, in location [`prowler/providers/aws/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/aws/services) - In the [Prowler Hub](https://hub.prowler.com/). For a more human-readable view. -The best reference to understand how to implement a new service is following the [service implementation documentation](./services.md#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used accross all AWS services. +The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used accross all AWS services. ### AWS Service Common Patterns @@ -74,12 +76,12 @@ The best reference to understand how to implement a new service is following the ## Specific Patterns in AWS Checks -The AWS checks pattern is described in [checks page](./checks.md). You can find all the right now implemented checks: +The AWS checks pattern is described in [checks page](/developer-guide/checks). You can find all the right now implemented checks: - Directly in the code, within each service folder, each check has its own folder named after the name of the check. (e.g. [`prowler/providers/aws/services/s3/s3_bucket_acl_prohibited/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/aws/services/s3/s3_bucket_acl_prohibited)) - In the [Prowler Hub](https://hub.prowler.com/). For a more human-readable view. -The best reference to understand how to implement a new check is following the [check creation documentation](./checks.md#creating-a-check) and taking other similar checks as reference. +The best reference to understand how to implement a new check is following the [check creation documentation](/developer-guide/checks#creating-a-check) and taking other similar checks as reference. ### Check Report Class diff --git a/docs/developer-guide/azure-details.md b/docs/developer-guide/azure-details.mdx similarity index 84% rename from docs/developer-guide/azure-details.md rename to docs/developer-guide/azure-details.mdx index d1eb6b73be..d298a13bf4 100644 --- a/docs/developer-guide/azure-details.md +++ b/docs/developer-guide/azure-details.mdx @@ -1,12 +1,14 @@ -# Azure Provider +--- +title: 'Azure Provider' +--- In this page you can find all the details about [Microsoft Azure](https://azure.microsoft.com/) provider implementation in Prowler. -By default, Prowler will audit all the subscriptions that it is able to list in the Microsoft Entra tenant, and tenant Entra ID service. To configure it, follow the [Azure getting started guide](../tutorials/azure/getting-started-azure.md). +By default, Prowler will audit all the subscriptions that it is able to list in the Microsoft Entra tenant, and tenant Entra ID service. To configure it, follow the [Azure getting started guide](/user-guide/providers/azure/getting-started-azure). ## Azure Provider Classes Architecture -The Azure provider implementation follows the general [Provider structure](./provider.md). This section focuses on the Azure-specific implementation, highlighting how the generic provider concepts are realized for Azure in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](./provider.md). In next subsection you can find a list of the main classes of the Azure provider. +The Azure provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the Azure-specific implementation, highlighting how the generic provider concepts are realized for Azure in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). In next subsection you can find a list of the main classes of the Azure provider. ### `AzureProvider` (Main Class) @@ -32,7 +34,7 @@ The Azure provider implementation follows the general [Provider structure](./pro ### `AzureService` (Service Base Class) - **Location:** [`prowler/providers/azure/lib/service/service.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/azure/lib/service/service.py) -- **Purpose:** Abstract base class that all Azure service-specific classes inherit from. This implements the generic service pattern (described in [service page](./services.md#service-base-class)) specifically for Azure. +- **Purpose:** Abstract base class that all Azure service-specific classes inherit from. This implements the generic service pattern (described in [service page](/developer-guide/services#service-base-class)) specifically for Azure. - **Key Azure Responsibilities:** - Receives an `AzureProvider` instance to access session, identity, and configuration. - Manages clients for all services by subscription. @@ -50,12 +52,12 @@ The Azure provider implementation follows the general [Provider structure](./pro ## Specific Patterns in Azure Services -The generic service pattern is described in [service page](./services.md#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: +The generic service pattern is described in [service page](/developer-guide/services#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: - Directly in the code, in location [`prowler/providers/azure/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/azure/services) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new service is following the [service implementation documentation](./services.md#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used accross all Azure services. +The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used accross all Azure services. ### Azure Service Common Patterns @@ -68,12 +70,12 @@ The best reference to understand how to implement a new service is following the ## Specific Patterns in Azure Checks -The Azure checks pattern is described in [checks page](./checks.md). You can find all the currently implemented checks: +The Azure checks pattern is described in [checks page](/developer-guide/checks). You can find all the currently implemented checks: - Directly in the code, within each service folder, each check has its own folder named after the name of the check. (e.g. [`prowler/providers/azure/services/storage/storage_blob_public_access_level_is_disabled/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/azure/services/storage/storage_blob_public_access_level_is_disabled)) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new check is the [Azure check implementation documentation](./checks.md#creating-a-check) and taking other similar checks as reference. +The best reference to understand how to implement a new check is the [Azure check implementation documentation](/developer-guide/checks#creating-a-check) and taking other similar checks as reference. ### Check Report Class diff --git a/docs/developer-guide/check-metadata-guidelines.md b/docs/developer-guide/check-metadata-guidelines.mdx similarity index 99% rename from docs/developer-guide/check-metadata-guidelines.md rename to docs/developer-guide/check-metadata-guidelines.mdx index 84994c2855..2b6524bcc6 100644 --- a/docs/developer-guide/check-metadata-guidelines.md +++ b/docs/developer-guide/check-metadata-guidelines.mdx @@ -1,8 +1,10 @@ -# Check Metadata Guidelines +--- +title: 'Check Metadata Guidelines' +--- ## Introduction -This guide provides comprehensive guidelines for creating check metadata in Prowler. For basic information on check metadata structure, refer to the [check metadata](./checks.md#metadata-structure-for-prowler-checks) section. +This guide provides comprehensive guidelines for creating check metadata in Prowler. For basic information on check metadata structure, refer to the [check metadata](/developer-guide/checks#metadata-structure-for-prowler-checks) section. ## Check Title Guidelines diff --git a/docs/developer-guide/checks.md b/docs/developer-guide/checks.mdx similarity index 91% rename from docs/developer-guide/checks.md rename to docs/developer-guide/checks.mdx index 90e8cc7fd0..7a86686071 100644 --- a/docs/developer-guide/checks.md +++ b/docs/developer-guide/checks.mdx @@ -1,4 +1,6 @@ -# Prowler Checks +--- +title: 'Prowler Checks' +--- This guide explains how to create new checks in Prowler. @@ -12,7 +14,7 @@ The most common high level steps to create a new check are: 1. Prerequisites: - Verify the check does not already exist by searching [Prowler Hub](https://hub.prowler.com) or checking `prowler/providers//services///`. - - Ensure required provider and service exist. If not, follow the [Provider](./provider.md) and [Service](./services.md) documentation to create them. + - Ensure required provider and service exist. If not, follow the [Provider](/developer-guide/provider) and [Service](/developer-guide/services) documentation to create them. - Confirm the service has implemented all required methods and attributes for the check (in most cases, you will need to add or modify some methods in the service to get the data you need for the check). 2. Navigate to the service directory. The path should be as follows: `prowler/providers//services/`. 3. Create a check-specific folder. The path should follow this pattern: `prowler/providers//services//`. Adhere to the [Naming Format for Checks](#naming-format-for-checks). @@ -20,7 +22,7 @@ The most common high level steps to create a new check are: 5. Run the check locally to ensure it works as expected. For checking you can use the CLI in the next way: - To ensure the check has been detected by Prowler: `poetry run python prowler-cli.py --list-checks | grep `. - To run the check, to find possible issues: `poetry run python prowler-cli.py --log-level ERROR --verbose --check `. -6. Create comprehensive tests for the check that cover multiple scenarios including both PASS (compliant) and FAIL (non-compliant) cases. For detailed information about test structure and implementation guidelines, refer to the [Testing](./unit-testing.md) documentation. +6. Create comprehensive tests for the check that cover multiple scenarios including both PASS (compliant) and FAIL (non-compliant) cases. For detailed information about test structure and implementation guidelines, refer to the [Testing](/developer-guide/unit-testing) documentation. 7. If the check and its corresponding tests are working as expected, you can submit a PR to Prowler. ### Naming Format for Checks @@ -39,8 +41,8 @@ The name components are: Each check in Prowler follows a straightforward structure. Within the newly created folder, three files must be added to implement the check logic: - `__init__.py` (empty file) – Ensures Python treats the check folder as a package. -- `.py` (code file) – Contains the check logic, following the prescribed format. Please refer to the [prowler's check code structure](./checks.md#prowlers-check-code-structure) for more information. -- `.metadata.json` (metadata file) – Defines the check's metadata for contextual information. Please refer to the [check metadata](./checks.md#metadata-structure-for-prowler-checks) for more information. +- `.py` (code file) – Contains the check logic, following the prescribed format. Please refer to the [prowler's check code structure](/developer-guide/checks#prowlers-check-code-structure) for more information. +- `.metadata.json` (metadata file) – Defines the check's metadata for contextual information. Please refer to the [check metadata](/developer-guide/checks#metadata-structure-for-prowler-checks) for more information. ## Prowler's Check Code Structure @@ -50,9 +52,10 @@ Below the code for a generic check is presented. It is strongly recommended to c Report fields are the most dependent on the provider, consult the `CheckReport` class for more information on what can be included in the report [here](https://github.com/prowler-cloud/prowler/blob/master/prowler/lib/check/models.py). -???+ note - Legacy providers (AWS, Azure, GCP, Kubernetes) follow the `Check_Report_` naming convention. This is not recommended for current instances. Newer providers adopt the `CheckReport` naming convention. Learn more at [Prowler Code](https://github.com/prowler-cloud/prowler/tree/master/prowler/lib/check/models.py). + +Legacy providers (AWS, Azure, GCP, Kubernetes) follow the `Check_Report_` naming convention. This is not recommended for current instances. Newer providers adopt the `CheckReport` naming convention. Learn more at [Prowler Code](https://github.com/prowler-cloud/prowler/tree/master/prowler/lib/check/models.py). + ```python title="Generic Check Class" # Required Imports # Import the base Check class and the provider-specific CheckReport class @@ -122,11 +125,11 @@ Each check **must** populate the `report.status` and `report.status_extended` fi The severity of each check is defined in the metadata file using the `Severity` field. Severity values are always lowercase and must be one of the predefined categories below. -- `critical` – Issue that must be addressed immediately. -- `high` – Issue that should be addressed as soon as possible. -- `medium` – Issue that should be addressed within a reasonable timeframe. -- `low` – Issue that can be addressed in the future. -- `informational` – Not an issue but provides valuable information. +- `critical` – Highest potential impact with broad exposure that could affect core security boundaries or business operations. +- `high` – Substantial potential impact with significant exposure that could affect important security controls or resources. +- `medium` – Moderate potential impact with limited exposure that weakens defense layers but has contained scope. +- `low` – Minimal potential impact with negligible exposure that represents minor gaps in security posture. +- `informational` – Provides valuable information but does not affect the security posture. If the check involves multiple scenarios that may alter its severity, adjustments can be made dynamically within the check's logic using the severity `report.check_metadata.Severity` attribute: @@ -213,7 +216,7 @@ Each check **must** populate the report with an unique identifier for the audite ### Configurable Checks in Prowler -See [Configurable Checks](./configurable-checks.md) for detailed information on making checks configurable using the `audit_config` object and configuration file. +See [Configurable Checks](/developer-guide/configurable-checks) for detailed information on making checks configurable using the `audit_config` object and configuration file. ## Metadata Structure for Prowler Checks @@ -273,16 +276,17 @@ The `CheckTitle` field must be plain text, clearly and succinctly define **the b **Always write the `CheckTitle` to describe the *PASS* case**, the desired secure or compliant state of the resource(s). This helps ensure that findings are easy to interpret and that the title always reflects the best practice being met. -For detailed guidelines on writing effective check titles, including how to determine singular vs. plural scope and common mistakes to avoid, see [Check Title Guidelines](./check-metadata-guidelines.md#check-title-guidelines). +For detailed guidelines on writing effective check titles, including how to determine singular vs. plural scope and common mistakes to avoid, see [Check Title Guidelines](/developer-guide/check-metadata-guidelines#check-title-guidelines). #### CheckType -???+ warning - This field is only applicable to the AWS provider. + +This field is only applicable to the AWS provider. + It follows the [AWS Security Hub Types](https://docs.aws.amazon.com/securityhub/latest/userguide/asff-required-attributes.html#Types) format using the pattern `namespace/category/classifier`. -For the complete AWS Security Hub selection guidelines, see [Check Type Guidelines](./check-metadata-guidelines.md#check-type-guidelines-aws-only). +For the complete AWS Security Hub selection guidelines, see [Check Type Guidelines](/developer-guide/check-metadata-guidelines#check-type-guidelines-aws-only). #### ServiceName @@ -314,13 +318,13 @@ The type of resource being audited. This field helps categorize and organize fin A concise, natural language explanation that **clearly describes what the finding means**, focusing on clarity and context rather than technical implementation details. Use simple paragraphs with line breaks if needed, but avoid sections, code blocks, or complex formatting. This field is limited to maximum 400 characters. -For detailed writing guidelines and common mistakes to avoid, see [Description Guidelines](./check-metadata-guidelines.md#description-guidelines). +For detailed writing guidelines and common mistakes to avoid, see [Description Guidelines](/developer-guide/check-metadata-guidelines#description-guidelines). #### Risk A clear, natural language explanation of **why this finding poses a cybersecurity risk**. Focus on how it may impact confidentiality, integrity, or availability. If those do not apply, describe any relevant operational or financial risks. Use simple paragraphs with line breaks if needed, but avoid sections, code blocks, or complex formatting. Limit your explanation to 400 characters. -For detailed writing guidelines and common mistakes to avoid, see [Risk Guidelines](./check-metadata-guidelines.md#risk-guidelines). +For detailed writing guidelines and common mistakes to avoid, see [Risk Guidelines](/developer-guide/check-metadata-guidelines#risk-guidelines). #### RelatedUrl @@ -328,9 +332,10 @@ For detailed writing guidelines and common mistakes to avoid, see [Risk Guidelin #### AdditionalURLs -???+ warning - URLs must be valid and not repeated. + +URLs must be valid and not repeated. + A list of official documentation URLs for further reading. These should be authoritative sources that provide additional context, best practices, or detailed information about the security control being checked. Prefer official provider documentation, security standards, or well-established security resources. Avoid third-party blogs or unofficial sources unless they are highly reputable and directly relevant. #### Remediation @@ -345,17 +350,17 @@ Provides both code examples and best practice recommendations for addressing the - **Terraform**: HashiCorp Configuration Language (HCL) code with an example of a compliant configuration. - **Other**: Manual steps through web interfaces or other tools to make the finding compliant. - For detailed guidelines on writing remediation code, see [Remediation Code Guidelines](./check-metadata-guidelines.md#remediation-code-guidelines). + For detailed guidelines on writing remediation code, see [Remediation Code Guidelines](/developer-guide/check-metadata-guidelines#remediation-code-guidelines). - **Recommendation** - - **Text**: Generic best practice guidance in natural language using Markdown format (maximum 400 characters). For writing guidelines, see [Recommendation Guidelines](./check-metadata-guidelines.md#recommendation-guidelines). + - **Text**: Generic best practice guidance in natural language using Markdown format (maximum 400 characters). For writing guidelines, see [Recommendation Guidelines](/developer-guide/check-metadata-guidelines#recommendation-guidelines). - **Url**: [Prowler Hub URL](https://hub.prowler.com/) of the check. This URL is always composed by `https://hub.prowler.com/check/`. #### Categories One or more functional groupings used for execution filtering (e.g., `internet-exposed`). You can define new categories just by adding to this field. -For the complete list of available categories, see [Categories Guidelines](./check-metadata-guidelines.md#categories-guidelines). +For the complete list of available categories, see [Categories Guidelines](/developer-guide/check-metadata-guidelines#categories-guidelines). #### DependsOn diff --git a/docs/developer-guide/configurable-checks.md b/docs/developer-guide/configurable-checks.mdx similarity index 94% rename from docs/developer-guide/configurable-checks.md rename to docs/developer-guide/configurable-checks.mdx index aad072ddd5..76b339601d 100644 --- a/docs/developer-guide/configurable-checks.md +++ b/docs/developer-guide/configurable-checks.mdx @@ -1,4 +1,6 @@ -# Configurable Checks in Prowler +--- +title: 'Configurable Checks in Prowler' +--- Prowler empowers users to extend and adapt cloud security coverage by making checks configurable through the use of the `audit_config` object. This approach enables customization of checks to meet specific requirements through a configuration file. @@ -41,6 +43,6 @@ When adding a new configurable check to Prowler, update the following files: - **Test Fixtures:** If tests depend on this configuration, add the variable to `tests/config/fixtures/config.yaml`. - **Documentation:** Document the new variable in the list of configurable checks in `docs/tutorials/configuration_file.md`. -For a complete list of checks that already support configuration, see the [Configuration File Tutorial](../tutorials/configuration_file.md). +For a complete list of checks that already support configuration, see the [Configuration File Tutorial](/user-guide/cli/tutorials/configuration_file). This approach ensures that checks are easily configurable, making Prowler highly adaptable to different environments and requirements. diff --git a/docs/developer-guide/debugging.md b/docs/developer-guide/debugging.mdx similarity index 98% rename from docs/developer-guide/debugging.md rename to docs/developer-guide/debugging.mdx index a323ec0187..647b23c614 100644 --- a/docs/developer-guide/debugging.md +++ b/docs/developer-guide/debugging.mdx @@ -1,4 +1,6 @@ -# Debugging in Prowler +--- +title: 'Debugging in Prowler' +--- Debugging in Prowler simplifies the development process, allowing developers to efficiently inspect and resolve unexpected issues during execution. diff --git a/docs/developer-guide/documentation.md b/docs/developer-guide/documentation.md deleted file mode 100644 index 50ac0956e7..0000000000 --- a/docs/developer-guide/documentation.md +++ /dev/null @@ -1,28 +0,0 @@ -## Contributing to Documentation - -Prowler documentation is built using `mkdocs`, allowing contributors to easily add or enhance documentation. - -### Installation and Setup - -Install all necessary dependencies using: `poetry install --with docs`. - -1. Install `mkdocs` using your preferred package manager. - -2. Running the Documentation Locally -Navigate to the `prowler` repository folder. -Start the local documentation server by running: `mkdocs serve`. -Open `http://localhost:8000` in your browser to view live updates. - -3. Making Documentation Changes -Make all needed changes to docs or add new documents. Edit existing Markdown (.md) files inside `prowler/docs`. -To add new sections or files, update the `mkdocs.yaml` file located in the root directory of Prowler’s repository. - -4. Submitting Changes - -Once documentation updates are complete: - -Submit a pull request for review. - -The Prowler team will assess and merge contributions. - -Your efforts help improve Prowler documentation—thank you for contributing! diff --git a/docs/developer-guide/documentation.mdx b/docs/developer-guide/documentation.mdx new file mode 100644 index 0000000000..f1fae30d35 --- /dev/null +++ b/docs/developer-guide/documentation.mdx @@ -0,0 +1,61 @@ +--- +title: 'Contributing to Documentation' +--- + +Prowler documentation is built using [Mintlify](https://www.mintlify.com/docs), allowing contributors to easily add or enhance documentation. + +## Documentation Structure + +The Prowler documentation is organized into several sections. The main ones are: + +- **Getting Started**: Provides an overview of the Prowler platform and its different solutions, including Prowler Cloud/App, Prowler CLI, Prowler MCP Server, Prowler Hub, and Prowler Lighthouse AI. This section helps new users understand which Prowler solution best fits their needs and includes product comparisons. + +- **Guides**: Contains practical tutorials and how-to guides organized by product (Prowler Cloud/App, CLI) and provider (AWS, Azure, GCP, Kubernetes, Microsoft 365, GitHub, etc.). This section covers authentication, integrations, compliance, and advanced usage scenarios. + +- **Developer Guide**: Documentation for contributors looking to extend Prowler functionality. This includes guides on creating providers, services, checks, output formats, integrations, and compliance frameworks. Provider-specific implementation details and testing strategies are also covered here. + +- **Troubleshooting**: Common issues, error messages, and their solutions. This section helps users resolve problems encountered during installation, configuration, or execution. + + +## AI-Driven Documentation + +As mentioned in the [Introduction](/developer-guide/introduction#ai-driven-contributions), we have specialized resources to enhance AI-driven development. + +This includes the [AGENTS.md](https://github.com/prowler-cloud/prowler/blob/master/docs/AGENTS.md) file that contains the guidelines and style guide for the AI agents in the Prowler documentation. + +## Local Development + + + + ```bash + npm i -g mint + ``` + For detailed instructions, check the [Mintlify documentation](https://www.mintlify.com/docs/installation). + + + + Start the local development server to preview changes in real-time. + + ```bash + mint dev + ``` + + A local preview of your documentation will be available at http://localhost:3000 + + + + Edit existing Markdown (.mdx) files inside the `docs` directory or add new documents. + + For reference about formatting, check the [Mintlify documentation](https://www.mintlify.com/docs/create/text). + + To add new sections or files, update the [`docs/docs.json`](https://github.com/prowler-cloud/prowler/blob/master/docs/docs.json) file to include them in the navigation. + + + + Once documentation updates are complete, [submit a pull request for review](/developer-guide/introduction#sending-the-pull-request). + + The Prowler team will assess and merge contributions. + + + +Your efforts help improve Prowler documentation. Thank you for contributing! 🤘 diff --git a/docs/developer-guide/gcp-details.md b/docs/developer-guide/gcp-details.mdx similarity index 88% rename from docs/developer-guide/gcp-details.md rename to docs/developer-guide/gcp-details.mdx index dc13958a05..6011b784c9 100644 --- a/docs/developer-guide/gcp-details.md +++ b/docs/developer-guide/gcp-details.mdx @@ -1,12 +1,14 @@ -# Google Cloud Provider +--- +title: 'Google Cloud Provider' +--- This page details the [Google Cloud Platform (GCP)](https://cloud.google.com/) provider implementation in Prowler. -By default, Prowler will audit all the GCP projects that the authenticated identity can access. To configure it, follow the [GCP getting started guide](../tutorials/gcp/getting-started-gcp.md). +By default, Prowler will audit all the GCP projects that the authenticated identity can access. To configure it, follow the [GCP getting started guide](/user-guide/providers/gcp/getting-started-gcp). ## GCP Provider Classes Architecture -The GCP provider implementation follows the general [Provider structure](./provider.md). This section focuses on the GCP-specific implementation, highlighting how the generic provider concepts are realized for GCP in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](./provider.md). +The GCP provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the GCP-specific implementation, highlighting how the generic provider concepts are realized for GCP in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). ### Main Class @@ -32,7 +34,7 @@ The GCP provider implementation follows the general [Provider structure](./provi ### `GCPService` (Service Base Class) - **Location:** [`prowler/providers/gcp/lib/service/service.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/gcp/lib/service/service.py) -- **Purpose:** Abstract base class that all GCP service-specific classes inherit from. This implements the generic service pattern (described in [service page](./services.md#service-base-class)) specifically for GCP. +- **Purpose:** Abstract base class that all GCP service-specific classes inherit from. This implements the generic service pattern (described in [service page](/developer-guide/services#service-base-class)) specifically for GCP. - **Key GCP Responsibilities:** - Receives a `GcpProvider` instance to access session, identity, and configuration. - Manages clients for all services by project. @@ -95,12 +97,12 @@ def _get_instances(self): ## Specific Patterns in GCP Services -The generic service pattern is described in [service page](./services.md#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: +The generic service pattern is described in [service page](/developer-guide/services#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: - Directly in the code, in location [`prowler/providers/gcp/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/gcp/services) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new service is following the [service implementation documentation](./services.md#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used accross all GCP services. +The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other services already implemented as reference. In next subsection you can find a list of common patterns that are used accross all GCP services. ### GCP Service Common Patterns @@ -117,12 +119,12 @@ The best reference to understand how to implement a new service is following the ## Specific Patterns in GCP Checks -The GCP checks pattern is described in [checks page](./checks.md). You can find all the currently implemented checks: +The GCP checks pattern is described in [checks page](/developer-guide/checks). You can find all the currently implemented checks: - Directly in the code, within each service folder, each check has its own folder named after the name of the check. (e.g. [`prowler/providers/gcp/services/iam/iam_sa_user_managed_key_unused/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/gcp/services/iam/iam_sa_user_managed_key_unused)) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new check is following the [GCP check implementation documentation](./checks.md#creating-a-check) and taking other similar checks as reference. +The best reference to understand how to implement a new check is following the [GCP check implementation documentation](/developer-guide/checks#creating-a-check) and taking other similar checks as reference. ### Check Report Class diff --git a/docs/developer-guide/github-details.md b/docs/developer-guide/github-details.mdx similarity index 86% rename from docs/developer-guide/github-details.md rename to docs/developer-guide/github-details.mdx index 97a1cffac3..1b5546cd16 100644 --- a/docs/developer-guide/github-details.md +++ b/docs/developer-guide/github-details.mdx @@ -1,12 +1,14 @@ -# GitHub Provider +--- +title: 'GitHub Provider' +--- This page details the [GitHub](https://github.com/) provider implementation in Prowler. -By default, Prowler will audit the GitHub account - scanning all repositories, organizations, and applications that your configured credentials can access. To configure it, follow the [GitHub getting started guide](../tutorials/github/getting-started-github.md). +By default, Prowler will audit the GitHub account - scanning all repositories, organizations, and applications that your configured credentials can access. To configure it, follow the [GitHub getting started guide](/user-guide/providers/github/getting-started-github). ## GitHub Provider Classes Architecture -The GitHub provider implementation follows the general [Provider structure](./provider.md). This section focuses on the GitHub-specific implementation, highlighting how the generic provider concepts are realized for GitHub in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](./provider.md). +The GitHub provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the GitHub-specific implementation, highlighting how the generic provider concepts are realized for GitHub in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). ### `GithubProvider` (Main Class) @@ -48,12 +50,12 @@ The GitHub provider implementation follows the general [Provider structure](./pr ## Specific Patterns in GitHub Services -The generic service pattern is described in [service page](./services.md#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: +The generic service pattern is described in [service page](/developer-guide/services#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: - Directly in the code, in location [`prowler/providers/github/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/github/services) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new service is following the [service implementation documentation](./services.md#adding-a-new-service) and by taking other already implemented services as reference. +The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and by taking other already implemented services as reference. ### GitHub Service Common Patterns @@ -66,12 +68,12 @@ The best reference to understand how to implement a new service is following the ## Specific Patterns in GitHub Checks -The GitHub checks pattern is described in [checks page](./checks.md). You can find all the currently implemented checks in: +The GitHub checks pattern is described in [checks page](/developer-guide/checks). You can find all the currently implemented checks in: - Directly in the code, within each service folder, each check has its own folder named after the name of the check. (e.g. [`prowler/providers/github/services/repository/repository_secret_scanning_enabled/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/github/services/repository/repository_secret_scanning_enabled)) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new check is the [GitHub check implementation documentation](./checks.md#creating-a-check) and by taking other checks as reference. +The best reference to understand how to implement a new check is the [GitHub check implementation documentation](/developer-guide/checks#creating-a-check) and by taking other checks as reference. ### Check Report Class diff --git a/docs/developer-guide/integration-testing.md b/docs/developer-guide/integration-testing.md deleted file mode 100644 index 0d03319c5e..0000000000 --- a/docs/developer-guide/integration-testing.md +++ /dev/null @@ -1,3 +0,0 @@ -# Integration Tests - -Coming soon ... \ No newline at end of file diff --git a/docs/developer-guide/integration-testing.mdx b/docs/developer-guide/integration-testing.mdx new file mode 100644 index 0000000000..e8f0f804f8 --- /dev/null +++ b/docs/developer-guide/integration-testing.mdx @@ -0,0 +1,5 @@ +--- +title: 'Integration Tests' +--- + +Coming soon ... diff --git a/docs/developer-guide/integrations.md b/docs/developer-guide/integrations.mdx similarity index 99% rename from docs/developer-guide/integrations.md rename to docs/developer-guide/integrations.mdx index e3f6789a8e..cb4b242d8c 100644 --- a/docs/developer-guide/integrations.md +++ b/docs/developer-guide/integrations.mdx @@ -1,4 +1,6 @@ -# Creating a New Integration +--- +title: 'Creating a New Integration' +--- ## Introduction @@ -151,7 +153,7 @@ Refer to the [Prowler Developer Guide](https://docs.prowler.com/projects/prowler # More properties and methods ``` - + * Test Connection Method: * Validating Credentials or Tokens diff --git a/docs/developer-guide/introduction.md b/docs/developer-guide/introduction.md deleted file mode 100644 index abc83fb682..0000000000 --- a/docs/developer-guide/introduction.md +++ /dev/null @@ -1,176 +0,0 @@ -# Introduction to developing in Prowler - -Extending Prowler - -Prowler can be extended in various ways, with common use cases including: - -- New security checks -- New compliance frameworks -- New output formats -- New integrations -- New proposed features - -All the relevant information for these cases is included in this guide. - -## Getting the Code and Installing All Dependencies - -### Prerequisites - -Before proceeding, ensure the following: - -- Git is installed. -- Python 3.9 or higher is installed. -- `poetry` is installed to manage dependencies. - -### Forking the Prowler Repository - -To contribute to Prowler, fork the Prowler GitHub repository. This allows you to propose changes, submit new features, and fix bugs. For guidance on forking, refer to the [official GitHub documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo?tool=webui#forking-a-repository). - -### Cloning Your Forked Repository - -Once your fork is created, clone it using the following commands: - -``` -git clone https://github.com//prowler -cd prowler -``` - -### Dependency Management and Environment Isolation - -To prevent conflicts between environments, we recommend using `poetry`, a Python dependency management solution. Install it by following the [instructions](https://python-poetry.org/docs/#installation). - -### Installing Dependencies - -To install all required dependencies, including those needed for development, run: - -``` -poetry install --with dev -eval $(poetry env activate) -``` - -???+ important - Starting from Poetry v2.0.0, `poetry shell` has been deprecated in favor of `poetry env activate`. - If your poetry version is below 2.0.0 you must keep using `poetry shell` to activate your environment. - In case you have any doubts, consult the [Poetry environment activation guide](https://python-poetry.org/docs/managing-environments/#activating-the-environment). - -## Contributing to Prowler - -### Ways to Contribute - -Here are some ideas for collaborating with Prowler: - -1. **Review Current Issues**: Check out our [GitHub Issues](https://github.com/prowler-cloud/prowler/issues) page. We often tag issues as `good first issue` - these are perfect for new contributors as they are typically well-defined and manageable in scope. - -2. **Expand Prowler's Capabilities**: Prowler is constantly evolving, and you can be a part of its growth. Whether you are adding checks, supporting new services, or introducing integrations, your contributions help improve the tool for everyone. Here is how you can get involved: - - - **Adding New Checks** - Want to improve Prowler's detection capabilities for your favorite cloud provider? You can contribute by writing new checks. To get started, follow the [create a new check guide](./checks.md). - - - **Adding New Services** - One key service for your favorite cloud provider is missing? Add it to Prowler! To add a new service, check out the [create a new service guide](./services.md). Do not forget to include relevant checks to validate functionality. - - - **Adding New Providers** - If you would like to extend Prowler to work with a new cloud provider, follow the [create a new provider guide](./provider.md). This typically involves setting up new services and checks to ensure compatibility. - - - **Adding New Output Formats** - Want to tailor how results are displayed or exported? You can add custom output formats by following the [create a new output format guide](./outputs.md). - - - **Adding New Integrations** - Prowler can work with other tools and platforms through integrations. If you would like to add one, see the [create a new integration guide](./integrations.md). - - - **Proposing or Implementing Features** - Got an idea to make Prowler better? Whether it is a brand-new feature or an enhancement to an existing one, you are welcome to propose it or help implement community-requested improvements. - -3. **Improve Documentation**: Help make Prowler more accessible by enhancing our documentation, fixing typos, or adding examples/tutorials. See the tutorial of how we write our documentation [here](./documentation.md). - -4. **Bug Fixes**: If you find any issues or bugs, you can report them in the [GitHub Issues](https://github.com/prowler-cloud/prowler/issues) page and if you want you can also fix them. - -Remember, our community is here to help! If you need guidance, do not hesitate to ask questions in the issues or join our [Slack workspace](https://goto.prowler.com/slack). - -### Pre-Commit Hooks - -This repository uses Git pre-commit hooks managed by the [pre-commit](https://pre-commit.com/) tool, it is installed with `poetry install --with dev`. Next, run the following command in the root of this repository: - -```shell -pre-commit install -``` - -Successful installation should produce the following output: - -```shell -pre-commit installed at .git/hooks/pre-commit -``` - -### Code Quality and Security Checks - -Before merging pull requests, several automated checks and utilities ensure code security and updated dependencies: - -???+ note - These should have been already installed if `poetry install --with dev` was already run. - -- [`bandit`](https://pypi.org/project/bandit/) for code security review. -- [`safety`](https://pypi.org/project/safety/) and [`dependabot`](https://github.com/features/security) for dependencies. -- [`hadolint`](https://github.com/hadolint/hadolint) and [`dockle`](https://github.com/goodwithtech/dockle) for container security. -- [`Snyk`](https://docs.snyk.io/integrations/snyk-container-integrations/container-security-with-docker-hub-integration) for container security in Docker Hub. -- [`clair`](https://github.com/quay/clair) for container security in Amazon ECR. -- [`vulture`](https://pypi.org/project/vulture/), [`flake8`](https://pypi.org/project/flake8/), [`black`](https://pypi.org/project/black/), and [`pylint`](https://pypi.org/project/pylint/) for formatting and best practices. - -Additionally, ensure the latest version of [`TruffleHog`](https://github.com/trufflesecurity/trufflehog) is installed to scan for sensitive data in the code. Follow the official [installation guide](https://github.com/trufflesecurity/trufflehog?tab=readme-ov-file#floppy_disk-installation) for setup. - -### Dependency Management - -All dependencies are listed in the `pyproject.toml` file. - -For proper code documentation, refer to the following and follow the code documentation practices presented there: [Google Python Style Guide - Comments and Docstrings](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings). - -???+ note - If you encounter issues when committing to the Prowler repository, use the `--no-verify` flag with the `git commit` command. - -### Repository Folder Structure - -Understanding the layout of the Prowler codebase will help you quickly find where to add new features, checks, or integrations. The following is a high-level overview from the root of the repository: - -``` -prowler/ -├── prowler/ # Main source code for Prowler SDK (CLI, providers, services, checks, compliances, config, etc.) -├── api/ # API server and related code -├── dashboard/ # Local Dashboard extracted from the CLI output -├── ui/ # Web UI components -├── util/ # Utility scripts and helpers -├── tests/ # Prowler SDK test suite -├── docs/ # Documentation, including this guide -├── examples/ # Example output formats for providers and scripts -├── permissions/ # Permission-related files and policies -├── contrib/ # Community-contributed scripts or modules -├── kubernetes/ # Kubernetes deployment files -├── .github/ # GitHub related files (workflows, issue templates, etc.) -├── pyproject.toml # Python project configuration (Poetry) -├── poetry.lock # Poetry lock file -├── README.md # Project overview and getting started -├── Makefile # Common development commands -├── Dockerfile # SDK Docker container -├── docker-compose.yml # Prowler App Docker compose -└── ... # Other supporting files -``` - -## Pull Request Checklist - -When creating or reviewing a pull request in https://github.com/prowler-cloud/prowler, follow [this checklist](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md#checklist). - -## Contribution Appreciation - -If you enjoy swag, we’d love to thank you for your contribution with laptop stickers or other Prowler merchandise! - -To request swag: Share your pull request details in our [Slack workspace](https://goto.prowler.com/slack). - -You can also reach out to Toni de la Fuente on [Twitter](https://twitter.com/ToniBlyx)—his DMs are open! - -# Testing a Pull Request from a Specific Branch - -To test Prowler from a specific branch (for example, to try out changes from a pull request before it is merged), you can use `pipx` to install directly from GitHub: - -```sh -pipx install "git+https://github.com/prowler-cloud/prowler.git@branch-name" -``` - -Replace `branch-name` with the name of the branch you want to test. This will install Prowler in an isolated environment, allowing you to try out the changes safely. \ No newline at end of file diff --git a/docs/developer-guide/introduction.mdx b/docs/developer-guide/introduction.mdx new file mode 100644 index 0000000000..9d1ea742b1 --- /dev/null +++ b/docs/developer-guide/introduction.mdx @@ -0,0 +1,216 @@ +--- +title: 'Introduction to developing in Prowler' +--- + +Thanks for your interest in contributing to Prowler! + +Prowler can be extended in various ways. This guide provides the different ways to contribute and how to get started. + +## Contributing to Prowler + +### Review Current Issues +Check out our [GitHub Issues](https://github.com/prowler-cloud/prowler/issues) page for ideas to contribute. + + + We tag issues as `good first issue` for new contributors. These are typically well-defined and manageable in scope. + + + We tag issues as `help wanted` for other issues that require more time to complete. + + + +### Expand Prowler's Capabilities +Prowler is constantly evolving. Contributions to checks, services, or integrations help improve the tool for everyone. Here is how to get involved: + + + + Want to improve Prowler's detection capabilities for your favorite cloud provider? You can contribute by writing new checks. + + + One key service for your favorite cloud provider is missing? Add it to Prowler! Do not forget to include relevant checks to validate functionality. + + + If you would like to extend Prowler to work with a new cloud provider, this typically involves setting up new services and checks to ensure compatibility. + + + Want to tailor how results are displayed or exported? You can add custom output formats. + + + Prowler can work with other tools and platforms through integrations. + + + Propose brand-new features or enhancements to existing ones, or help implement community-requested improvements. + + + +### Improve Documentation +Help make Prowler more accessible by enhancing our documentation, fixing typos, or adding examples/tutorials. + + + + Enhance our documentation, fix typos, or add examples/tutorials. + + + +### Bug Fixes +If you find any issues or bugs, you can report them in the [GitHub Issues](https://github.com/prowler-cloud/prowler/issues) page and if you want you can also fix them. + + + + Report or fix issues or bugs. + + + +Remember, our community is here to help! If you need guidance, do not hesitate to ask questions in the issues or join our [ Slack workspace](https://goto.prowler.com/slack). + + + +## Setting up your development environment + +### Prerequisites + +Before proceeding, ensure the following: + +- Git is installed. +- Python 3.9 or higher is installed. +- `poetry` is installed to manage dependencies. + +### Forking the Prowler Repository + +Fork the Prowler GitHub repository to contribute to Prowler. This allows proposing changes, submitting new features, and fixing bugs. For guidance on forking, refer to the [official GitHub documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo?tool=webui#forking-a-repository). + +### Cloning Your Forked Repository + +Once your fork is created, clone it using the following commands (replace `` with your GitHub username): + +``` +git clone https://github.com//prowler +cd prowler +``` + +### Dependency Management and Environment Isolation + +To prevent conflicts between environments, we recommend using `poetry`, a Python dependency management solution. Install it by following the [instructions](https://python-poetry.org/docs/#installation). + +### Installing Dependencies + +To install all required dependencies, including those needed for development, run: + +``` +poetry install --with dev +eval $(poetry env activate) +``` + + +Starting from Poetry v2.0.0, `poetry shell` has been deprecated in favor of `poetry env activate`. +If your poetry version is below 2.0.0 you must keep using `poetry shell` to activate your environment. +In case you have any doubts, consult the [Poetry environment activation guide](https://python-poetry.org/docs/managing-environments/#activating-the-environment). + + + + +### Pre-Commit Hooks + +This repository uses Git pre-commit hooks managed by the [pre-commit](https://pre-commit.com/) tool, it is installed with `poetry install --with dev`. Next, run the following command in the root of this repository: + +```shell +pre-commit install +``` + +Successful installation should produce the following output: + +```shell +pre-commit installed at .git/hooks/pre-commit +``` + +### Code Quality and Security Checks + +Before merging pull requests, several automated checks and utilities ensure code security and updated dependencies: + + +These should have been already installed if `poetry install --with dev` was already run. + + +- [`bandit`](https://pypi.org/project/bandit/) for code security review. +- [`safety`](https://pypi.org/project/safety/) and [`dependabot`](https://github.com/features/security) for dependencies. +- [`hadolint`](https://github.com/hadolint/hadolint) and [`dockle`](https://github.com/goodwithtech/dockle) for container security. +- [`Snyk`](https://docs.snyk.io/integrations/snyk-container-integrations/container-security-with-docker-hub-integration) for container security in Docker Hub. +- [`clair`](https://github.com/quay/clair) for container security in Amazon ECR. +- [`vulture`](https://pypi.org/project/vulture/), [`flake8`](https://pypi.org/project/flake8/), [`black`](https://pypi.org/project/black/), and [`pylint`](https://pypi.org/project/pylint/) for formatting and best practices. + +Additionally, ensure the latest version of [`TruffleHog`](https://github.com/trufflesecurity/trufflehog) is installed to scan for sensitive data in the code. Follow the official [installation guide](https://github.com/trufflesecurity/trufflehog?tab=readme-ov-file#floppy_disk-installation) for setup. + +### AI-Driven Contributions + +If you are using AI assistants to help with your contributions, Prowler provides specialized resources to enhance AI-driven development: + +- **Prowler MCP Server**: The [Prowler MCP Server](/getting-started/products/prowler-mcp) provides AI assistants with access to the entire Prowler ecosystem, including security checks, compliance frameworks, documentation, and more. This enables AI tools to better understand Prowler's architecture and help you create contributions that align with project standards. + +- **AGENTS.md Files**: Each component of the Prowler monorepo includes an `AGENTS.md` file that contains specific guidelines for AI agents working on that component. These files provide context about project structure, coding standards, and best practices. When working on a specific component, refer to the relevant `AGENTS.md` file (e.g., `prowler/AGENTS.md`, `ui/AGENTS.md`, `api/AGENTS.md`) to ensure your AI assistant follows the appropriate guidelines. + +These resources help ensure that AI-assisted contributions maintain consistency with Prowler's codebase and development practices. + +### Dependency Management + +All dependencies are listed in the `pyproject.toml` file. + +For proper code documentation, refer to the following and follow the code documentation practices presented there: [Google Python Style Guide - Comments and Docstrings](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings). + + +If you encounter issues when committing to the Prowler repository, use the `--no-verify` flag with the `git commit` command. + + +### Repository Folder Structure + +The Prowler codebase layout helps quickly locate where to add new features, checks, or integrations. The following is a high-level overview from the root of the repository: + +``` +prowler/ +├── prowler/ # Main source code for Prowler SDK (CLI, providers, services, checks, compliances, config, etc.) +├── api/ # API server and related code +├── dashboard/ # Local Dashboard extracted from the CLI output +├── ui/ # Web UI components +├── util/ # Utility scripts and helpers +├── tests/ # Prowler SDK test suite +├── docs/ # Documentation, including this guide +├── examples/ # Example output formats for providers and scripts +├── permissions/ # Permission-related files and policies +├── contrib/ # Community-contributed scripts or modules +├── kubernetes/ # Kubernetes deployment files +├── .github/ # GitHub-related files (workflows, issue templates, etc.) +├── pyproject.toml # Python project configuration (Poetry) +├── poetry.lock # Poetry lock file +├── README.md # Project overview and getting started +├── Makefile # Common development commands +├── Dockerfile # SDK Docker container +├── docker-compose.yml # Prowler App Docker compose +└── ... # Other supporting files +``` + +## Sending the Pull Request + +When creating or reviewing a pull request in [Prowler](https://github.com/prowler-cloud/prowler), follow [this template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md) and fill it with the relevant information: + +- **Context** and **Description** of the change: This will help the reviewers to understand the change and the purpose of the pull request. +- **Steps to review**: A detailed description of how to review the change. +- **Checklist**: A mandatory checklist of the things that should be reviewed before merging the pull request. + +## Contribution Appreciation + +If you enjoy swag, we'd love to thank you for your contribution with laptop stickers or other Prowler merchandise! + +To request swag: Share your pull request details in our [Slack workspace](https://goto.prowler.com/slack). + +You can also reach out to Toni de la Fuente on [Twitter](https://twitter.com/ToniBlyx)—his DMs are open! + +## Testing a Pull Request from a Specific Branch + +To test Prowler from a specific branch (for example, to try out changes from a pull request before it is merged), you can use `pipx` to install directly from GitHub: + +```sh +pipx install "git+https://github.com/prowler-cloud/prowler.git@branch-name" +``` + +Replace `branch-name` with the name of the branch you want to test. This will install Prowler in an isolated environment, allowing you to try out the changes safely. + +For more details on testing go to the [Testing section](/developer-guide/unit-testing) of this documentation. \ No newline at end of file diff --git a/docs/developer-guide/kubernetes-details.md b/docs/developer-guide/kubernetes-details.mdx similarity index 83% rename from docs/developer-guide/kubernetes-details.md rename to docs/developer-guide/kubernetes-details.mdx index be4b72f383..0ccda94f38 100644 --- a/docs/developer-guide/kubernetes-details.md +++ b/docs/developer-guide/kubernetes-details.mdx @@ -1,12 +1,14 @@ -# Kubernetes Provider +--- +title: 'Kubernetes Provider' +--- This page details the [Kubernetes](https://kubernetes.io/) provider implementation in Prowler. -By default, Prowler will audit all namespaces in the Kubernetes cluster accessible by the configured context. To configure it, see the [In-Cluster Execution](../tutorials/kubernetes/in-cluster.md) or [Non In-Cluster Execution](../tutorials/kubernetes/outside-cluster.md) guides. +By default, Prowler will audit all namespaces in the Kubernetes cluster accessible by the configured context. To configure it, see the [In-Cluster Execution](/user-guide/providers/kubernetes/getting-started-k8s#in-cluster-execution) or [Non In-Cluster Execution](/user-guide/providers/kubernetes/getting-started-k8s#non-in-cluster-execution) guides. ## Kubernetes Provider Classes Architecture -The Kubernetes provider implementation follows the general [Provider structure](./provider.md). This section focuses on the Kubernetes-specific implementation, highlighting how the generic provider concepts are realized for Kubernetes in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](./provider.md). +The Kubernetes provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the Kubernetes-specific implementation, highlighting how the generic provider concepts are realized for Kubernetes in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). ### `KubernetesProvider` (Main Class) @@ -31,7 +33,7 @@ The Kubernetes provider implementation follows the general [Provider structure]( ### `KubernetesService` (Service Base Class) - **Location:** [`prowler/providers/kubernetes/lib/service/service.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/kubernetes/lib/service/service.py) -- **Purpose:** Abstract base class that all Kubernetes service-specific classes inherit from. This implements the generic service pattern (described in [service page](./services.md#service-base-class)) specifically for Kubernetes. +- **Purpose:** Abstract base class that all Kubernetes service-specific classes inherit from. This implements the generic service pattern (described in [service page](/developer-guide/services#service-base-class)) specifically for Kubernetes. - **Key Kubernetes Responsibilities:** - Receives a `KubernetesProvider` instance to access session, identity, and configuration. - Manages the Kubernetes API client and context. @@ -50,12 +52,12 @@ The Kubernetes provider implementation follows the general [Provider structure]( ## Specific Patterns in Kubernetes Services -The generic service pattern is described in [service page](./services.md#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: +The generic service pattern is described in [service page](/developer-guide/services#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: - Directly in the code, in location [`prowler/providers/kubernetes/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/kubernetes/services) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new service is following the [service implementation documentation](./services.md#adding-a-new-service) and taking other already implemented services as reference. +The best reference to understand how to implement a new service is following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and taking other already implemented services as reference. ### Kubernetes Service Common Patterns @@ -69,12 +71,12 @@ The best reference to understand how to implement a new service is following the ## Specific Patterns in Kubernetes Checks -The Kubernetes checks pattern is described in [checks page](./checks.md). You can find all the currently implemented checks in: +The Kubernetes checks pattern is described in [checks page](/developer-guide/checks). You can find all the currently implemented checks in: - Directly in the code, within each service folder, each check has its own folder named after the name of the check. (e.g. [`prowler/providers/kubernetes/services/rbac/rbac_minimize_wildcard_use_roles/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/kubernetes/services/rbac/rbac_minimize_wildcard_use_roles)) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new check is following the [Kubernetes check implementation documentation](./checks.md#creating-a-check) and taking other checks as reference. +The best reference to understand how to implement a new check is following the [Kubernetes check implementation documentation](/developer-guide/checks#creating-a-check) and taking other checks as reference. ### Check Report Class diff --git a/docs/developer-guide/lighthouse.md b/docs/developer-guide/lighthouse.mdx similarity index 89% rename from docs/developer-guide/lighthouse.md rename to docs/developer-guide/lighthouse.mdx index e01067844f..25afd51728 100644 --- a/docs/developer-guide/lighthouse.md +++ b/docs/developer-guide/lighthouse.mdx @@ -1,4 +1,6 @@ -# Extending Prowler Lighthouse AI +--- +title: 'Extending Prowler Lighthouse AI' +--- This guide helps developers customize and extend Prowler Lighthouse AI by adding or modifying AI agents. @@ -15,9 +17,10 @@ AI agents fall into two main categories: Prowler Lighthouse AI is an autonomous agent - selecting the right tool(s) based on the users query. -???+ note - To learn more about AI agents, read [Anthropic's blog post on building effective agents](https://www.anthropic.com/engineering/building-effective-agents). + +To learn more about AI agents, read [Anthropic's blog post on building effective agents](https://www.anthropic.com/engineering/building-effective-agents). + ### LLM Dependency The autonomous nature of agents depends on the underlying LLM. Autonomous agents using identical system prompts and tools but powered by different LLM providers might approach user queries differently. Agent with one LLM might solve a problem efficiently, while with another it might take a different route or fail entirely. @@ -30,7 +33,7 @@ Prowler Lighthouse AI uses a multi-agent architecture orchestrated by the [Langg ### Architecture Components -Prowler Lighthouse architecture +Prowler Lighthouse architecture Prowler Lighthouse AI integrates with the NextJS application: @@ -67,9 +70,10 @@ Modifying the supervisor prompt allows you to: - Modify task delegation to specialized agents - Set up guardrails (query types to answer or decline) -???+ note - The supervisor agent should not have its own tools. This design keeps the system modular and maintainable. + +The supervisor agent should not have its own tools. This design keeps the system modular and maintainable. + ### How to Create New Specialized Agents The supervisor agent and all specialized agents are defined in the `route.ts` file. The supervisor agent uses [langgraph-supervisor](https://www.npmjs.com/package/@langchain/langgraph-supervisor), while other agents use the prebuilt [create-react-agent](https://langchain-ai.github.io/langgraphjs/how-tos/create-react-agent/). @@ -77,14 +81,16 @@ The supervisor agent and all specialized agents are defined in the `route.ts` fi To add new capabilities or all Lighthouse AI to interact with other APIs, create additional specialized agents: 1. First determine what the new agent would do. Create a detailed prompt defining the agent's purpose and capabilities. You can see an example from [here](https://github.com/prowler-cloud/prowler/blob/master/ui/lib/lighthouse/prompts.ts#L359-L385). -???+ note - Ensure that the new agent's capabilities don't collide with existing agents. For example, if there's already a *findings_agent* that talks to findings APIs don't create a new agent to do the same. + +Ensure that the new agent's capabilities don't collide with existing agents. For example, if there's already a *findings_agent* that talks to findings APIs don't create a new agent to do the same. + 2. Create necessary tools for the agents to access specific data or perform actions. A tool is a specialized function that extends the capabilities of LLM by allowing it to access external data or APIs. A tool is triggered by LLM based on the description of the tool and the user's query. For example, the description of `getScanTool` is "Fetches detailed information about a specific scan by its ID." If the description doesn't convey what the tool is capable of doing, LLM will not invoke the function. If the description of `getScanTool` was set to something random or not set at all, LLM will not answer queries like "Give me the critical issues from the scan ID xxxxxxxxxxxxxxx" -???+ note - Ensure that one tool is added to one agent only. Adding tools is optional. There can be agents with no tools at all. + +Ensure that one tool is added to one agent only. Adding tools is optional. There can be agents with no tools at all. + 3. Use the `createReactAgent` function to define a new agent. For example, the rolesAgent name is "roles_agent" and has access to call tools "*getRolesTool*" and "*getRoleTool*" ```js const rolesAgent = createReactAgent({ diff --git a/docs/developer-guide/llm-details.md b/docs/developer-guide/llm-details.mdx similarity index 92% rename from docs/developer-guide/llm-details.md rename to docs/developer-guide/llm-details.mdx index 0c49422ada..3647552c0f 100644 --- a/docs/developer-guide/llm-details.md +++ b/docs/developer-guide/llm-details.mdx @@ -1,12 +1,14 @@ -# LLM Provider +--- +title: 'LLM Provider' +--- This page details the [Large Language Model (LLM)](https://en.wikipedia.org/wiki/Large_language_model) provider implementation in Prowler. -The LLM provider enables security testing of language models using red team techniques. By default, Prowler uses the built-in LLM configuration that targets OpenAI models with comprehensive security test suites. To configure it, follow the [LLM getting started guide](../tutorials/llm/getting-started-llm.md). +The LLM provider enables security testing of language models using red team techniques. By default, Prowler uses the built-in LLM configuration that targets OpenAI models with comprehensive security test suites. To configure it, follow the [LLM getting started guide](/user-guide/providers/llm/getting-started-llm). ## LLM Provider Classes Architecture -The LLM provider implementation follows the general [Provider structure](./provider.md). This section focuses on the LLM-specific implementation, highlighting how the generic provider concepts are realized for LLM security testing in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](./provider.md). +The LLM provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the LLM-specific implementation, highlighting how the generic provider concepts are realized for LLM security testing in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). ### Main Class diff --git a/docs/developer-guide/m365-details.md b/docs/developer-guide/m365-details.mdx similarity index 86% rename from docs/developer-guide/m365-details.md rename to docs/developer-guide/m365-details.mdx index 65fcd3f622..185e1e022b 100644 --- a/docs/developer-guide/m365-details.md +++ b/docs/developer-guide/m365-details.mdx @@ -1,8 +1,10 @@ -# Microsoft 365 (M365) Provider +--- +title: 'Microsoft 365 (M365) Provider' +--- This page details the [Microsoft 365 (M365)](https://www.microsoft.com/en-us/microsoft-365) provider implementation in Prowler. -By default, Prowler will audit the Microsoft Entra ID tenant and its supported services. To configure it, follow the [M365 getting started guide](../tutorials/microsoft365/getting-started-m365.md). +By default, Prowler will audit the Microsoft Entra ID tenant and its supported services. To configure it, follow the [M365 getting started guide](/user-guide/providers/microsoft365/getting-started-m365). --- @@ -15,14 +17,14 @@ By default, Prowler will audit the Microsoft Entra ID tenant and its supported s - **Required modules:** - [ExchangeOnlineManagement](https://www.powershellgallery.com/packages/ExchangeOnlineManagement/3.6.0) (â‰Ĩ 3.6.0) - [MicrosoftTeams](https://www.powershellgallery.com/packages/MicrosoftTeams/6.6.0) (â‰Ĩ 6.6.0) -- If you use Prowler Cloud or the official containers, PowerShell is pre-installed. For local or pip installations, you must install PowerShell and the modules yourself. See [Authentication: Supported PowerShell Versions](../tutorials/microsoft365/authentication.md#supported-powershell-versions) and [Needed PowerShell Modules](../tutorials/microsoft365/authentication.md#required-powershell-modules). -- For more details and troubleshooting, see [Use of PowerShell in M365](../tutorials/microsoft365/use-of-powershell.md). +- If you use Prowler Cloud or the official containers, PowerShell is pre-installed. For local or pip installations, you must install PowerShell and the modules yourself. See [Authentication: Supported PowerShell Versions](/user-guide/providers/microsoft365/authentication#supported-powershell-versions) and [Needed PowerShell Modules](/user-guide/providers/microsoft365/authentication#required-powershell-modules). +- For more details and troubleshooting, see [Use of PowerShell in M365](/user-guide/providers/microsoft365/use-of-powershell). --- ## M365 Provider Classes Architecture -The M365 provider implementation follows the general [Provider structure](./provider.md). This section focuses on the M365-specific implementation, highlighting how the generic provider concepts are realized for M365 in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](./provider.md). +The M365 provider implementation follows the general [Provider structure](/developer-guide/provider). This section focuses on the M365-specific implementation, highlighting how the generic provider concepts are realized for M365 in Prowler. For a full overview of the provider pattern, base classes, and extension guidelines, see [Provider documentation](/developer-guide/provider). ### `M365Provider` (Main Class) @@ -73,12 +75,12 @@ The M365 provider implementation follows the general [Provider structure](./prov ## Specific Patterns in M365 Services -The generic service pattern is described in [service page](./services.md#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: +The generic service pattern is described in [service page](/developer-guide/services#service-structure-and-initialisation). You can find all the currently implemented services in the following locations: - Directly in the code, in location [`prowler/providers/m365/services/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/m365/services) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new service is by following the [service implementation documentation](./services.md#adding-a-new-service) and by taking other already implemented services as reference. +The best reference to understand how to implement a new service is by following the [service implementation documentation](/developer-guide/services#adding-a-new-service) and by taking other already implemented services as reference. ### M365 Service Common Patterns @@ -92,12 +94,12 @@ The best reference to understand how to implement a new service is by following ## Specific Patterns in M365 Checks -The M365 checks pattern is described in [checks page](./checks.md). You can find all the currently implemented checks in: +The M365 checks pattern is described in [checks page](/developer-guide/checks). You can find all the currently implemented checks in: - Directly in the code, within each service folder, each check has its own folder named after the name of the check. (e.g. [`prowler/providers/m365/services/entra/entra_users_mfa_enabled/`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers/m365/services/entra/entra_users_mfa_enabled)) - In the [Prowler Hub](https://hub.prowler.com/) for a more human-readable view. -The best reference to understand how to implement a new check is following the [M365 check implementation documentation](./checks.md#creating-a-check) and by taking other checks as reference. +The best reference to understand how to implement a new check is following the [M365 check implementation documentation](/developer-guide/checks#creating-a-check) and by taking other checks as reference. ### Check Report Class diff --git a/docs/developer-guide/outputs.md b/docs/developer-guide/outputs.mdx similarity index 99% rename from docs/developer-guide/outputs.md rename to docs/developer-guide/outputs.mdx index 60a9f6c498..15b785b799 100644 --- a/docs/developer-guide/outputs.md +++ b/docs/developer-guide/outputs.mdx @@ -1,4 +1,6 @@ -# Create a Custom Output Format +--- +title: 'Create a Custom Output Format' +--- ## Introduction diff --git a/docs/developer-guide/provider.md b/docs/developer-guide/provider.md deleted file mode 100644 index a258ff333d..0000000000 --- a/docs/developer-guide/provider.md +++ /dev/null @@ -1,172 +0,0 @@ -# Prowler Providers - -## Introduction - -Providers form the backbone of Prowler, enabling security assessments across various cloud environments. - -A provider is any platform or service that offers resources, data, or functionality that can be audited for security and compliance. This includes: - -- Cloud Infrastructure Providers (like Amazon Web Services, Microsoft Azure, and Google Cloud) -- Software as a Service (SaaS) Platforms (like Microsoft 365) -- Development Platforms (like GitHub) -- Container Orchestration Platforms (like Kubernetes) -- Database-as-a-Service Platforms (like MongoDB Atlas) - -For providers supported by Prowler, refer to [Prowler Hub](https://hub.prowler.com/). - -???+ important - There are some custom providers added by the community, like [NHN Cloud](https://www.nhncloud.com/), that are not maintained by the Prowler team, but can be used in the Prowler CLI. They can be checked directly at the [Prowler GitHub repository](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). - -## Adding a New Provider - -To integrate an unsupported Prowler provider and implement its security checks, create a dedicated folder for all related files (e.g., services, checks)." - -This folder must be placed within [`prowler/providers//`](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). - -Within this folder the following folders are also to be created: - -- `lib` – Stores additional utility functions and core files required by every provider. The following files and subfolders are commonly found in every provider's `lib` folder: - - - `service/service.py` – Provides a generic service class to be inherited by all services. - - `arguments/arguments.py` – Handles provider-specific argument parsing. - - `mutelist/mutelist.py` – Manages the mutelist functionality for the provider. - -- `services` – Stores all [services](./services.md) that the provider offers and want to be audited by [Prowler checks](./checks.md). - -- `__init__.py` (empty) – Ensures Python recognizes this folder as a package. - -- `_provider.py` – Defines authentication logic, configurations, and other provider-specific data. - -- `models.py` – Contains necessary models for the new provider. - -By adhering to this structure, Prowler can effectively support services and security checks for additional providers. - -???+ important - If your new provider requires a Python library (such as an official SDK or API client) to connect to its services, make sure to add it as a dependency in the `pyproject.toml` file. This ensures that all contributors and users have the necessary packages installed when working with your provider. - -## Provider Structure in Prowler - -Prowler's provider architecture is designed to facilitate security audits through a generic service tailored to each provider. This is accomplished by passing the necessary parameters to the constructor, which initializes all required session values. - -### Base Class - -All Prowler providers inherit from the same base class located in [`prowler/providers/common/provider.py`](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/common/provider.py). It is an [abstract base class](https://docs.python.org/3/library/abc.html) that defines the interface for all provider classes. - -### Provider Class - -#### Provider Implementation Guidance - -Given the complexity and variability of providers, use existing provider implementations as templates when developing new integrations. - -- [AWS](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/aws/aws_provider.py) -- [GCP](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/gcp/gcp_provider.py) -- [Azure](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/azure/azure_provider.py) -- [Kubernetes](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/kubernetes/kubernetes_provider.py) -- [Microsoft365](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/microsoft365/microsoft365_provider.py) -- [GitHub](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/github/github_provider.py) -- [MongoDB Atlas](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/mongodbatlas/mongodbatlas_provider.py) - -### Basic Provider Implementation: Pseudocode Example - -To simplify understanding, the following pseudocode outlines the fundamental structure of a provider, including library imports necessary for authentication. - -```python title="Provider Example Class" - -# Library Imports for Authentication - -# When implementing authentication for a provider, import the required libraries. - -from prowler.config.config import load_and_validate_config_file -from prowler.lib.logger import logger -from prowler.lib.mutelist.mutelist import parse_mutelist_file -from prowler.lib.utils.utils import print_boxes -from prowler.providers.common.models import Audit_Metadata -from prowler.providers.common.provider import Provider -from prowler.providers..models import ( - # All provider models needed. - ProviderSessionModel, - ProviderIdentityModel, - ProviderOutputOptionsModel -) - -class NewProvider(Provider): - # All properties from the class, some of which are properties in the base class. - _type: str = "" - _session: - _identity: - _audit_config: dict - _output_options: ProviderOutputOptionsModel - _mutelist: dict - audit_metadata: Audit_Metadata - - def __init__(self, arguments): - """ - Initializes the NewProvider instance. - Args: - arguments (dict): A dictionary containing configuration arguments. - """ - logger.info("Setting provider ...") - - # Initializing the Provider Session - - # Steps: - - # - Retrieve Account Information - # - Extract relevant account identifiers (subscriptions, projects, or other service references) from the provided arguments. - - # Establish a Session - - # Use the method enforced by the parent class to set up the session: - self._session = self.setup_session(credentials_file) - - # Define Provider Identity - # Assign the identity class, typically provided by the Python provider library: - self._identity = () - - # Configure the Provider - # Set the provider-specific configuration. - self._audit_config = load_and_validate_config_file( - self._type, arguments.config_file - ) - - # All the enforced properties by the parent class. - @property - def identity(self): - return self._identity - - @property - def session(self): - return self._session - - @property - def type(self): - return self._type - - @property - def audit_config(self): - return self._audit_config - - @property - def output_options(self): - return self._output_options - - def setup_session(self, ): - """ - Sets up the Provider session. - - Args: - Can include all necessary arguments to set up the session - - Returns: - Credentials necessary to communicate with the provider. - """ - pass - - """ - This method is enforced by parent class and is used to print all relevant - information during the prowler execution as a header of execution. - Displaying Account Information with Color Formatting. In Prowler, Account IDs, usernames, and other identifiers are typically displayed using color formatting provided by the colorama module (Fore). - """ - def print_credentials(self): - pass -``` diff --git a/docs/developer-guide/provider.mdx b/docs/developer-guide/provider.mdx new file mode 100644 index 0000000000..85dafc6300 --- /dev/null +++ b/docs/developer-guide/provider.mdx @@ -0,0 +1,3458 @@ +--- +title: 'Prowler Providers' +--- + +## Introduction + +Providers form the backbone of Prowler, enabling security assessments across various cloud environments. + +A provider is any platform or service that offers resources, data, or functionality that can be audited for security and compliance. This includes: + +- Cloud Infrastructure Providers (like Amazon Web Services, Microsoft Azure, and Google Cloud) +- Software as a Service (SaaS) Platforms (like Microsoft 365) +- Development Platforms (like GitHub) +- Container Orchestration Platforms (like Kubernetes) +- Database-as-a-Service Platforms (like MongoDB Atlas) + +For providers supported by Prowler, refer to [Prowler Hub](https://hub.prowler.com/). + + + There are some custom providers added by the community, like [NHN Cloud](https://www.nhncloud.com/), that are not maintained by the Prowler team, but can be used in the Prowler CLI. The main purpose of this documentation is to guide you through creating a new provider and integrating it not only in the CLI, but also in the API and UI. Non official providers can be checked directly at the [Prowler GitHub repository](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). + +--- + +## Provider Types in Prowler + +Prowler supports several types of providers, each with its own implementation pattern and use case. Understanding these differences is key to designing your provider correctly. + +### Classifying your Provider + +Before implementing a new provider, you need to determine which type it belongs to. This classification will guide your implementation approach and help you choose the right patterns and libraries. + +#### Decision Criteria + +Once you have decided the provider you want or need to add to Prowler, the next step is to study how to retrieve data from it. Based on that, the provider will fall into one of the following types: SDK, API or Tool/Wrapper (maybe in the future there will be new types but for now this are the only ones). + +**Choose SDK Provider if:** + +- The target platform/service has an **official Python SDK** available +- The target platform/service has a **non-official Python SDK** available but it's been updated and maintained +- You need to support **multiple authentication methods** (profiles, service principals, IAM roles, etc.) +- The SDK provides **built-in session management**, retry logic, and error handling +- You want to leverage **SDK-specific features** like credential chaining, role assumption, etc +- The platform is a **major cloud provider** (AWS, Azure, GCP, etc.) or has mature SDK support + +**Choose API Provider if:** + +- The target platform has a **REST API** but **no official Python SDK** +- The target platform has a **non-official Python SDK** available but it's not updated and maintained +- You need to implement **custom authentication flows** (OAuth, token-based, etc.) +- The platform is a **custom or community service** without official SDK support +- You want to use **standard HTTP libraries** like `requests` for API calls +- The platform exposes **well-documented REST endpoints** but lacks SDK tooling + +**Choose Tool/Wrapper Provider if:** + +- You're integrating a **third-party security tool** or library +- The tool provides **scanning capabilities** that need to be adapted to Prowler's interface +- You don't need **authentication or session management** (the tool handles this) +- You need to **map tool arguments** and **convert outputs** to Prowler's format + +**Special Case - Hybrid Providers:** + +- Some providers may **combine multiple approaches** (e.g., SDK + Tool wrapper, SDK + API, etc.) +- Example: M365 uses **msgraph SDK** for authentication and some checks, and **PowerShell wrapper** for other checks that the SDK doesn't support +- These require **custom implementation patterns** that blend different provider types + +#### Classification Examples + +| Provider | Type | Reasoning | +| ----------- | ------ | ----------------------------------------------------------------- | +| AWS | SDK | Official boto3 SDK, multiple auth methods, mature ecosystem | +| Azure | SDK | Official azure-identity SDK, service principals, managed identity | +| GCP | SDK | Official google-auth SDK, service accounts, ADC support | +| Kubernetes | SDK | Official kubernetes SDK, service accounts, ADC support | +| NHN Cloud | API | Custom REST API, no official SDK, community provider | +| MongoDB Atlas| API | Custom REST API, no official SDK | +| IAC | Tool | Third-party security tool that uses trivy, no auth needed, output conversion| +| M365 | Hybrid | Combines msgraph SDK for auth + PowerShell wrapper for operations | +| GitHub | Hybrid | Non-Official PyGithub SDK but it's been updated and maintained + Official graphql API requests| + +#### Questions to Ask Yourself + +**1. Does the platform have an official Python SDK?** + +- Yes → Consider SDK Provider +- No → Continue to question 2 + +**2. Does the platform have a non-official Python SDK?** + +- Yes → Then if the SDK is updated and maintained, consider SDK Provider, otherwise continue to question 3. +- No → Continue to question 3 + +**3. Is this a third-party security tool or library?** + +- Yes → Consider Tool/Wrapper Provider +- No → Continue to question 4 + +**4. Does the platform expose a REST API?** + +- Yes → Consider API Provider +- No → You may need a custom approach + +![FlowChart Decision](../img/provider-decision-tree.png) + +#### Implementation Complexity + +- **SDK Providers**: Low complexity. You have mature examples like AWS, Azure, GCP, Kubernetes, etc. that you can leverage to implement your provider. +- **API Providers**: Medium complexity. You need to implement the authentication and session management, and the API calls to the provider. You now have NHN and MongoDB Atlas as example to follow. +- **Tool/Wrapper Providers**: High complexity. You need to implement the argument/output mapping to the provider and handle problems that the tool/wrapper may have. You now have IAC and the PowerShell wrapper as example to follow. +- **Hybrid Providers**: High complexity. You need to "customize" your provider, mixing the other types of providers in order to achieve the desired result. You have M365 (msgraph SDK + PowerShell wrapper) and Github (PyGithub SDK + graphql API requests) as examples. + +### Determining Regional vs Non-Regional Architecture + +After classifying your provider type, the next critical decision is determining whether your provider operates with **regional concepts** or is **global/non-regional**. This decision fundamentally affects how your provider and services are structured and executed. + +#### Regional Providers + +Regional providers operate across multiple geographic locations and require region-specific resource discovery and iteration. + +**Examples:** + +- **AWS**: Has regions like `us-east-1`, `eu-west-1`, `ap-southeast-2` +- **Azure**: Has regions like `East US`, `West Europe`, `Australia East` +- **GCP**: Has regions like `us-central1`, `europe-west1`, `asia-southeast1` + +**Implementation Requirements:** + +- Must implement region discovery and iteration +- Services must be instantiated per region or handle multi-region data +- Checks must execute across all available/specified regions +- Resource ARNs/IDs must include region information +- Region-specific client initialization + +**Execution Pattern:** + +```python +# Regional provider execution pattern +for region in provider.get_regions(): + regional_client = service.get_regional_client(region) + regional_resources = regional_client.discover_resources() + # Process regional resources +``` + +#### Non-Regional (Global) Providers + +Non-regional providers operate globally without geographic partitioning. + +**Examples:** + +- **GitHub**: Repositories, organizations are global concepts +- **M365**: Tenants operate globally across Microsoft datacenters +- **Kubernetes**: Clusters are independent units without regional concepts + +**Implementation Requirements:** + +- Single global client/session +- No region iteration required +- Global resource discovery +- Simpler resource identification (no region in ARNs/IDs) +- Single audit execution + +**Execution Pattern:** + +```python +# Non-regional provider execution pattern +global_client = service.get_client() +global_resources = global_client.discover_resources() +# Process all resources in single iteration +``` + +#### Decision Matrix + +| Aspect | Regional Provider | Non-Regional Provider | +| ------------------------ | ------------------------ | ---------------------- | +| **Client Init** | Per-region clients | Single global client | +| **Resource Discovery** | Iterate through regions | Single discovery call | +| **Resource ARN/ID** | Include region | Global identifier/None | +| **Audit Execution** | Multi-region loops | Single execution | +| **Service Architecture** | Region-aware services | Global services | +| **Performance** | Parallelizable by region | Linear execution | + +#### Region Discovery + +Region discovery is the process of getting the list of regions that are available for the account. This is done by the provider and is stored in the `prowler/providers//lib/regions/_regions.py` file. + +```python +# File: prowler/providers/aws/aws_provider.py +def get_aws_enabled_regions(self, current_session: Session) -> set: + """get_aws_enabled_regions returns a set of enabled AWS regions""" + try: + # EC2 Client to check enabled regions + service = "ec2" + default_region = self.get_default_region(service) + ec2_client = current_session.client(service, region_name=default_region) + + enabled_regions = set() + # With AllRegions=False we only get the enabled regions for the account + for region in ec2_client.describe_regions(AllRegions=False).get("Regions", []): + enabled_regions.add(region.get("RegionName")) + + return enabled_regions + except Exception as error: + logger.error(f"{error.__class__.__name__}: {error}") + return set() +``` + +The function returns a JSON file containing the list of regions for the provider. It is used to retrieve the provider’s regions and to validate the region specified by the user. + +```json +# File: prowler/providers/aws/aws_regions_by_service.json (extract) +{ + "services": { + "ec2": { + "regions": { + "aws": [ + "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", + "ap-northeast-3", "ap-south-1", "ap-southeast-1", "ap-southeast-2", + "ca-central-1", "eu-central-1", "eu-north-1", "eu-south-1", + "eu-west-1", "eu-west-2", "eu-west-3", "me-south-1", + "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2" + ], + "aws-cn": ["cn-north-1", "cn-northwest-1"], + "aws-us-gov": ["us-gov-east-1", "us-gov-west-1"] + } + } + } +} +``` + +### Regional Service Implementation + +For detailed guidance on implementing services for regional services, including code examples, service architecture, and check execution patterns, see the [Regional Service Implementation](./services#regional-service-implementation) section in the Services documentation. + +**Key concepts covered:** + +- Threading and parallel processing across regions +- Service implementation patterns for regional providers +- Cross-region resource attribution and ARN handling +- Best practices for performance and error isolation + +## Step 1: Create the Provider Backend (CLI Integration) + +Once the type of provider and its regional architecture are determined, the next step is to start creating the code of the provider. + +### SDK Providers + +General aspects to consider when implementing a new SDK provider: + +**Definition:** + +- Use the official SDK of the provider to interact with its resources and APIs. +- Examples: AWS (boto3), Azure (azure-identity), GCP (google-auth), Kubernetes (kubernetes), M365 (msal/msgraph), GitHub (PyGithub). + +**Typical Use Cases:** + +- Cloud platforms and services with mature Python SDKs. +- Need to support multiple authentication methods (profiles, service principals, etc). +- Providers that offer comprehensive Python libraries for resource management. + +**Key Characteristics:** + +- Authentication and session management handled by the SDK. +- Arguments: Depends on the provider, but for example we can have `profile`, `region`, `tenant_id`, `client_id`, `client_secret`, etc. +- Outputs: Standardized via SDK models and responses. +- Session objects that can be reused across multiple API calls. +- Built-in retry logic and error handling. + +**Implementation Details:** + +- SDK providers typically use credential objects or session objects provided by the official SDK. +- They often support multiple authentication methods (several types of credentials, configuration files, IAM roles, etc.). +- Session management includes token refresh, connection pooling, and retry mechanisms. +- Resource discovery and enumeration is usually straightforward through SDK methods. + +--- + +### Implementation Guide for SDK Providers + +Now it's time to start creating the code needed to implement the provider. + +#### Step 1: Create the Provider Structure + +**Explanation:** +SDK providers require a specific folder structure to organize authentication, configuration, and service management. This structure follows Prowler's conventions and ensures proper integration with the CLI and API. + +**Required Structure:** + +``` +prowler/providers// +├── __init__.py +├── _provider.py +├── models.py +├── exceptions/ +│ ├── __init__.py +│ └── exceptions.py +├── services/ +│ ├── service_name1/ +│ └── service_name2/ +└── lib/ + ├── __init__.py + ├── arguments/ + │ ├── __init__.py + │ └── arguments.py + ├── mutelist/ + │ ├── __init__.py + │ └── mutelist.py + ├── regions/ + │ ├── __init__.py + │ └── _regions.py + └── service/ + ├── __init__.py + └── service.py +``` + +**Key Components:** + +- **`_provider.py`**: Main provider class with authentication and session management +- **`models.py`**: Data structures for identity, session, and provider-specific information +- **`exceptions/`**: Custom exception classes for error handling +- **`services/`**: Folder that contains all the provider services, how to make a new service is explained in another section. +- **`lib/arguments/`**: CLI argument validation and parsing +- **`lib/mutelist/`**: Resource exclusion and muting functionality +- **`lib/regions/`**: Region management and validation. If the provider is NOT regional, this folder will not be created. +- **`lib/service/`**: Base service class for provider-specific services + +#### Step 2: Implement the Provider Class + +**Explanation:** +The provider class is the core component that handles authentication, session management, and identity information. It inherits from Prowler's base Provider class and implements SDK-specific authentication flows. All providers must share, as far as possible, common patterns for session setup, identity management, and credential validation. + +Nevertheless, you may encounter changes and must adapt the implementation logic accordingly. A basic example of a common provider implementation is the following: + +**File:** `prowler/providers//_provider.py` + +```python +import os +from typing import Optional, Union +from prowler.providers.common.provider import Provider +from prowler.providers.common.models import Audit_Metadata, Connection +from prowler.config.config import load_and_validate_config_file, get_default_mute_file_path +from prowler.lib.logger import logger +from prowler.lib.utils.utils import print_boxes + +# Import your SDK and all the needed libraries for the provider. +import your_sdk_library +from your_sdk_library.auth_methods import ClientSecretCredential, ProfileCredential, DefaultCredential + +# Import the needed exceptions, mutelist and models for the provider. +from prowler.providers..exceptions.exceptions import Exceptions +from prowler.providers..mutelist.mutelist import Mutelist +from prowler.providers..models import NeededModels + +class YourProvider(Provider): + """ + YourProvider class is the main class for the Your Provider. + + This class is responsible for initializing the provider, setting up the session, + validating credentials, and managing identity information. + + Attributes: + _type (str): The provider type. + _session (YourSDKSession): The provider session. + _identity (YourProviderIdentityInfo): The provider identity information. + _audit_config (dict): The audit configuration. + _mutelist (YourProviderMutelist): The provider mutelist. + audit_metadata (Audit_Metadata): The audit metadata. + """ + + _type: str = "your_provider" + _session: your_sdk_library.Session + _identity: YourProviderIdentityInfo + _audit_config: dict + _mutelist: YourProviderMutelist + audit_metadata: Audit_Metadata + + def __init__( + self, + # Authentication parameters + client_id: str = None, + client_secret: str = None, + tenant_id: str = None, + # Configuration + config_path: str = None, + config_content: dict = None, + mutelist_path: str = None, + mutelist_content: dict = None, + # Additional provider-specific parameters + region: str = None, + profile: str = None, + ): + """ + Initializes the YourProvider instance. + + Args: + client_id: The client ID for authentication + client_secret: The client secret for authentication + tenant_id: The tenant ID for authentication + config_path: Path to the configuration file + config_content: Configuration content as dictionary + mutelist_path: Path to the mutelist file + mutelist_content: Mutelist content as dictionary + region: The region to use + profile: The profile to use + + Raises: + YourProviderSetUpSessionError: If session setup fails + YourProviderInvalidCredentialsError: If credentials are invalid + """ + logger.info("Initializing YourProvider ...") + + # Setup session using SDK + self._session = self.setup_session( + client_id, client_secret, tenant_id, region, profile + ) + + # Get identity information + self._identity = self.setup_identity(self._session) + + # Load configuration + if config_content: + self._audit_config = config_content + else: + if not config_path: + config_path = default_config_file_path + self._audit_config = load_and_validate_config_file(self._type, config_path) + + # Setup mutelist + if mutelist_content: + self._mutelist = YourProviderMutelist(mutelist_content=mutelist_content) + else: + if not mutelist_path: + mutelist_path = get_default_mute_file_path(self._type) + self._mutelist = YourProviderMutelist(mutelist_path=mutelist_path) + + Provider.set_global_provider(self) + + @staticmethod + def setup_session( + client_id: str = None, + client_secret: str = None, + tenant_id: str = None, + region: str = None, + profile: str = None, + ) -> your_sdk_library.Session: + """ + Sets up the provider session using the provided credentials. + + This method handles the authentication flow and creates a session object + that can be used to interact with the provider's services. + + Args: + client_id: The client ID for authentication + client_secret: The client secret for authentication + tenant_id: The tenant ID for authentication + region: The region to use + profile: The profile to use + + Returns: + YourSDKSession: The authenticated session object + + Raises: + YourProviderSetUpSessionError: If session setup fails + """ + try: + logger.debug("Creating session ...") + + # Determine authentication method based on provided parameters + if client_id and client_secret and tenant_id: + # Use client credentials authentication + credentials = your_sdk_library.ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret + ) + auth_method = "Client Credentials" + elif profile: + # Use profile-based authentication + credentials = your_sdk_library.ProfileCredential(profile=profile) + auth_method = "Profile" + else: + # Use default authentication (environment variables, etc.) + credentials = your_sdk_library.DefaultCredential() + auth_method = "Default" + + # Create session with credentials + session = your_sdk_library.Session( + credentials=credentials, + region=region + ) + + logger.debug(f"Session created using {auth_method} authentication") + return session + + except Exception as error: + logger.critical(f"Failed to setup session: {error}") + raise YourProviderSetUpSessionError( + original_exception=error, + file=os.path.basename(__file__), + ) + + def setup_identity(self, session: your_sdk_library.Session) -> YourProviderIdentityInfo: + """ + Gets identity information from the provider session. + + This method retrieves account information, user details, and other + identity-related data from the provider. + + Args: + session: The authenticated session object + + Returns: + YourProviderIdentityInfo: The identity information + + Raises: + YourProviderSetUpIdentityError: If identity setup fails + """ + try: + # Use SDK to get account/identity information + identity_info = session.get_identity() + + return YourProviderIdentityInfo( + account_id=identity_info.account_id, + account_name=identity_info.account_name, + region=identity_info.region, + user_id=identity_info.user_id, + # Add other identity fields as needed + ) + except Exception as e: + logger.error(f"Failed to get identity information: {e}") + raise YourProviderSetUpIdentityError( + original_exception=e, + file=os.path.basename(__file__), + ) + + @property + def identity(self): + """Returns the provider identity information.""" + return self._identity + + @property + def session(self): + """Returns the provider session object.""" + return self._session + + @property + def type(self): + """Returns the provider type.""" + return self._type + + @property + def audit_config(self): + """Returns the audit configuration.""" + return self._audit_config + + @property + def mutelist(self): + """Returns the provider mutelist.""" + return self._mutelist + + def print_credentials(self): + """ + Display account information with color formatting. + + This method prints the provider credentials and account information + in a formatted way using colorama for better readability. + """ + from colorama import Fore, Style + from prowler.lib.utils.utils import print_boxes + + report_lines = [ + f" Account ID: {Fore.YELLOW}{self._identity.account_id}{Style.RESET_ALL}", + f" Account Name: {Fore.YELLOW}{self._identity.account_name}{Style.RESET_ALL}", + f" Region: {Fore.YELLOW}{self._identity.region}{Style.RESET_ALL}", + f" User ID: {Fore.YELLOW}{self._identity.user_id}{Style.RESET_ALL}", + ] + report_title = f"{Style.BRIGHT}Using the {self._type.upper()} credentials below:{Style.RESET_ALL}" + print_boxes(report_lines, report_title) + + @staticmethod + def test_connection( + client_id: str = None, + client_secret: str = None, + tenant_id: str = None, + region: str = None, + profile: str = None, + raise_on_exception: bool = True, + provider_id: str = None, + ) -> Connection: + """ + Test connection to the provider. + + This method validates the provided credentials and tests the connection + to the provider's services. + + Args: + client_id: The client ID for authentication + client_secret: The client secret for authentication + tenant_id: The tenant ID for authentication + region: The region to test + profile: The profile to use + raise_on_exception: Whether to raise exceptions or return Connection object + provider_id: The provider ID to validate against + + Returns: + Connection: Connection test result + + Raises: + YourProviderSetUpSessionError: If session setup fails + YourProviderInvalidCredentialsError: If credentials are invalid + """ + try: + # Create temporary session for testing + test_session = YourProvider.setup_session( + client_id, client_secret, tenant_id, region, profile + ) + + # Test the connection by getting identity + identity = YourProvider.setup_identity(test_session) + + # Validate provider ID if provided + if provider_id and identity.account_id != provider_id: + raise YourProviderInvalidProviderIdError( + file=os.path.basename(__file__), + ) + + return Connection( + status=True, + message=f"Successfully connected to {provider_id or 'provider'}", + error=None, + ) + except Exception as e: + if raise_on_exception: + raise e + return Connection( + status=False, + message="Failed to connect", + error=str(e), + ) + + def get_regions(self) -> set: + """ + Get available regions for the provider. + + Returns: + set: Set of available region names + """ + # Implementation depends on your provider + # Example for cloud providers that support regions + return {"region1", "region2", "region3"} + + def get_services(self) -> list: + """ + Get available services for the provider. + + Returns: + list: List of available service names + """ + # Implementation depends on your provider + return ["service1", "service2", "service3"] +``` + +#### Step 3: Create Models + +**Explanation:** +Models define the data structures used by your provider. They include identity information, session details, and provider-specific configurations. These models ensure type safety and consistent data handling across the provider. + +**File:** `prowler/providers//models.py` + +```python +# Import the needed generic libraries for the provider. +from pydantic import BaseModel +from dataclasses import dataclass +from typing import Optional, List + +# Import the needed Prowler libraries for the provider. +from prowler.providers.common.models import ProviderOutputOptions +from prowler.config.config import output_file_timestamp + +class YourProviderIdentityInfo: + """ + Identity information for the provider. + + This class holds all the identity-related information retrieved + from the provider, including account details and user information. + """ + account_id: str + account_name: str + region: str + user_id: str + # Add other identity fields as needed + +class YourProviderSession: + """ + Session object that contains the credentials and authentication details for the provider. + + This class holds the actual credentials and authentication information needed + to establish a connection with the provider's services. + """ + # Authentication credentials + access_key: str + secret_key: str + # Or for other providers: + # client_id: str + # client_secret: str + # tenant_id: str + + # Connection details + region: str +``` + +#### Step 4: Implement Arguments + +**Explanation:** +Argument validation ensures that the provider receives valid configuration parameters. This step is crucial for preventing runtime errors and providing clear error messages to users. The validation should check for required parameters and validate their format. + +**File:** `prowler/providers//lib/arguments/arguments.py` + +```python +def init_parser(self): + """Init the Provider CLI parser""" + _parser = self.subparsers.add_parser( + "", parents=[self.common_providers_parser], help=" Provider" + ) + # Authentication Modes + _auth_subparser = _parser.add_argument_group("Authentication Modes") + _auth_modes_group = _auth_subparser.add_mutually_exclusive_group() + _auth_modes_group.add_argument( + "--credentials-file", + nargs="?", + metavar="FILE_PATH", + help="Authenticate using a Service Account Application Credentials JSON file", + ) + _auth_modes_group.add_argument( + "--impersonate-service-account", + nargs="?", + metavar="SERVICE_ACCOUNT", + help="Impersonate a Service Account", + ) + _parser.add_argument( + "--your-provider-region", + help="Your Provider Region", + type=str, + ) + _parser.add_argument( + "--env-auth", + action="store_true", + help="Use User and Password environment variables authentication to log in against ", + ) + # More arguments for the provider. +``` + +#### Step 5: Implement Mutelist + +**Explanation:** +The mutelist functionality allows users to exclude specific resources or checks from the audit. This is useful for handling false positives or excluding resources that are intentionally configured differently. + +**File:** `prowler/providers//lib/mutelist/mutelist.py` + +```python +from prowler.lib.mutelist.mutelist import Mutelist +from prowler.lib.check.models import CheckReportYourProvider + +class YourProviderMutelist(Mutelist): + """ + Mutelist implementation for YourProvider. + + This class handles the muting functionality for the provider, + allowing users to exclude specific checks or resources from audits. + """ + + def is_finding_muted(self, finding: CheckReportYourProvider) -> bool: + """ + Check if a specific finding is muted. + + Args: + finding: The finding to check + """ + return self.is_muted(finding.check_id, finding.resource_id) +``` + +#### Step 6: Implement Regions + +**Explanation:** +Region management is essential for cloud providers that operate across multiple geographic locations. This component handles region validation and provides region-specific functionality. + + +Regions are optional, only if the provider has regions, for example Github does not have regions, but AWS does. + + +**File:** `prowler/providers//lib/regions/_regions.py` + +```python +from typing import List, Set + +def get_regions() -> List[str]: + """ + Get list of available regions for the provider. + + Returns: + List[str]: List of available region names + """ + return [ + "region1", + "region2", + "region3", + # ... other regions + ] + +def validate_region(region: str) -> bool: + """ + Validate if a region is supported. + + Args: + region: The region to validate + + Returns: + bool: True if the region is valid, False otherwise + """ + return region in get_regions() + +def get_default_region() -> str: + """ + Get the default region for the provider. + + Returns: + str: The default region name + """ + return "region1" + +def get_global_region() -> str: + """ + Get the global region for the provider. + + Returns: + str: The global region name + """ + return "global" +``` + +#### Step 7: Create Custom Exceptions + +**Explanation:** +Custom exceptions are needed to be able to handle the errors in a more specific way. Prowler uses a structured exception system with error codes, messages, and remediation steps. + +**File:** `prowler/providers//exceptions.py` + +```python +from prowler.exceptions.exceptions import ProwlerException + + +# Exceptions codes from 7000 to 7999 are reserved for YourProvider exceptions (Numbers as example) +class YourProviderBaseException(ProwlerException): + """Base class for YourProvider Errors.""" + + YOUR_PROVIDER_ERROR_CODES = { + (7001, "YourProviderCredentialsError"): { + "message": "Error loading credentials for YourProvider", + "remediation": "Check the credentials and ensure they are properly set up. API_KEY and API_SECRET are required.", + }, + (7002, "YourProviderAuthenticationError"): { + "message": "Authentication failed with YourProvider", + "remediation": "Check the API credentials and ensure they are valid and have proper permissions.", + }, + (7003, "YourProviderInvalidRegionError"): { + "message": "Invalid region provided for YourProvider", + "remediation": "Check the region and ensure it is a valid region for YourProvider.", + }, + (7004, "YourProviderSetUpSessionError"): { + "message": "Error setting up session", + "remediation": "Check the session setup and ensure it is properly configured.", + }, + (7005, "YourProviderInvalidProviderIdError"): { + "message": "Provider does not match with the expected account_id", + "remediation": "Check the provider and ensure it matches the expected account_id.", + }, + } + + def __init__(self, code, file=None, original_exception=None, message=None): + provider = "YourProvider" + error_info = self.YOUR_PROVIDER_ERROR_CODES.get((code, self.__class__.__name__)) + if message: + error_info["message"] = message + super().__init__( + code=code, + source=provider, + file=file, + original_exception=original_exception, + error_info=error_info, + ) + + +class YourProviderCredentialsError(YourProviderBaseException): + """Base class for YourProvider credentials errors.""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7001, file=file, original_exception=original_exception, message=message + ) + + +class YourProviderAuthenticationError(YourProviderCredentialsError): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7002, file=file, original_exception=original_exception, message=message + ) + + +class YourProviderInvalidRegionError(YourProviderBaseException): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7003, file=file, original_exception=original_exception, message=message + ) + + +class YourProviderSetUpSessionError(YourProviderCredentialsError): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7004, file=file, original_exception=original_exception, message=message + ) + + +class YourProviderInvalidProviderIdError(YourProviderBaseException): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 7005, file=file, original_exception=original_exception, message=message + ) +``` + +#### Step 8: Implement Service Base Class + +**Explanation:** +The service base class defines a common interface for all services in your provider, since they will inherit from it. It defines the client to make requests to, the audit configuration and the fixer configuration. + +**File:** `prowler/providers//lib/service/service.py` + +```python +from prowler.providers.._provider import Provider + +class YourProviderService(BaseService): + """ + Base service class for YourProvider services. + + This class provides common functionality for all services + within the provider, including session management and error handling. + """ + + def __init__(self, provider: Provider): + """ + Initialize the service. + + Args: + provider: The provider instance + """ + self.client = provider.session.get_client(self.service_name) + self.audit_config = provider.audit_config + self.fixer_config = provider.fixer_config +``` + +#### Step 9: Register in CLI + +**Explanation:** +Add your provider to the available providers in the CLI. + +**File:** `prowler/lib/cli/parser.py` + +```python +class ProwlerArgumentParser: + # Set the default parser + def __init__(self): + # CLI Arguments + self.parser = argparse.ArgumentParser( + prog="prowler", + formatter_class=RawTextHelpFormatter, + usage="prowler [-h] [--version] {aws,azure,gcp,kubernetes,m365,github,nhn,dashboard,iac,your_provider} ...", + epilog=""" +Available Cloud Providers: + {aws,azure,gcp,kubernetes,m365,github,iac,nhn,your_provider} + aws AWS Provider + azure Azure Provider + gcp GCP Provider + kubernetes Kubernetes Provider + m365 Microsoft 365 Provider + github GitHub Provider + iac IaC Provider (Preview) + nhn NHN Provider (Unofficial) + your_provider Your Provider + +Available components: + dashboard Local dashboard + +To see the different available options on a specific component, run: + prowler {provider|dashboard} -h|--help + +Detailed documentation at https://docs.prowler.com +""", +``` + +#### Step 10: Register in Main + +**Explanation:** +Main registration makes your provider discoverable by Prowler's core system. It's needed to add your provider to the output options and to the compliance evaluation. + +**File:** `prowler/__main__.py` + +```python +# In the prowler setup output options section + if provider == "aws": + output_options = AWSOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "azure": + output_options = AzureOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "gcp": + output_options = GCPOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "kubernetes": + output_options = KubernetesOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "github": + output_options = GithubOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "m365": + output_options = M365OutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "nhn": + output_options = NHNOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + elif provider == "iac": + output_options = IACOutputOptions( + args, bulk_checks_metadata + ) + elif provider == "your_provider": + output_options = YourProviderOutputOptions( + args, bulk_checks_metadata, global_provider.identity + ) + + + # Setup Compliance Options + elif provider == "your_provider": + for compliance_name in input_compliance_frameworks: + if compliance_name.startswith("cis_"): + # Generate CIS Finding Object (example of compliance with CIS framework) + filename = ( + f"{output_options.output_directory}/compliance/" + f"{output_options.output_filename}_{compliance_name}.csv" + ) + cis = YourProviderCIS( + findings=finding_outputs, + compliance=bulk_compliance_frameworks[compliance_name], + file_path=filename, + ) + generated_outputs["compliance"].append(cis) + cis.batch_write_data_to_file() +``` + +#### Step 11: Register in the list of providers + +**Explanation:** +This is needed to be able to use the provider in the generic checks. The provider must be registered in the `init_global_provider` method to handle CLI arguments and initialization. + +**File:** `prowler/providers/common/provider.py` + +```python +elif "your_provider" in provider_class_name.lower(): + provider_class( + username=arguments.your_provider_username, + password=arguments.your_provider_password, + tenant_id=arguments.your_provider_tenant_id, + config_path=arguments.config_file, + mutelist_path=arguments.mutelist_file, + fixer_config=fixer_config, + ) +``` + +#### Step 12: Add to Config + +**Explanation:** +Configuration registration ensures your provider is recognized by Prowler's configuration system. This enables proper handling of provider-specific settings and defaults. + +**File:** `prowler/config/config.py` + +```python +class Provider(str, Enum): + AWS = "aws" + AZURE = "azure" + GCP = "gcp" + KUBERNETES = "kubernetes" + M365 = "m365" + GITHUB = "github" + YOUR_PROVIDER = "your_provider" # Add your provider here +``` + +In some cases, you may need to create a new configuration file for your provider, for example, the AWS one that is inside `prowler/providers/aws/config.py`. + +#### Step 13: Create Compliance Files + +**Explanation:** +Compliance files define the security checks and standards that your provider supports. These JSON files map security controls to specific checks and provide remediation guidance. It's needed to create the folder with an init file to ensure the provider will work, however, adding different compliance files is optional. + +**Folder:** `prowler/compliance//` + +```json +{ + "Framework": "CIS", + "Version": "1.0", + "Provider": "your_provider", + "Description": "Description of the compliance framework", + # The requirements depends on the framework, for example, CIS has a requirements section with the checks and attributes. + "Requirements": [ + { + "Id": "1.1.1", + "Description": "Description of the requirement", + "Checks": ["your_provider_check_1", "your_provider_check_2"], + "Attributes": [] + } + ] +} +``` + +#### Step 14: Add Output Support + +**Explanation:** +Output support ensures that your provider's results are properly formatted in Prowler's various output formats (CSV, JSON, HTML, etc.). This step integrates your provider with Prowler's reporting system. + +**File:** `prowler/lib/outputs/summary_table.py` + +```python +# Add your provider case in the display_summary_table function +elif provider.type == "your_provider": + entity_type = "Your Entity Type" + audited_entities = provider.identity.your_entity_field +``` + +**File:** `prowler/lib/outputs/finding.py` + +```python +# Add your provider case in the fill_common_finding_data function +elif provider.type == "your_provider": + output_data["auth_method"] = f"Your Auth Method: {get_nested_attribute(provider, 'identity.auth_type')}" + output_data["account_uid"] = get_nested_attribute(provider, "identity.account_id") + output_data["account_name"] = get_nested_attribute(provider, "identity.account_name") + output_data["resource_name"] = check_output.resource_name + output_data["resource_uid"] = check_output.resource_id + output_data["region"] = check_output.location # or your location field +``` + +**File:** `prowler/lib/outputs/outputs.py` + +```python +# Add your provider case in the stdout_report function +if finding.check_metadata.Provider == "your_provider": + details = finding.your_location_field # e.g., finding.location, finding.namespace, etc. +``` + +#### Step 15: Generate the HTML Report + +**Explanation:** +The HTML file is needed to be able to generate the HTML report. This step involves adding support for your provider in the HTML output generation system to ensure proper display of assessment summaries and findings. + +**File:** `prowler/lib/outputs/html/html.py` + +```python +@staticmethod +def get_your_provider_assessment_summary(provider: Provider) -> str: + """ + get_your_provider_assessment_summary gets the HTML assessment summary for your provider + + Args: + provider (Provider): the provider object + + Returns: + str: the HTML assessment summary + """ + try: + return f""" +
+
+
+ Your Provider Assessment Summary +
+
    +
  • + Your Entity Type: {provider.identity.your_entity_field} +
    • +
  • + Your Location Field: {provider.identity.your_location_field} +
    • +
+
+
+
+
+
+ Your Provider Credentials +
+
    +
  • + Authentication Method: {provider.auth_method} +
    • +
  • + Identity ID: {provider.identity.identity_id} +
    • +
+
+
""" + except Exception as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}] -- {error}" + ) + return "" +``` + +#### Step 16: Add the Check Report Model + +**Explanation:** +Add the provider to the generic models, this is needed to be able to use the provider in the generic checks. + +**File:** `prowler/providers/check/models.py` + +```python +@dataclass +class CheckReportYourProvider(CheckReport): + """ + Check report for YourProvider. + """ + resource_name: str + resource_id: str + + def _init_(self, metadata: Dict, resource: Any) -> None: + super()._init_(metadata, resource) + self.resource_name = resource.name + self.resource_id = resource.id +``` + +#### Step 17: Add Dependencies + +**Explanation:** +Dependencies ensure that your provider's required libraries are available when Prowler is installed. This step adds the necessary SDK or API client to Prowler's dependency management. + +**File:** `pyproject.toml` + +```toml +[tool.poetry.dependencies] +python = "^3.9" +# ... other dependencies +your-sdk-library = "^1.0.0" # Add your SDK dependency +``` + +#### Step 18: Create Tests + +**Explanation:** +Testing ensures that your provider works correctly and maintains compatibility as Prowler evolves. Comprehensive tests cover authentication, session management, and provider-specific functionality. + +**Folder:** `tests/providers//` + +```python +import pytest +from prowler.providers.your_provider.your_provider import YourProvider + +class TestYourProvider: + """Test cases for YourProvider.""" + + def test_provider_initialization_with_client_credentials(self): + """Test provider initialization with client credentials.""" + provider = YourProvider( + client_id="test_client_id", + client_secret="test_client_secret", + tenant_id="test_tenant_id" + ) + assert provider.type == "your_provider" + assert provider.identity is not None + assert provider.session is not None + + def test_provider_initialization_with_profile(self): + """Test provider initialization with profile.""" + provider = YourProvider( + profile="test_profile" + ) + assert provider.type == "your_provider" + assert provider.identity is not None + + def test_connection_test(self): + """Test connection functionality.""" + result = YourProvider.test_connection( + client_id="test_client_id", + client_secret="test_client_secret", + tenant_id="test_tenant_id" + ) + # Add assertions based on expected behavior + + def test_identity_retrieval(self): + """Test identity information retrieval.""" + provider = YourProvider( + client_id="test_client_id", + client_secret="test_client_secret", + tenant_id="test_tenant_id" + ) + assert provider.identity.account_id is not None + assert provider.identity.account_name is not None + + def test_argument_validation(self): + """Test argument validation.""" + from prowler.providers.your_provider.lib.arguments.arguments import ( + validate_your_provider_arguments + ) + + # Valid arguments + validate_your_provider_arguments( + client_id="test_client_id", + client_secret="test_client_secret", + tenant_id="test_tenant_id" + ) + + # Invalid arguments + with pytest.raises(ValueError, match="at least one authentication method"): + validate_your_provider_arguments() +``` + +#### Step 19: Update Documentation + +**Explanation:** +Documentation updates ensure that users can find information about your provider in Prowler's documentation. This includes examples, configuration guides, and troubleshooting information. + +Update the provider documentation to include your new provider in the examples and implementation guidance. + +--- + +### API Providers + +**Definition:** + +- Interact directly with the provider's REST API using HTTP requests (e.g., via `requests`). +- Examples: NHN Cloud. + +**Typical Use Cases:** + +- Providers without an official Python SDK. +- Providers with a non-official Python SDK that is not updated and maintained. +- Providers that expose REST APIs and meet above requirements. + +**Key Characteristics:** + +- Manual management of authentication (tokens, username/password, etc). +- Arguments: Depends on the provider, for example, `username`, `password`, `tenant_id`, etc. +- Outputs: Dicts or custom models based on API responses. +- Custom HTTP session management with headers and authentication. +- Manual handling of pagination, rate limiting, and error responses. + +**Implementation Details:** + +- API providers require manual HTTP request management using libraries like `requests`. +- Authentication typically involves obtaining tokens via login endpoints or OAuth flows. +- Session management includes setting appropriate headers (Authorization, Content-Type, etc.). +- Resource discovery often requires multiple API calls to different endpoints. +- Error handling and retry logic must be implemented manually. + +--- + +### Implementation Guide for API Providers + +#### Step 1: Create the Provider Structure + +**Explanation:** +API providers require the same structure as the SDK providers, the main difference would be that due to the lack of an official Python SDK, some methods could be implemented differently or not implemented at all. + +**Required Structure:** + +``` +prowler/providers// +├── __init__.py +├── _provider.py +├── models.py +├── exceptions/ +│ ├── __init__.py +│ └── exceptions.py +├── services/ +│ ├── service_name1/ +│ └── service_name2/ +└── lib/ + ├── __init__.py + ├── arguments/ + │ ├── __init__.py + │ └── arguments.py + ├── mutelist/ + │ ├── __init__.py + │ └── mutelist.py + ├── regions/ + │ ├── __init__.py + │ └── regions.py + └── service/ + ├── __init__.py + └── service.py +``` + +**Key Components:** + +- **`_provider.py`**: Main provider class with HTTP session management +- **`models.py`**: Data structures for identity and API responses +- **`exceptions/`**: Custom exception classes for API errors +- **`services/`**: Folder that contains all the provider services +- **`lib/arguments/`**: CLI argument validation and parsing +- **`lib/mutelist/`**: Resource exclusion and muting functionality +- **`lib/regions/`**: Region management and validation. If the provider is NOT regional, this folder will not be created. +- **`lib/service/`**: Base service class for provider-specific services + +#### Step 2: Implement the Provider Class + +**Explanation:** +The provider class is the core component that handles HTTP session management, authentication, and identity information. It inherits from Prowler's base Provider class and implements API-specific authentication flows using direct HTTP requests. + +**File:** `prowler/providers//_provider.py` + +```python +import os +from typing import Optional +import requests +from prowler.providers.common.provider import Provider +from prowler.providers.common.models import Audit_Metadata, Connection +from prowler.config.config import load_and_validate_config_file, get_default_mute_file_path +from prowler.lib.logger import logger +from prowler.lib.utils.utils import print_boxes + +# Import the needed exceptions, mutelist and models for the provider. +from prowler.providers..exceptions.exceptions import Exceptions +from prowler.providers..lib.mutelist.mutelist import Mutelist +from prowler.providers..models import NeededModels + +class APIProvider(Provider): + """ + APIProvider class is the main class for the API Provider. + + This class is responsible for initializing the provider, setting up the HTTP session, + validating credentials, and managing identity information through direct API calls. + + Attributes: + _type (str): The provider type. + _session (requests.Session): The HTTP session for API calls. + _identity (APIIdentityInfo): The provider identity information. + _audit_config (dict): The audit configuration. + _mutelist (APIMutelist): The provider mutelist. + audit_metadata (Audit_Metadata): The audit metadata. + """ + + _type: str = "api_provider" + _session: Optional[requests.Session] + _identity: APIIdentityInfo + _audit_config: dict + _mutelist: APIMutelist + audit_metadata: Audit_Metadata + + def __init__( + self, + # Authentication parameters + username: str = None, + password: str = None, + tenant_id: str = None, + # Configuration + config_path: str = None, + config_content: dict = None, + mutelist_path: str = None, + mutelist_content: dict = None, + fixer_config: dict = None, + ): + """ + Initializes the APIProvider instance. + + Args: + username: The API username for authentication + password: The API password for authentication + tenant_id: The tenant ID for authentication + config_path: Path to the configuration file + config_content: Configuration content as dictionary + mutelist_path: Path to the mutelist file + mutelist_content: Mutelist content as dictionary + fixer_config: Fixer configuration dictionary + + Raises: + ValueError: If required authentication parameters are missing + """ + logger.info("Initializing APIProvider ...") + + # 1) Store argument values with environment variable fallback + self._username = username or os.getenv("YOUR_PROVIDER_USERNAME") + self._password = password or os.getenv("YOUR_PROVIDER_PASSWORD") + self._tenant_id = tenant_id or os.getenv("YOUR_PROVIDER_TENANT_ID") + + # Validate required parameters + if not all([self._username, self._password, self._tenant_id]): + raise ValueError("APIProvider requires username, password and tenant_id") + + # 2) Load audit_config, fixer_config, mutelist + self._fixer_config = fixer_config if fixer_config else {} + + if config_content: + self._audit_config = config_content + else: + if not config_path: + config_path = default_config_file_path + self._audit_config = load_and_validate_config_file(self._type, config_path) + + if mutelist_content: + self._mutelist = APIMutelist(mutelist_content=mutelist_content) + else: + if not mutelist_path: + mutelist_path = get_default_mute_file_path(self._type) + self._mutelist = APIMutelist(mutelist_path=mutelist_path) + + # 3) Initialize session/token + self._token = None + self._session = None + self.setup_session() + + # 4) Create identity object + self._identity = APIIdentityInfo( + tenant_id=self._tenant_id, + username=self._username, + ) + + Provider.set_global_provider(self) + + @property + def type(self) -> str: + """Returns the type of the provider.""" + return self._type + + @property + def identity(self) -> APIIdentityInfo: + """Returns the provider identity information.""" + return self._identity + + @property + def session(self) -> requests.Session: + """Returns the HTTP session for API calls.""" + return self._session + + @property + def audit_config(self) -> dict: + """Returns the audit configuration.""" + return self._audit_config + + @property + def fixer_config(self) -> dict: + """Returns the fixer configuration.""" + return self._fixer_config + + @property + def mutelist(self) -> APIMutelist: + """Returns the provider mutelist.""" + return self._mutelist + + def print_credentials(self) -> None: + """ + Display account information with color formatting. + + This method prints the provider credentials and account information + in a formatted way using colorama for better readability. + """ + from colorama import Style + + report_lines = [ + f" Username: {self._username}", + f" TenantID: {self._tenant_id}", + ] + report_title = f"{Style.BRIGHT}Using the {self._type.upper()} credentials below:{Style.RESET_ALL}" + print_boxes(report_lines, report_title) + + def setup_session(self) -> None: + """ + Implement API authentication method by calling the provider's authentication endpoint. + + This method performs the authentication flow to obtain an access token + and creates a requests.Session with the appropriate headers for API calls. + """ + # Example for a Keystone-like authentication + url = "https://api.your-provider.com/v2.0/tokens" + data = { + "auth": { + "tenantId": self._tenant_id, + "passwordCredentials": { + "username": self._username, + "password": self._password, + }, + } + } + + try: + response = requests.post(url, json=data, timeout=10) + if response.status_code == 200: + resp_json = response.json() + self._token = resp_json["access"]["token"]["id"] + + # Create session with authentication headers + sess = requests.Session() + sess.headers.update({ + "X-Auth-Token": self._token, + "Content-Type": "application/json" + }) + self._session = sess + logger.info("API token acquired successfully and session is set up.") + else: + logger.critical( + f"Failed to get token. Status: {response.status_code}, Body: {response.text}" + ) + raise ValueError("Failed to get API token") + except Exception as e: + logger.critical(f"[setup_session] Error: {e}") + raise e + + @staticmethod + def test_connection( + username: str, + password: str, + tenant_id: str, + raise_on_exception: bool = True, + ) -> Connection: + """ + Test connection to the API provider by performing: + 1) Authentication token request + 2) (Optional) a small test API call to confirm credentials are valid + + Args: + username: The API username + password: The API password + tenant_id: The tenant ID + raise_on_exception: If True, raise the caught exception; + if False, return Connection(error=exception). + + Returns: + Connection: Connection test result + """ + try: + # 1) Validate arguments + if not username or not password or not tenant_id: + error_msg = "API test_connection error: missing username/password/tenant_id" + logger.error(error_msg) + raise ValueError(error_msg) + + # 2) Request authentication token + token_url = "https://api.your-provider.com/v2.0/tokens" + data = { + "auth": { + "tenantId": tenant_id, + "passwordCredentials": { + "username": username, + "password": password, + }, + } + } + + resp = requests.post(token_url, json=data, timeout=10) + if resp.status_code != 200: + error_msg = f"Failed to get token. Status: {resp.status_code}, Body: {resp.text}" + logger.error(error_msg) + if raise_on_exception: + raise Exception(error_msg) + return Connection(error=Exception(error_msg)) + + # Success + token_json = resp.json() + api_token = token_json["access"]["token"]["id"] + logger.info("API test_connection: Successfully acquired token.") + + # 3) (Optional) Test API call to confirm credentials are valid + test_endpoint = f"https://api.your-provider.com/v2/{tenant_id}/test" + headers = { + "X-Auth-Token": api_token, + "Content-Type": "application/json", + } + + test_resp = requests.get(test_endpoint, headers=headers, timeout=10) + if test_resp.status_code == 200: + logger.info("API test_connection: Test call success. Credentials valid.") + return Connection(is_connected=True) + else: + error_msg = f"Test call failed. Status: {test_resp.status_code}, Body: {test_resp.text}" + logger.error(error_msg) + if raise_on_exception: + raise Exception(error_msg) + return Connection(error=Exception(error_msg)) + + except Exception as e: + logger.critical(f"{e.__class__.__name__}[{e.__traceback__.tb_lineno}]: {e}") + if raise_on_exception: + raise e + return Connection(error=e) + + @staticmethod + def validate_arguments(username: str, password: str, tenant_id: str) -> None: + """ + Ensures that username, password, and tenant_id are not empty. + + Args: + username: The username to validate + password: The password to validate + tenant_id: The tenant ID to validate + + Raises: + ValueError: If any required parameter is missing + """ + if not username or not password or not tenant_id: + raise ValueError("API Provider requires username, password and tenant_id.") +``` + +#### Step 3: Create Models + +**Explanation:** +Models define the data structures used by your API provider. They include identity information and API response structures. These models ensure type safety and consistent data handling across the provider. + +**File:** `prowler/providers//models.py` + +This step is common with SDK providers so you can follow the same pattern as [there](#step-3-create-models). + +#### Step 4: Implement Arguments + +**Explanation:** +Argument validation ensures that the API provider receives valid configuration parameters. This step is crucial for preventing runtime errors and providing clear error messages to users. + +**File:** `prowler/providers//lib/arguments/arguments.py` + +Arguments depends on the provider and not the type, so the pattern for this step is the same as the [SDK providers](#step-4-implement-arguments). + +#### Step 5: Implement Mutelist + +**Explanation:** +The mutelist functionality allows users to exclude specific resources or checks from the audit. This is useful for handling false positives or excluding resources that are intentionally configured differently. + +**File:** `prowler/providers//lib/mutelist/mutelist.py` + +The implementation of the mutelist is the same as the [SDK providers](#step-5-implement-mutelist). + +#### Step 6: Implement Regions + +**Explanation:** +Region management is essential for cloud providers that operate across multiple geographic locations. This component handles region validation and provides region-specific functionality. + + +Regions are optional, only if the provider has regions, for example Github does not have regions, but AWS does. + + +**File:** `prowler/providers//lib/regions/_regions.py` + +The implementation of the regions is the same as the [SDK providers](#step-6-implement-regions). + +#### Step 7: Create Custom Exceptions + +**Explanation:** +Custom exceptions provide specific error handling for API-related issues, making debugging and error reporting more effective. Prowler uses a structured exception system with error codes, messages, and remediation steps. + +**File:** `prowler/providers//exceptions/exceptions.py` + +```python +from prowler.exceptions.exceptions import ProwlerException + + +# Exceptions codes from 8000 to 8999 are reserved for API Provider exceptions (example numbers) +class APIProviderBaseException(ProwlerException): + """Base class for API Provider Errors.""" + + APIProvider_ERROR_CODES = { + (8000, "APIProviderCredentialsError"): { + "message": "API Provider credentials not found or invalid", + "remediation": "Check the API Provider API credentials and ensure they are properly set.", + }, + (8001, "APIProviderAuthenticationError"): { + "message": "API Provider authentication failed", + "remediation": "Check the API Provider API credentials and ensure they are valid.", + }, + (8002, "APIProviderSessionError"): { + "message": "API Provider session setup failed", + "remediation": "Check the session setup and ensure it is properly configured.", + }, + (8003, "APIProviderIdentityError"): { + "message": "API Provider identity setup failed", + "remediation": "Check credentials and ensure they are properly set up for API Provider.", + }, + (8004, "APIProviderAPIError"): { + "message": "API Provider API call failed", + "remediation": "Check the API request and ensure it is properly formatted.", + }, + (8005, "APIProviderRateLimitError"): { + "message": "API Provider API rate limit exceeded", + "remediation": "Reduce the number of API requests or wait before making more requests.", + }, + } + + def __init__(self, code, file=None, original_exception=None, message=None): + provider = "API Provider" + error_info = self.APIProvider_ERROR_CODES.get((code, self.__class__.__name__)) + if message: + error_info["message"] = message + super().__init__( + code=code, + source=provider, + file=file, + original_exception=original_exception, + error_info=error_info, + ) + + +class APIProviderCredentialsError(APIProviderBaseException): + """Exception for API Provider credentials errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8000, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderAuthenticationError(APIProviderBaseException): + """Exception for API Provider authentication errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8001, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderSessionError(APIProviderBaseException): + """Exception for API Provider session setup errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8002, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderIdentityError(APIProviderBaseException): + """Exception for API Provider identity setup errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8003, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderAPIError(APIProviderBaseException): + """Exception for API Provider API errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8004, + file=file, + original_exception=original_exception, + message=message, + ) + + +class APIProviderRateLimitError(APIProviderBaseException): + """Exception for API Provider rate limit errors""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + code=8005, + file=file, + original_exception=original_exception, + message=message, + ) +``` + +#### Step 8: Implement Service Base Class + +**Explanation:** +The service base class defines a common interface for all services in your provider, since they will inherit from it. It defines the client to make requests to, the audit configuration and the fixer configuration. + +**File:** `prowler/providers//lib/service/service.py` + +```python +from prowler.providers.._provider import Provider + +class APIProviderService(BaseService): + """ + Base service class for API Provider services. + + This class provides common functionality for all services + within the provider, including session management and error handling. + """ + + def __init__(self, provider: Provider): + """ + Initialize the service. + + Args: + provider: The provider instance + """ + self.client = provider.session.get_client(self.service_name) + self.audit_config = provider.audit_config + self.fixer_config = provider.fixer_config + self.session = provider.session + self.base_url = provider.session.base_url + self.auth = HTTPDigestAuth( + provider.session.public_key, + provider.session.private_key, + ) + self.headers = { + "Authorization": self.auth.encode(), + "Content-Type": "application/json", + } +``` + +#### Step 9: Register in CLI + +**Explanation:** +Add your provider to the available providers in the CLI. + +**File:** `prowler/lib/cli/parser.py` + +This step is the same as the [SDK providers](#step-9-register-in-cli). + +#### Step 10: Register in Main + +**Explanation:** +Main registration makes your provider discoverable by Prowler's core system. It's needed to add your provider to the output options and to the compliance evaluation. + +**File:** `prowler/__main__.py` + +This step is the same as the [SDK providers](#step-10-register-in-main). + +#### Step 11: Register in the list of providers + +**Explanation:** +This is needed to be able to use the provider in the generic checks. The provider must be registered in the `init_global_provider` method to handle CLI arguments and initialization. + +**File:** `prowler/providers/common/provider.py` + +This step is the same as the [SDK providers](#step-11-register-in-the-list-of-providers). + +#### Step 12: Add to Config + +**Explanation:** +Configuration registration ensures your API provider is recognized by Prowler's configuration system. This enables proper handling of provider-specific settings and defaults. + +**File:** `prowler/config/config.py` + +This step is the same as the [SDK providers](#step-12-add-to-config). + +#### Step 13: Create Compliance Files + +**Explanation:** +Compliance files define the security checks and standards that your provider supports. These JSON files map security controls to specific checks and provide remediation guidance. It's needed to create the folder with an init file to ensure the provider will work, however, adding different compliance files is optional. + +**Folder:** `prowler/compliance//` + +This step is the same as the [SDK providers](#step-13-create-compliance-files). + +#### Step 14: Add Output Support + +**Explanation:** +Output support ensures that your provider's results are properly formatted in Prowler's various output formats (CSV, JSON, HTML, etc.). This step integrates your provider with Prowler's reporting system. + +**File:** `prowler/lib/outputs/summary_table.py` + +This step is the same as the [SDK providers](#step-14-add-output-support). + +#### Step 15: Generate the HTML Report + +**Explanation:** +The HTML file is needed to be able to generate the HTML report. This step involves adding support for your provider in the HTML output generation system to ensure proper display of assessment summaries and findings. + +**File:** `prowler/lib/outputs/html/html.py` + +This step is the same as the [SDK providers](#step-15-generate-the-html-report). + +#### Step 16: Add the Check Report Model + +**Explanation:** +Add the provider to the generic models, this is needed to be able to use the provider in the generic checks. + +**File:** `prowler/providers/check/models.py` + +This step is the same as the [SDK providers](#step-16-add-the-check-report-model). + +#### Step 17: Create Tests + +**Explanation:** +Testing ensures that your API provider works correctly and maintains compatibility as Prowler evolves. Comprehensive tests cover authentication, session management, and API-specific functionality. + +**Folder:** `tests/providers//` + +```python +import pytest +from prowler.providers.api_provider.api_provider import APIProvider + +class TestAPIProvider: + """Test cases for APIProvider.""" + + def test_provider_initialization(self): + """Test provider initialization with valid credentials.""" + provider = APIProvider( + username="test_user", + password="test_password", + tenant_id="test_tenant" + ) + assert provider.type == "your_api_provider" + assert provider.identity is not None + assert provider.session is not None + + def test_connection_test(self): + """Test connection functionality.""" + result = APIProvider.test_connection( + username="test_user", + password="test_password", + tenant_id="test_tenant" + ) + # Add assertions based on expected behavior + + def test_argument_validation(self): + """Test argument validation.""" + from prowler.providers.api_provider.api_provider import ( + APIProvider + ) + + # Valid arguments + APIProvider.validate_arguments( + username="test_user", + password="test_password", + tenant_id="test_tenant" + ) + + # Invalid arguments + with pytest.raises(ValueError, match="requires username, password and tenant_id"): + APIProvider.validate_arguments("", "", "") + + def test_session_setup(self): + """Test session setup.""" + provider = APIProvider( + username="test_user", + password="test_password", + tenant_id="test_tenant" + ) + assert provider.session is not None + assert "X-Auth-Token" in provider.session.headers +``` + +#### Step 18: Update Documentation + +**Explanation:** +Documentation updates ensure that users can find information about your API provider in Prowler's documentation. This includes examples, configuration guides, and troubleshooting information. + +Update the provider documentation to include your new API provider in the examples and implementation guidance. + +--- + +### Tool/Wrapper Providers + +**Definition:** + +- Integrate third-party tools as libraries or subprocesses (e.g., Trivy for IaC). +- Examples: IaC (Trivy). + +**Typical Use Cases:** + +- Providers that require integration with external security tools. +- Tools that need to be executed as subprocesses or imported as libraries. +- Providers that require specific tool configurations and argument mapping. +- Legacy systems or tools that don't have direct API access. + +**Key Characteristics:** + +- No session/identity management required (tool handles this internally). +- Arguments: specific to the tool, but for example: `scan_path`, `frameworks`, `exclude_path`, `scan_repository_url`, etc. +- Outputs: Tool-specific output formats that need to be parsed and converted. +- Tool execution and output parsing. +- Configuration file mapping and argument translation. + +**Implementation Details:** + +- Tool providers typically execute external tools as subprocesses (e.g., `pwsh` or `trivy` command). +- They require mapping between Prowler's interface and the tool's arguments. +- Output parsing and conversion to Prowler's standard format is crucial. +- Tool-specific configuration files and validation. +- Repository cloning and temporary file management for remote scans (if needed). + +**Note:** This guide provides a general framework for integrating any external tool. The specific implementation details (like repository cloning, authentication tokens, etc.) will depend on your particular tool's requirements. The core pattern is: integrate with Prowler's CLI, execute your tool via subprocess, and parse the output into Prowler's format. + +--- + +### Implementation Guide for Tool/Wrapper Providers + +#### Step 1: Create the Provider Structure + +**Explanation:** +Tool/Wrapper providers require a specific folder structure to organize tool integration, configuration, and service management. This structure follows Prowler's conventions and ensures proper integration with the CLI and API. + +**Required Structure:** + +``` +prowler/providers// +├── __init__.py +├── _provider.py +├── models.py +└── lib/ + ├── __init__.py + └── arguments/ + ├── __init__.py + └── arguments.py +``` + +**Key Components:** + +- **`_provider.py`**: Main provider class with tool integration +- **`models.py`**: Data structures for tool output and configuration +- **`lib/arguments/`**: CLI argument validation and parsing + +#### Step 2: Implement the Provider Class + +**Explanation:** +The provider class is the core component that handles tool integration, execution, and output parsing. It inherits from Prowler's base Provider class and implements tool-specific execution flows using subprocesses or library calls. + +**File:** `prowler/providers//_provider.py` + +```python +import json +import subprocess +import sys +from typing import List + +from colorama import Fore, Style + +from prowler.config.config import ( + default_config_file_path, + load_and_validate_config_file, +) +from prowler.lib.check.models import CheckReportYourTool +from prowler.lib.logger import logger +from prowler.lib.utils.utils import print_boxes +from prowler.providers.common.models import Audit_Metadata +from prowler.providers.common.provider import Provider + +class ToolProvider(Provider): + """ + ToolProvider class is the main class for the Your Tool Provider. + + This class is responsible for initializing the provider, executing the external tool, + parsing tool output, and converting results to Prowler's standard format. + + Attributes: + _type (str): The provider type. + _session: Not used for tool providers. + _identity (str): Simple identity for tool providers. + _audit_config (dict): The audit configuration. + audit_metadata (Audit_Metadata): The audit metadata. + """ + + _type: str = "your_tool_provider" + audit_metadata: Audit_Metadata + + def __init__( + self, + # Tool-specific parameters + scan_path: str = ".", + tool_specific_arg: str = "default_value", + exclude_path: list[str] = [], + # Configuration + config_path: str = None, + config_content: dict = None, + fixer_config: dict = {}, + # Authentication (if needed for your tool) + auth_token: str = None, + auth_username: str = None, + ): + """ + Initializes the ToolProvider instance. + + Args: + scan_path: Path to the folder containing files to scan + tool_specific_arg: Tool-specific argument for your external tool + exclude_path: List of paths to exclude from scan + config_path: Path to the configuration file + config_content: Configuration content as dictionary + fixer_config: Fixer configuration dictionary + auth_token: Authentication token for your tool (if needed) + auth_username: Username for your tool (if needed) + + Raises: + ValueError: If required parameters are missing + """ + logger.info("Instantiating YourTool Provider...") + + # Store tool-specific parameters + self.scan_path = scan_path + self.tool_specific_arg = tool_specific_arg + self.exclude_path = exclude_path + self.region = "global" + self.audited_account = "local-tool" + self._session = None + self._identity = "prowler" + self._auth_method = "No auth" + + # Handle tool authentication if needed + if auth_token: + self.auth_token = auth_token + self._auth_method = "Token" + logger.info("Using token for tool authentication") + elif auth_username: + self.auth_username = auth_username + self._auth_method = "Username" + logger.info("Using username for tool authentication") + logger.info("Using username for tool authentication") + else: + logger.debug("No authentication method provided; proceeding without authentication.") + + # Audit Config + if config_content: + self._audit_config = config_content + else: + if not config_path: + config_path = default_config_file_path + self._audit_config = load_and_validate_config_file(self._type, config_path) + + # Fixer Config + self._fixer_config = fixer_config + + # Mutelist (not needed for tool providers since tools have their own mutelist logic) + self._mutelist = None + + Provider.set_global_provider(self) + + @property + def auth_method(self): + """Returns the authentication method used.""" + return self._auth_method + + @property + def type(self): + """Returns the type of the provider.""" + return self._type + + @property + def identity(self): + """Returns the provider identity.""" + return self._identity + + @property + def session(self): + """Returns the session (not used for tool providers).""" + return self._session + + @property + def audit_config(self): + """Returns the audit configuration.""" + return self._audit_config + + @property + def fixer_config(self): + """Returns the fixer configuration.""" + return self._fixer_config + + def setup_session(self): + """Tool providers don't need a session since they use external tools directly""" + return None + + def _process_check(self, finding: dict, check: dict, status: str) -> CheckReportYourTool: + """ + Process a single check (failed or passed) and create a CheckReportYourTool object. + + Args: + finding: The finding object from tool output + check: The individual check data + status: The status of the check ("FAIL", "PASS", or "MUTED") + + Returns: + CheckReportYourTool: The processed check report + """ + try: + metadata_dict = { + "Provider": "your_tool_provider", + "CheckID": check.get("check_id", ""), + "CheckTitle": check.get("check_name", ""), + "CheckType": ["Your Tool Provider"], + "ServiceName": finding["check_type"], + "SubServiceName": "", + "ResourceIdTemplate": "", + "Severity": ( + check.get("severity", "low").lower() + if check.get("severity") + else "low" + ), + "ResourceType": "your_tool", + "Description": check.get("check_name", ""), + "Risk": "", + "RelatedUrl": ( + check.get("guideline", "") if check.get("guideline") else "" + ), + "Remediation": { + "Code": { + "NativeIaC": "", + "Terraform": "", + "CLI": "", + "Other": "", + }, + "Recommendation": { + "Text": "", + "Url": ( + check.get("guideline", "") if check.get("guideline") else "" + ), + }, + }, + "Categories": [], + "DependsOn": [], + "RelatedTo": [], + "Notes": "", + } + + # Convert metadata dict to JSON string + metadata = json.dumps(metadata_dict) + + report = CheckReportYourTool(metadata=metadata, finding=check) + report.status = status + report.resource_tags = check.get("entity_tags", {}) + report.status_extended = check.get("check_name", "") + if status == "MUTED": + report.muted = True + return report + except Exception as error: + logger.critical( + f"{error.__class__.__name__}:{error.__traceback__.tb_lineno} -- {error}" + ) + sys.exit(1) + + def run(self) -> List[CheckReportYourTool]: + """ + Main execution method that handles tool execution. + + Returns: + List[CheckReportYourTool]: List of check reports from the tool scan + """ + return self.run_scan(self.scan_path, self.exclude_path) + + def run_scan( + self, directory: str, exclude_path: list[str] + ) -> List[CheckReportYourTool]: + """ + Execute the external tool and parse its output. + + Args: + directory: Directory to scan + frameworks: List of frameworks to scan + exclude_path: List of paths to exclude + + Returns: + List[CheckReportYourTool]: List of check reports + """ + try: + logger.info(f"Running YourTool scan on {directory} ...") + + # Build the tool command + tool_command = [ + "your_tool_command", + # Add your tool-specific arguments here, this are just examples + "-d", + directory, + "-o", + "json", + "-f", + ",".join(frameworks), + ] + if exclude_path: + tool_command.extend(["--skip-path", ",".join(exclude_path)]) + + # Run the tool with JSON output + process = subprocess.run( + tool_command, + capture_output=True, + text=True, + ) + + # Log tool's error output if any + if process.stderr: + logger.error(process.stderr) + + try: + output = json.loads(process.stdout) + if not output: + logger.warning("No findings returned from YourTool scan") + return [] + except Exception as error: + logger.critical( + f"{error.__class__.__name__}:{error.__traceback__.tb_lineno} -- {error}" + ) + sys.exit(1) + + reports = [] + + # If only one framework has findings, the output is a dict, otherwise it's a list of dicts + if isinstance(output, dict): + output = [output] + + # Process all frameworks findings + for finding in output: + results = finding.get("results", {}) + + # Process failed checks + failed_checks = results.get("failed_checks", []) + for failed_check in failed_checks: + report = self._process_check(finding, failed_check, "FAIL") + reports.append(report) + + # Process passed checks + passed_checks = results.get("passed_checks", []) + for passed_check in passed_checks: + report = self._process_check(finding, passed_check, "PASS") + reports.append(report) + + # Process skipped checks (muted) + skipped_checks = results.get("skipped_checks", []) + for skipped_check in skipped_checks: + report = self._process_check(finding, skipped_check, "MUTED") + reports.append(report) + + return reports + + except Exception as error: + if "No such file or directory: 'your_tool_command'" in str(error): + logger.critical("Please, install your_tool using 'pip install your_tool'") + sys.exit(1) + logger.critical( + f"{error.__class__.__name__}:{error.__traceback__.tb_lineno} -- {error}" + ) + sys.exit(1) + + def print_credentials(self): + """ + Display scan information with color formatting. + + This method prints the tool scan information in a formatted way + using colorama for better readability. + """ + if self.scan_repository_url: + report_title = ( + f"{Style.BRIGHT}Scanning remote repository:{Style.RESET_ALL}" + ) + report_lines = [ + f"Repository: {Fore.YELLOW}{self.scan_repository_url}{Style.RESET_ALL}", + ] + else: + report_title = ( + f"{Style.BRIGHT}Scanning local directory:{Style.RESET_ALL}" + ) + report_lines = [ + f"Directory: {Fore.YELLOW}{self.scan_path}{Style.RESET_ALL}", + ] + + if self.exclude_path: + report_lines.append( + f"Excluded paths: {Fore.YELLOW}{', '.join(self.exclude_path)}{Style.RESET_ALL}" + ) + + report_lines.append( + f"Frameworks: {Fore.YELLOW}{', '.join(self.frameworks)}{Style.RESET_ALL}" + ) + + report_lines.append( + f"Authentication method: {Fore.YELLOW}{self.auth_method}{Style.RESET_ALL}" + ) + + print_boxes(report_lines, report_title) +``` + +#### Step 3: Create Models + +**Explanation:** +Models define the data structures used by your tool provider. They include output options and tool-specific configurations. These models ensure type safety and consistent data handling across the provider. + +**File:** `prowler/providers//models.py` + +```python +from prowler.config.config import output_file_timestamp +from prowler.providers.common.models import ProviderOutputOptions + +class YourToolOutputOptions(ProviderOutputOptions): + """ + YourToolOutputOptions overrides ProviderOutputOptions for tool-specific output logic. + For example, generating a filename that includes the tool name. + + Attributes inherited from ProviderOutputOptions: + - output_filename (str): The base filename used for generated reports. + - output_directory (str): The directory to store the output files. + - ... see ProviderOutputOptions for more details. + + Methods: + - __init__: Customizes the output filename logic for the tool provider. + """ + + def __init__(self, arguments, bulk_checks_metadata): + super().__init__(arguments, bulk_checks_metadata) + + # If --output-filename is not specified, build a default name. + if not getattr(arguments, "output_filename", None): + self.output_filename = f"prowler-output-your_tool-{output_file_timestamp}" + # If --output-filename was explicitly given, respect that + else: + self.output_filename = arguments.output_filename +``` + +#### Step 4: Implement Arguments + +**Explanation:** +Argument validation ensures that the tool provider receives valid configuration parameters. This step is crucial for preventing runtime errors and providing clear error messages to users. + +**File:** `prowler/providers//lib/arguments/arguments.py` + +```python +# Add your tool-specific choices if needed +TOOL_SPECIFIC_CHOICES = [ + "option1", + "option2", + "option3", + # Add your tool's supported options +] + +def init_parser(self): + """Init the Provider CLI parser""" + _parser = self.subparsers.add_parser( + "", parents=[self.common_providers_parser], help=" Provider" + ) + + # Scan Path + _scan_subparser = _parser.add_argument_group("Scan Path") + _scan_subparser.add_argument( + "--scan-path", + "-P", + dest="scan_path", + default=".", + help="Path to the folder containing your files to scan. Default: current directory.", + ) + + _scan_subparser.add_argument( + "--tool-specific-arg", + dest="tool_specific_arg", + default="default_value", + choices=TOOL_SPECIFIC_CHOICES, + help="Tool-specific argument for your external tool. Default: default_value", + ) + + _scan_subparser.add_argument( + "--exclude-path", + dest="exclude_path", + nargs="+", + default=[], + help="Comma-separated list of paths to exclude from the scan. Default: none", + ) + + # Authentication (if needed for your tool) + _scan_subparser.add_argument( + "--auth-token", + dest="auth_token", + nargs="?", + default=None, + help="Authentication token for your tool. If not provided, will use YOUR_TOOL_AUTH_TOKEN env var.", + ) + _scan_subparser.add_argument( + "--auth-username", + dest="auth_username", + nargs="?", + default=None, + help="Username for your tool authentication. If not provided, will use YOUR_TOOL_AUTH_USERNAME env var.", + ) + +def validate_arguments(arguments): + """ + Validate tool-specific arguments. + + Args: + arguments: The parsed arguments + + Returns: + tuple: (is_valid, error_message) + """ + scan_path = getattr(arguments, "scan_path", None) + scan_repository_url = getattr(arguments, "scan_repository_url", None) + + if scan_path and scan_repository_url: + # If scan_path is set to default ("."), allow scan_repository_url + if scan_path != ".": + return ( + False, + "--scan-path (-P) and --scan-repository-url (-R) are mutually exclusive. Please specify only one.", + ) + return (True, "") +``` + +#### Step 5: Register in CLI + +**Explanation:** +Add your provider to the available providers in the CLI. + +**File:** `prowler/lib/cli/parser.py` + +This step is the same as the [SDK providers](#step-9-register-in-cli). + +#### Step 6: Register in Main + +**Explanation:** +Main registration makes your provider discoverable by Prowler's core system. It's needed to add your provider to the output options and to the compliance evaluation. + +**File:** `prowler/__main__.py` + +This step is the same as the [SDK providers](#step-10-register-in-main). + +#### Step 7: Register in the list of providers + +**Explanation:** +This is needed to be able to use the provider in the generic checks. The provider must be registered in the `init_global_provider` method to handle CLI arguments and initialization. + +**File:** `prowler/providers/common/provider.py` + +This step is the same as the [SDK providers](#step-11-register-in-the-list-of-providers). + +#### Step 8: Add to Config + +**Explanation:** +Configuration registration ensures your tool provider is recognized by Prowler's configuration system. This enables proper handling of provider-specific settings and defaults. + +**File:** `prowler/config/config.py` + +This step is the same as the [SDK providers](#step-12-add-to-config). + +In some cases, you may need to create a new configuration file for your provider, for example, the AWS one that is inside `prowler/providers/aws/config.py`. + +#### Step 9: Create Compliance Files + +**Explanation:** +Compliance files define the security checks and standards that your provider supports. These JSON files map security controls to specific checks and provide remediation guidance. It's needed to create the folder with an init file to ensure the provider will work, however, adding different compliance files is optional. + +**Folder:** `prowler/compliance//` + +This step is the same as the [SDK providers](#step-13-create-compliance-files). + +#### Step 10: Add Output Support + +**Explanation:** +Output support ensures that your provider's results are properly formatted in Prowler's various output formats (CSV, JSON, HTML, etc.). This step integrates your provider with Prowler's reporting system. + +**File:** `prowler/lib/outputs/summary_table.py` + +This step is the same as the [SDK providers](#step-14-add-output-support). + +#### Step 11: Generate the HTML Report + +**Explanation:** +The HTML file is needed to be able to generate the HTML report. This step involves adding support for your provider in the HTML output generation system to ensure proper display of assessment summaries and findings. + +**File:** `prowler/lib/outputs/html/html.py` + +This step is the same as the [SDK providers](#step-15-generate-the-html-report). + +#### Step 12: Add the Check Report Model + +**Explanation:** +Add the provider to the generic models, this is needed to be able to use the provider in the generic checks. + +**File:** `prowler/providers/check/models.py` + +This step is the same as the [SDK providers](#step-16-add-the-check-report-model). + +#### Step 13: Create Tests + +**Explanation:** +Testing ensures that your tool provider works correctly and maintains compatibility as Prowler evolves. Comprehensive tests cover tool execution, output parsing, and provider-specific functionality. + +**Folder:** `tests/providers//` + +```python +import pytest +import tempfile +import os +from prowler.providers.your_tool_provider.your_tool_provider import ToolProvider + +class TestToolProvider: + """Test cases for ToolProvider.""" + + def test_provider_initialization(self): + """Test provider initialization with valid parameters.""" + provider = ToolProvider( + scan_path=".", + frameworks=["framework1"] + ) + assert provider.type == "your_tool_provider" + assert provider.identity == "prowler" + assert provider.scan_path == "." + + def test_tool_execution(self): + """Test tool execution and output parsing.""" + provider = ToolProvider(scan_path=".") + # Mock the subprocess call and test output parsing + # This will depend on your specific tool's output format + + def test_argument_validation(self): + """Test argument validation.""" + from prowler.providers.your_tool_provider.lib.arguments.arguments import ( + validate_arguments + ) + + # Valid arguments + class MockArgs: + scan_path = "." + tool_specific_arg = "value" + + is_valid, message = validate_arguments(MockArgs()) + assert is_valid is True + + # Add more test cases as needed for your specific tool provider + + def test_print_credentials(self): + """Test print_credentials method.""" + provider = ToolProvider( + scan_path="/test/path", + frameworks=["framework1"] + ) + # This should not raise any exceptions + provider.print_credentials() +``` + +#### Step 14: Update Documentation + +**Explanation:** +Documentation updates ensure that users can find information about your tool provider in Prowler's documentation. This includes examples, configuration guides, and troubleshooting information. + +Update the provider documentation to include your new tool provider in the examples and implementation guidance. + +--- + + +## Step 2: Integrate the Provider in the API + +This step is required only if you want your provider to be available in the API and UI. The API integration involves several components: + +### 2.1. Backend API Models + +**Location:** `api/src/backend/api/models.py` + +Add your provider to the `ProviderChoices` enum and implement UID validation: + +```python +class ProviderChoices(models.TextChoices): + AWS = "aws", "AWS" + AZURE = "azure", "Azure" + GCP = "gcp", "GCP" + KUBERNETES = "kubernetes", "Kubernetes" + M365 = "m365", "Microsoft 365" + GITHUB = "github", "GitHub" + NHN = "nhn", "NHN Cloud" + IAC = "iac", "Infrastructure as Code" + YOUR_PROVIDER = "your_provider", "Your Provider" # Add your provider here + +@staticmethod +def validate_your_provider_uid(value): + """Validate your provider UID format.""" + if not re.match(r"^your-regex-pattern$", value): + raise ModelValidationError( + detail="Your provider UID must follow the specified format.", + code="your-provider-uid", + pointer="/data/attributes/uid", + ) +``` + +**Provider Model:** +The `Provider` model already exists and supports all provider types. Ensure your provider type is included in the choices. + +### 2.2. Add the provider to the Provider Choices + +Update the `return_prowler_provider` function to include your provider. This function is crucial for the API to instantiate the correct provider class. + +**File:** `api/src/backend/api/utils.py` + +```python +from prowler.providers.your_provider.your_provider import YourProvider # Add your import + +def return_prowler_provider( + provider: Provider, +) -> [ + AwsProvider + | AzureProvider + | GcpProvider + | GithubProvider + | KubernetesProvider + | M365Provider + | YourProvider # Add your provider to the return type annotation +]: + """Return the Prowler provider class based on the given provider type.""" + match provider.provider: + case Provider.ProviderChoices.AWS.value: + prowler_provider = AwsProvider + case Provider.ProviderChoices.AZURE.value: + prowler_provider = AzureProvider + case Provider.ProviderChoices.GCP.value: + prowler_provider = GcpProvider + case Provider.ProviderChoices.KUBERNETES.value: + prowler_provider = KubernetesProvider + case Provider.ProviderChoices.M365.value: + prowler_provider = M365Provider + case Provider.ProviderChoices.GITHUB.value: + prowler_provider = GithubProvider + case Provider.ProviderChoices.YOUR_PROVIDER.value: # Add your provider here + prowler_provider = YourProvider + case _: + raise ValueError(f"Provider type {provider.provider} not supported") + return prowler_provider +``` + +**Also update the `initialize_prowler_provider` function:** + +```python +def initialize_prowler_provider( + provider: Provider, + mutelist_processor: Processor | None = None, +) -> ( + AwsProvider + | AzureProvider + | GcpProvider + | GithubProvider + | KubernetesProvider + | M365Provider + | YourProvider # Add your provider to the return type annotation +): + """Initialize a Prowler provider instance based on the given provider type.""" + prowler_provider = return_prowler_provider(provider) + prowler_provider_kwargs = get_prowler_provider_kwargs(provider, mutelist_processor) + return prowler_provider(**prowler_provider_kwargs) +``` + +**Note:** The `match` statement requires Python 3.10+. If you're using an older version, you can use traditional `if-elif` statements instead. + +### 2.3. API Serializers + +Create or update serializers for your provider. You'll need to add your provider to the validation logic: + +**File:** `api/src/backend/api/v1/serializers.py` + +```python +def validate_secret_based_on_provider(provider_type, secret): + """Validate provider-specific secrets.""" + if provider_type == Provider.ProviderChoices.AWS.value: + serializer = AWSProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.AZURE.value: + serializer = AzureProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.GCP.value: + serializer = GCPProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.YOUR_PROVIDER.value: # Add your provider here + serializer = YourProviderSecret(data=secret) + # ... other providers + + if serializer.is_valid(): + return serializer.validated_data + else: + raise serializers.ValidationError(serializer.errors) + +class YourProviderSecret(serializers.Serializer): + """Serializer for your provider credentials.""" + your_auth_field = serializers.CharField(required=True) + your_optional_field = serializers.CharField(required=False) + + class Meta: + resource_name = "provider-secrets" +``` + +Also update the providers included in the serializer: + +**File:** `api/src/backend/api/v1/serializer_utils/providers.py` + +```python +@extend_schema_field( + { + "oneOf": [ + # ... existing provider schemas ... + { + "type": "object", + "title": "Your Provider Credentials", + "properties": { + "your_auth_field": { + "type": "string", + "description": "Your provider authentication field description.", + }, + "your_optional_field": { + "type": "string", + "description": "Optional field for your provider (if applicable).", + }, + "your_required_field": { + "type": "string", + "description": "Required field for your provider authentication.", + } + }, + "required": ["your_auth_field", "your_required_field"] + }, + # ... other existing schemas ... + ] + } +) +``` + +### 2.4. Database Migration + +Create a new migration to add your provider to the database. This is crucial for the API to recognize your provider type. + +**File:** `api/src/backend/api/migrations/XXXX_your_provider.py` + +```python +# Generated by Django X.X.X on YYYY-MM-DD + +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "previous_migration_name"), # Update this to the latest migration + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("your_provider", "Your Provider"), # Add your provider here + ], + default="aws", + ), + ), + migrations.RunSQL( + "ALTER TYPE provider ADD VALUE IF NOT EXISTS 'your_provider';", + reverse_sql=migrations.RunSQL.noop, + ), + ] +``` + +**Important Notes:** + +- **Migration Number**: Use the next sequential number (e.g., if latest is 0044, use 0045) +- **Dependencies**: Update the `dependencies` list to point to the most recent migration +- **Choices Array**: Add your provider to the `choices` array with proper display name +- **SQL Operation**: The `RunSQL` operation adds your provider to the PostgreSQL enum type +- **Reverse SQL**: Use `migrations.RunSQL.noop` since adding enum values cannot be easily reversed + +**Migration Naming Convention:** + +- Format: `XXXX_your_provider.py` (e.g., `0045_your_provider.py`) +- Use descriptive names that indicate what the migration does +- Follow the existing pattern in the migrations folder + +### 2.5. Update the V1 Yaml + +Update the OpenAPI specification (`v1.yaml`) to include your provider in all relevant endpoints and schemas. This is crucial for API documentation and client generation. + +**File:** `api/src/backend/api/specs/v1.yaml` + +#### 2.5.1. Provider Enum Values + +Add your provider to the provider enum in the Provider schema: + +```yaml +# Around line 12150 in v1.yaml +Provider: + type: object + properties: + attributes: + properties: + provider: + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + - github + - your_provider # Add your provider here + type: string + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `your_provider` - Your Provider # Add your provider here +``` + +#### 2.5.2. Provider Credential Schemas + +Add your provider's credential schema to the integration configuration. This defines how your provider's credentials are structured: + +```yaml +# Around line 11100 in v1.yaml, in the integration configuration +- type: object + title: Your Provider Credentials # Add your provider here + properties: + your_auth_field: + type: string + description: Your provider authentication field description. + your_optional_field: + type: string + description: Optional field for your provider (if applicable). + your_required_field: + type: string + description: Required field for your provider authentication. + required: + - your_auth_field + - your_required_field +``` + +#### 2.5.3. Example Provider Schemas + +Here are examples of how existing providers are documented: + +**AWS Provider:** +```yaml +- type: object + title: AWS Static Credentials + properties: + aws_access_key_id: + type: string + description: The AWS access key ID. + aws_secret_access_key: + type: string + description: The AWS secret access key. + required: + - aws_access_key_id + - aws_secret_access_key + +- type: object + title: AWS Assume Role + properties: + role_arn: + type: string + description: The Amazon Resource Name (ARN) of the role to assume. + external_id: + type: string + description: An identifier to enhance security for role assumption. + required: + - role_arn + - external_id +``` + +**GitHub Provider:** +```yaml +- type: object + title: GitHub Personal Access Token + properties: + personal_access_token: + type: string + description: GitHub personal access token for authentication. + required: + - personal_access_token + +- type: object + title: GitHub OAuth App Token + properties: + oauth_app_token: + type: string + description: GitHub OAuth App token for authentication. + required: + - oauth_app_token +``` + +**M365 Provider:** +```yaml +- type: object + title: M365 Static Credentials + properties: + client_id: + type: string + description: The Azure application (client) ID for authentication in Azure AD. + client_secret: + type: string + description: The client secret associated with the application (client) ID. + tenant_id: + type: string + description: The Azure tenant ID, representing the directory where the application is registered. + user: + type: email + description: User microsoft email address. + password: + type: string + description: User password. + required: + - client_id + - client_secret + - tenant_id + - user + - password +``` + +#### 2.5.4. Important Notes + +- **Position**: Add your schema in the `oneOf` array alongside existing providers +- **Structure**: Follow the exact pattern of other providers (title, properties, required fields) +- **Descriptions**: Provide clear, helpful descriptions for each field +- **Required Fields**: Specify which fields are mandatory in the `required` array +- **Field Types**: Use appropriate JSON schema types (`string`, `integer`, `boolean`, `email`, etc.) +- **Validation**: Add any field-specific validation patterns or constraints +- **Documentation**: Ensure your provider appears in the generated API documentation + +### 2.6. Testing API Integration + +Create tests for your provider: + +**Location:** `api/src/backend/api/tests/` + +```python +class YourProviderAPITestCase(APITestCase): + def setUp(self): + self.user = User.objects.create_user(username='testuser', password='testpass') + self.client.force_authenticate(user=self.user) + + def test_create_your_provider(self): + data = { + 'provider': 'your_provider', + 'uid': 'valid-uid-123', + 'alias': 'Test Account' + } + response = self.client.post('/api/v1/providers/', data) + self.assertEqual(response.status_code, 201) + self.assertEqual(response.data['provider'], 'your_provider') + + def test_your_provider_uid_validation(self): + """Test UID validation for your provider.""" + invalid_uids = [ + 'invalid@uid', + '-invalid-start', + 'a' * 40, # Too long + ] + + for invalid_uid in invalid_uids: + data = { + 'provider': 'your_provider', + 'uid': invalid_uid, + 'alias': 'Test' + } + response = self.client.post('/api/v1/providers/', data) + self.assertEqual(response.status_code, 400) + self.assertIn('your-provider-uid', str(response.data)) + + def test_add_your_provider_credentials(self): + # Create provider first + provider = Provider.objects.create( + user=self.user, + provider='your_provider', + uid='valid-uid-123' + ) + + # Add credentials + credentials_data = { + 'secret_type': 'your_provider_credentials', + 'secret': { + 'your_auth_field': 'auth_value', + 'your_optional_field': 'optional_value' + }, + 'provider': provider.id + } + response = self.client.post('/api/v1/providers/secrets/', credentials_data) + self.assertEqual(response.status_code, 201) +``` + +#### 2.6.1. Add your mocked provider to the tests + +If needed, add your mocked provider to the tests config file so you can use it on the tests. + +**File:** `api/src/backend/conftest.py` + +```python +@pytest.fixture +def providers_fixture(tenants_fixture): + tenant, *_ = tenants_fixture + providerX = Provider.objects.create( + provider="your_provider", + uid="your_uid", + alias="your_alias", + tenant_id=tenant.id, + ) + return provider1, provider2, provider3, ... providerX +``` + +### 2.7. Compliance and Output Support + +Add your provider to the compliance export functionality: + +**File:** `api/src/backend/tasks/jobs/export.py` + +```python +COMPLIANCE_FRAMEWORKS = { + "aws": [...], + "azure": [...], + "gcp": [...], + "kubernetes": [...], + "m365": [...], + "github": [...], + "your_provider": [ # Add your provider here + (lambda name: name.startswith("cis_"), YourProviderCIS), + (lambda name: name.startswith("iso27001_"), YourProviderISO27001), + ], +} +``` + +If your provider has specific fields, add them to the finding transformation: + +**File:** `prowler/lib/outputs/finding.py` + +```python +def transform_api_finding(cls, finding, provider) -> "Finding": + # ... existing code ... + + # Your provider specific field + if provider.type == "your_provider": + finding.your_field = resource.your_field + + # ... rest of the code ... +``` + +### 2.8. API Endpoints + +Your provider will be available through these endpoints: + +- `GET /api/v1/providers/` - List all providers +- `POST /api/v1/providers/` - Create a new provider +- `GET /api/v1/providers/{id}/` - Get provider details +- `PUT /api/v1/providers/{id}/` - Update provider +- `DELETE /api/v1/providers/{id}/` - Delete provider +- `POST /api/v1/providers/secrets/` - Add provider credentials + +### 2.9. Update the provider if needed + +Depending on your provider's authentication requirements, you may need to add new authentication methods that are compatible with the API. This involves updating the provider class to support additional credential types beyond the basic ones. + +#### 2.9.1. Adding New Authentication Methods + +If your provider requires specific authentication methods, you'll need to: + +1. **Update the provider constructor** to accept new authentication parameters +2. **Extend the credential handling** to support the new authentication method +3. **Update the API serializers** to include the new credential fields +4. **Modify the OpenAPI specification** to document the new authentication schema + +#### 2.9.2. Example: GitHub Provider Authentication Methods + +The GitHub provider demonstrates how to implement multiple authentication methods: + +```python +# In prowler/providers/github/github_provider.py +def __init__( + self, + # Authentication methods + personal_access_token: str = "", + oauth_app_token: str = "", + github_app_key: str = "", + #Needed for the API integration + github_app_key_content: str = "", + github_app_id: int = 0, + # Provider configuration + config_path: str = None, + # ... other parameters +): + """ + Initialize GitHub provider. + + Args: + personal_access_token (str): GitHub personal access token. + oauth_app_token (str): GitHub OAuth App token. + github_app_key (str): GitHub App key. + github_app_key_content (str): GitHub App key content. + github_app_id (int): GitHub App ID. + config_path (str): Path to the audit configuration file. + # ... other parameters + """ + super().__init__( + personal_access_token, + oauth_app_token, + github_app_id, + github_app_key, + github_app_key_content, + ) +``` + +--- + +## Step 3: Integrate the Provider in the UI + +TBD + +--- + +## Provider Implementation Guidance + +Use existing providers as templates, this will help you to understand better the structure and the implementation will be easier: + +- [AWS (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/aws/aws_provider.py) +- [Azure (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/azure/azure_provider.py) +- [GCP (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/gcp/gcp_provider.py) +- [Kubernetes (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/kubernetes/kubernetes_provider.py) +- [M365 (SDK/Wrapper)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/m365/m365_provider.py) +- [GitHub (SDK)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/github/github_provider.py) +- [NHN (API)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/nhn/nhn_provider.py) +- [IAC (Tool)](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/iac/iac_provider.py) +- [MongoDB Atlas](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/mongodbatlas/mongodbatlas_provider.py) + +--- + +## Best Practices + +- **Code Quality & Documentation** + + - **Comprehensive Docstrings**: Every class, method, and function should have detailed docstrings following Prowler's format + ```python + def method_name(self, param: str) -> str: + """ + Brief description of what the method does. + + Args: + param: Description of the parameter + + Returns: + Description of the return value + + Raises: + ExceptionType: When and why this exception occurs + """ + ``` + + - **Type Hints**: Use type hints for all function parameters and return values + ```python + from typing import Optional, List, Dict, Any + ``` + + - **Logging**: Implement proper logging using Prowler's logger + ```python + from prowler.lib.logger import logger + + logger.info("Operation completed successfully") + logger.warning("Something to be aware of") + logger.error("Something went wrong") + logger.critical("Critical error that may cause failure") + ``` + +- **Error Handling & Validation** + + - **Custom Exceptions**: Create provider-specific exceptions for better error handling + - **Input Validation**: Validate all inputs and provide clear error messages + - **Graceful Degradation**: Handle errors gracefully without crashing the entire scan + - **Raise on Exception**: Use `raise_on_exception` parameter for test methods + +- **Testing & Quality Assurance** + + - **Comprehensive Test Coverage**: Aim for >80% test coverage + - **Test Naming**: Use descriptive test names: `test_method_name_scenario` + - **Test Organization**: Group related tests in test classes + - **Mock External Dependencies**: Mock external API calls and services + - **Test Edge Cases**: Include tests for error conditions and edge cases + - **End-to-End Testing**: Test the provider on real infrastructure + +- **Performance & Security** + + - **Session Management**: Reuse sessions when possible, don't create new ones unnecessarily + - **Rate Limiting**: Implement rate limiting for API calls to avoid hitting limits + - **Resource Cleanup**: Ensure proper cleanup of temporary resources + - **Authentication Security**: Never log sensitive credentials or tokens + +- **Code Organization** + + - **Single Responsibility**: Each method should have one clear purpose + - **Consistent Naming**: Follow Prowler's naming conventions + - **Modular Design**: Break complex functionality into smaller, testable methods + - **Configuration Management**: Use configuration files for provider-specific settings + +- **Documentation & Maintenance** + + - **README Updates**: Update provider-specific documentation + - **Changelog**: Document changes and new features + - **Examples**: Provide usage examples and common scenarios + - **Troubleshooting**: Include common issues and solutions + - **Documentation**: Update the provider documentation to include your new tool provider in the examples and implementation guidance. + +- **Integration Standards** + + - **CLI Consistency**: Follow Prowler's CLI argument patterns + - **Output Format**: Ensure outputs are compatible with Prowler's reporting system + - **Compliance Mapping**: Map provider checks to relevant compliance frameworks + - **Backward Compatibility**: Maintain compatibility when possible + +- **AI-Assisted Development** + + - **Use Rules**: Use rules to ensure the code generated by AI is following the way of working in Prowler. + +## Checklist for New Providers + +### CLI Integration Only + +**Phase 1: Research & Planning** + +- [ ] Soft research completed +- [ ] Spike date scheduled +- [ ] Deeper research completed +- [ ] Action plan created + +**Phase 2: Implementation** + +- [ ] Folder and files created in `prowler/providers/` +- [ ] Provider class implemented and inherits from `Provider` +- [ ] Authentication/session logic implemented +- [ ] Arguments/flags mapped and documented +- [ ] Outputs and metadata standardized +- [ ] Registered in the CLI +- [ ] Minimal usage example provided + +**Phase 3: Delivery** + +- [ ] PoC delivered +- [ ] MVP delivered +- [ ] Version 1 completed +- [ ] QA and documentation completed +- [ ] GA release ready + +### API Integration + +- [ ] All CLI integration items completed +- [ ] Provider added to `ProviderChoices` enum in API models +- [ ] API serializers created/updated for the provider +- [ ] API views support the new provider type +- [ ] Provider credentials model supports the new provider +- [ ] API endpoints tested and working +- [ ] Provider-specific validation implemented +- [ ] API tests created and passing + +### UI Integration + +- TBD + +--- + +## Next Steps + +- [How to add a new Service](./services) +- [How to add new Checks](./checks) +- [How to contribute](./introduction#contributing-to-prowler) diff --git a/docs/developer-guide/renaming-checks.md b/docs/developer-guide/renaming-checks.mdx similarity index 99% rename from docs/developer-guide/renaming-checks.md rename to docs/developer-guide/renaming-checks.mdx index fb19070e22..7a593542f4 100644 --- a/docs/developer-guide/renaming-checks.md +++ b/docs/developer-guide/renaming-checks.mdx @@ -1,4 +1,6 @@ -# Renaming Checks in Prowler +--- +title: 'Renaming Checks in Prowler' +--- To rename a check in Prowler, follow these steps when aligning with Check ID structure, fixing typos, or updating check logic that requires a new name. diff --git a/docs/developer-guide/security-compliance-framework.md b/docs/developer-guide/security-compliance-framework.mdx similarity index 97% rename from docs/developer-guide/security-compliance-framework.md rename to docs/developer-guide/security-compliance-framework.mdx index 95b7677605..bb01af8271 100644 --- a/docs/developer-guide/security-compliance-framework.md +++ b/docs/developer-guide/security-compliance-framework.mdx @@ -1,4 +1,6 @@ -# Creating a New Security Compliance Framework in Prowler +--- +title: 'Creating a New Security Compliance Framework in Prowler' +--- ## Introduction diff --git a/docs/developer-guide/services.md b/docs/developer-guide/services.md deleted file mode 100644 index 7615404cd9..0000000000 --- a/docs/developer-guide/services.md +++ /dev/null @@ -1,250 +0,0 @@ -# Prowler Services - -Here you can find how to create a new service, or to complement an existing one, for a [Prowler Provider](./provider.md). - -???+note - First ensure that the provider you want to add the service is already created. It can be checked [here](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). If the provider is not present, please refer to the [Provider](./provider.md) documentation to create it from scratch. - -## Introduction - -In Prowler, a **service** represents a specific solution or resource offered by one of the supported [Prowler Providers](./provider.md), for example, [EC2](https://aws.amazon.com/ec2/) in AWS, or [Microsoft Exchange](https://www.microsoft.com/en-us/microsoft-365/exchange/exchange-online) in M365. Services are the building blocks that allow Prowler interact directly with the various resources exposed by each provider. - -Each service is implemented as a class that encapsulates all the logic, data models, and API interactions required to gather and store information about that service's resources. All of this data is used by the [Prowler checks](./checks.md) to generate the security findings. - -## Adding a New Service - -To create a new service, a new folder must be created inside the specific provider following this pattern: `prowler/providers//services//`. - -Within this folder the following files are also to be created: - -- `__init__.py` (empty) – Ensures Python recognizes this folder as a package. -- `_service.py` – Contains all the logic and API calls of the service. -- `_client_.py` – Contains the initialization of the freshly created service's class so that the checks can use it. - -Once the files are create, you can check that the service has been created by running the following command: `poetry run python prowler-cli.py --list-services | grep `. - -## Service Structure and Initialisation - -The Prowler's service structure is as outlined below. To initialise it, just import the service client in a check. - -### Service Base Class - -All Prowler provider service should inherit from a common base class to avoid code duplication. This base class handles initialization and storage of functions and objects needed across services. The exact implementation depends on the provider's API requirements, but the following are the most common responsibilities: - -- Initialize/store clients to interact with the provider's API. -- Store the audit and fixer configuration. -- Implement threading logic where applicable. - -For reference, the base classes for each provider can be checked here: - -- [AWS Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/aws/lib/service/service.py) -- [GCP Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/gcp/lib/service/service.py) -- [Azure Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/azure/lib/service/service.py) -- [Kubernetes Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/kubernetes/lib/service/service.py) -- [M365 Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/m365/lib/service/service.py) -- [GitHub Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/github/lib/service/service.py) - -### Service Class - -Due to the complexity and differences across provider APIs, the following example demonstrates best practices for structuring a service in Prowler. - -File `_service.py`: - -```python title="Example Service Class" -from datetime import datetime -from typing import Optional - -# To use the Pydantic's BaseModel. -from pydantic import BaseModel - -# Prowler logging library. -from prowler.lib.logger import logger - -# Provider parent class. -from prowler.providers..lib.service.service import ServiceParentClass - -# Create a class for the Service. -class (ServiceParentClass): - def __init__(self, provider: Provider): - """Initialize the Service Class - - Args: - provider: Prowler Provider object. - """ - # Call Service Parent Class __init__. - # The __class__.__name__ is used to obtain it automatically. - # From the Service Class name, but a custom one can be passed. - # String in case the provider's API service name is different. - super().__init__(__class__.__name__, provider) - - # Create an empty dictionary of items to be gathered, using the unique ID as the dictionary's key, e.g., instances. - self. = {} - - # If parallelization can be carried out by regions or locations, the function __threading_call__ to be used must be implemented in the Service Parent Class. - # If it is not implemented, you can make it in a sequential way, just calling the function. - self.__threading_call__(self.__describe___) - - # If it is needed you can create another function to retrieve more data from the items. - # Here we are using the second parameter of the __threading_call__ function to create one thread per item. - # You can also make it sequential without using the __threading_call__ function iterating over the items inside the function. - self.__threading_call__(self.__describe___, self..values()) - - # In case of use the __threading_call__ function, you have to pass the regional_client to the function, as a parameter. - def __describe___(self, regional_client): - """Get all and store in the self. dictionary - - Args: - regional_client: Regional client object. - """ - logger.info(" - Describing ...") - - # A try-except block must be created in each function. - try: - # If pagination is supported by the provider, is always better to use it, call to the provider API to retrieve the desired data. - describe__paginator = regional_client.get_paginator("describe_") - - # Paginator to get every item. - for page in describe__paginator.paginate(): - - # Another try-except within the for loop to continue iterating in case something unexpected happens. - try: - - for in page[""]: - - # Adding Retrieved Resources to the Object - - # Once the resource has been retrieved, it must be included in the previously created object to ensure proper data handling within the service. - self.[] = - ( - arn=stack[""], - name=stack[""], - tags=stack.get("Tags", []), - region=regional_client.region, - ) - - except Exception as error: - logger.error( - f"{} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - - # Logging Errors in Exception Handling - - # When handling exceptions, use the following approach to log errors appropriately based on the cloud provider being used: - except Exception as error: - # Depending on each provider we can must use different fields in the logger, e.g.: AWS: regional_client.region or self.region, GCP: project_id and location, Azure: subscription - logger.error( - f"{} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - - def __describe___(self, item: ): - """Get details for a - - Args: - item: Item object. - """ - logger.info(" - Describing to get specific details...") - # A try-except block must be created in each function. - try: - - _details = self.regional_clients[.region].describe_( - =.name - ) - - # E.g., check if item is Public. This case is important: if values are being retrieved from a dictionary, the function "dict.get()" must be used with a default value in case this value is not present. - .public = _details.get("Public", False) - except Exception as error: - # Fields for logging errors with relevant item information, e.g.: AWS: .region, GCP: .project_id, Azure: .region - logger.error( - f"{.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) -``` - -???+note - To prevent false findings, when Prowler fails to retrieve items due to Access Denied or similar errors, the affected item's value is set to `None`. - -#### Resource Models - -Resource models define structured classes used within services to store and process data extracted from API calls. They are defined in the same file as the service class, but outside of the class, usually at the bottom of the file. - -Prowler leverages Pydantic's [BaseModel](https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel) to enforce data validation. - -```python title="Service Model" - -# Implementation Approach - -# Each service class should include custom model classes using Pydantic's BaseModel for the resources being audited. - -class (BaseModel): - """ holds a """ - - id: str - """[].id""" - - name: str - """[].name""" - - region: str - """[].region""" - - public: bool - """[].public""" - - # Optional attributes can be created set to None by default. - - tags: Optional[list] - """[].tags""" -``` - -#### Service Attributes - -*Optimized Data Storage with Python Dictionaries* - -Each group of resources within a service should be structured as a Python [dictionary](https://docs.python.org/3/tutorial/datastructures.html#dictionaries) to enable efficient lookups. The dictionary lookup operation has [O(1) complexity](https://en.wikipedia.org/wiki/Big_O_notation#Orders_of_common_functions), and lookups are constantly executed. - -*Assigning Unique Identifiers* - -Each dictionary key must be a unique ID to identify the resource in a univocal way. - -Example: - -```python -self.virtual_machines = {} -self.virtual_machines["vm-01234567890abcdef"] = VirtualMachine() -``` - -### Service Client - -Each Prowler service requires a service client to use the service in the checks. - -The following is the `_client.py` file, which contains the initialization of the freshly created service's class so that service checks can use it. This file is almost the same for all the services among the providers: - -```python -from prowler.providers.common.provider import Provider -from prowler.providers..services.._service import - -_client = (Provider.get_global_provider()) -``` - -## Provider Permissions in Prowler - -Before implementing a new service, verify that Prowler's existing permissions for each provider are sufficient. If additional permissions are required, refer to the relevant documentation and update accordingly. - -Provider-Specific Permissions Documentation: - -- [AWS](../tutorials/aws/authentication.md#required-permissions) -- [Azure](../tutorials/azure/authentication.md#required-permissions) -- [GCP](../tutorials/gcp/authentication.md#required-permissions) -- [M365](../tutorials/microsoft365/authentication.md#required-permissions) -- [GitHub](../tutorials/github/authentication.md) - -## Best Practices - -- When available in the provider, use threading or parallelization utilities for all methods that can be parallelized by to maximize performance and reduce scan time. -- Define a Pydantic `BaseModel` for every resource you manage, and use these models for all resource data handling. -- Log every major step (start, success, error) in resource discovery and attribute collection for traceability and debugging; include as much context as possible. -- Catch and log all exceptions, providing detailed context (region, subscription, resource, error type, line number) to aid troubleshooting. -- Use consistent naming for resource containers, unique identifiers, and model attributes to improve code readability and maintainability. -- Add docstrings to every method and comments to explain any service-specific logic, especially where provider APIs behave differently or have quirks. -- Collect and store resource tags and additional attributes to support richer checks and reporting. -- Leverage shared utility helpers for session setup, identifier parsing, and other cross-cutting concerns to avoid code duplication. This kind of code is typically stored in a `lib` folder in the service folder. -- Keep code modular, maintainable, and well-documented for ease of extension and troubleshooting. diff --git a/docs/developer-guide/services.mdx b/docs/developer-guide/services.mdx new file mode 100644 index 0000000000..ac7088dda0 --- /dev/null +++ b/docs/developer-guide/services.mdx @@ -0,0 +1,550 @@ +--- +title: 'Prowler Services' +--- + +Here you can find how to create a new service, or to complement an existing one, for a [Prowler Provider](/developer-guide/provider). + + +First ensure that the provider you want to add the service is already created. It can be checked [here](https://github.com/prowler-cloud/prowler/tree/master/prowler/providers). If the provider is not present, please refer to the [Provider](./provider.md) documentation to create it from scratch. + +## Introduction + +In Prowler, a **service** represents a specific solution or resource offered by one of the supported [Prowler Providers](/developer-guide/provider), for example, [EC2](https://aws.amazon.com/ec2/) in AWS, or [Microsoft Exchange](https://www.microsoft.com/en-us/microsoft-365/exchange/exchange-online) in M365. Services are the building blocks that allow Prowler interact directly with the various resources exposed by each provider. + +Each service is implemented as a class that encapsulates all the logic, data models, and API interactions required to gather and store information about that service's resources. All of this data is used by the [Prowler checks](/developer-guide/checks) to generate the security findings. + +## Adding a New Service + +To create a new service, a new folder must be created inside the specific provider following this pattern: `prowler/providers//services//`. + +Within this folder the following files are also to be created: + +- `__init__.py` (empty) – Ensures Python recognizes this folder as a package. +- `_service.py` – Contains all the logic and API calls of the service. +- `_client_.py` – Contains the initialization of the freshly created service's class so that the checks can use it. + +Once the files are create, you can check that the service has been created by running the following command: `poetry run python prowler-cli.py --list-services | grep `. + +## Service Structure and Initialisation + +The Prowler's service structure is as outlined below. To initialise it, just import the service client in a check. + +### Service Base Class + +All Prowler provider service should inherit from a common base class to avoid code duplication. This base class handles initialization and storage of functions and objects needed across services. The exact implementation depends on the provider's API requirements, but the following are the most common responsibilities: + +- Initialize/store clients to interact with the provider's API. +- Store the audit and fixer configuration. +- Implement threading logic where applicable. + +For reference, the base classes for each provider can be checked here: + +- [AWS Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/aws/lib/service/service.py) +- [GCP Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/gcp/lib/service/service.py) +- [Azure Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/azure/lib/service/service.py) +- [Kubernetes Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/kubernetes/lib/service/service.py) +- [M365 Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/m365/lib/service/service.py) +- [GitHub Service Base Class](https://github.com/prowler-cloud/prowler/blob/master/prowler/providers/github/lib/service/service.py) + +### Service Class + +Due to the complexity and differences across provider APIs, the following example demonstrates best practices for structuring a service in Prowler. + +File `_service.py`: + +```python title="Example Service Class" +from datetime import datetime +from typing import Optional + +# To use the Pydantic's BaseModel. +from pydantic import BaseModel + +# Prowler logging library. +from prowler.lib.logger import logger + +# Provider parent class. +from prowler.providers..lib.service.service import ServiceParentClass + +# Create a class for the Service. +class (ServiceParentClass): + def __init__(self, provider: Provider): + """Initialize the Service Class + + Args: + provider: Prowler Provider object. + """ + # Call Service Parent Class __init__. + # The __class__.__name__ is used to obtain it automatically. + # From the Service Class name, but a custom one can be passed. + # String in case the provider's API service name is different. + super().__init__(__class__.__name__, provider) + + # Create an empty dictionary of items to be gathered, using the unique ID as the dictionary's key, e.g., instances. + self. = {} + + # If parallelization can be carried out by regions or locations, the function __threading_call__ to be used must be implemented in the Service Parent Class. + # If it is not implemented, you can make it in a sequential way, just calling the function. + self.__threading_call__(self.__describe___) + + # If it is needed you can create another function to retrieve more data from the items. + # Here we are using the second parameter of the __threading_call__ function to create one thread per item. + # You can also make it sequential without using the __threading_call__ function iterating over the items inside the function. + self.__threading_call__(self.__describe___, self..values()) + + # In case of use the __threading_call__ function, you have to pass the regional_client to the function, as a parameter. + def __describe___(self, regional_client): + """Get all and store in the self. dictionary + + Args: + regional_client: Regional client object. + """ + logger.info(" - Describing ...") + + # A try-except block must be created in each function. + try: + # If pagination is supported by the provider, is always better to use it, call to the provider API to retrieve the desired data. + describe__paginator = regional_client.get_paginator("describe_") + + # Paginator to get every item. + for page in describe__paginator.paginate(): + + # Another try-except within the for loop to continue iterating in case something unexpected happens. + try: + + for in page[""]: + + # Adding Retrieved Resources to the Object + + # Once the resource has been retrieved, it must be included in the previously created object to ensure proper data handling within the service. + self.[] = + ( + arn=stack[""], + name=stack[""], + tags=stack.get("Tags", []), + region=regional_client.region, + ) + + except Exception as error: + logger.error( + f"{} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + + # Logging Errors in Exception Handling + + # When handling exceptions, use the following approach to log errors appropriately based on the cloud provider being used: + except Exception as error: + # Depending on each provider we can must use different fields in the logger, e.g.: AWS: regional_client.region or self.region, GCP: project_id and location, Azure: subscription + logger.error( + f"{} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + + def __describe___(self, item: ): + """Get details for a + + Args: + item: Item object. + """ + logger.info(" - Describing to get specific details...") + # A try-except block must be created in each function. + try: + + _details = self.regional_clients[.region].describe_( + =.name + ) + + # E.g., check if item is Public. This case is important: if values are being retrieved from a dictionary, the function "dict.get()" must be used with a default value in case this value is not present. + .public = _details.get("Public", False) + except Exception as error: + # Fields for logging errors with relevant item information, e.g.: AWS: .region, GCP: .project_id, Azure: .region + logger.error( + f"{.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) +``` + + +To prevent false findings, when Prowler fails to retrieve items due to Access Denied or similar errors, the affected item's value is set to `None`. + + +#### Resource Models + +Resource models define structured classes used within services to store and process data extracted from API calls. They are defined in the same file as the service class, but outside of the class, usually at the bottom of the file. + +Prowler leverages Pydantic's [BaseModel](https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel) to enforce data validation. + +```python title="Service Model" + +# Implementation Approach + +# Each service class should include custom model classes using Pydantic's BaseModel for the resources being audited. + +class (BaseModel): + """ holds a """ + + id: str + """[].id""" + + name: str + """[].name""" + + region: str + """[].region""" + + public: bool + """[].public""" + + # Optional attributes can be created set to None by default. + + tags: Optional[list] + """[].tags""" +``` + +#### Service Attributes + +_Optimized Data Storage with Python Dictionaries_ + +Each group of resources within a service should be structured as a Python [dictionary](https://docs.python.org/3/tutorial/datastructures.html#dictionaries) to enable efficient lookups. The dictionary lookup operation has [O(1) complexity](https://en.wikipedia.org/wiki/Big_O_notation#Orders_of_common_functions), and lookups are constantly executed. + +_Assigning Unique Identifiers_ + +Each dictionary key must be a unique ID to identify the resource in a univocal way. + +Example: + +```python +self.virtual_machines = {} +self.virtual_machines["vm-01234567890abcdef"] = VirtualMachine() +``` + +### Service Client + +Each Prowler service requires a service client to use the service in the checks. + +The following is the `_client.py` file, which contains the initialization of the freshly created service's class so that service checks can use it. This file is almost the same for all the services among the providers: + +```python +from prowler.providers.common.provider import Provider +from prowler.providers..services.._service import + +_client = (Provider.get_global_provider()) +``` + +## Provider Permissions in Prowler + +Before implementing a new service, verify that Prowler's existing permissions for each provider are sufficient. If additional permissions are required, refer to the relevant documentation and update accordingly. + +Provider-Specific Permissions Documentation: + +- [AWS](/user-guide/providers/aws/authentication#required-permissions) +- [Azure](/user-guide/providers/azure/authentication#required-permissions) +- [GCP](/user-guide/providers/gcp/authentication#required-permissions) +- [M365](/user-guide/providers/microsoft365/authentication#required-permissions) +- [GitHub](/user-guide/providers/github/authentication) + +## Service Architecture and Cross-Service Communication + +### Core Principle: Service Isolation with Client Communication + +Each service must contain **ONLY** the information unique to that specific service. When a check requires information from multiple services, it must use the **client objects** of other services rather than directly accessing their data structures. + +This architecture ensures: + +- **Loose coupling** between services +- **Clear separation of concerns** +- **Maintainable and testable code** +- **Consistent data access patterns** + +### Cross-Service Communication Pattern + +Instead of services directly accessing each other's internal data, checks should import and use client objects: + +**❌ INCORRECT - Direct data access:** + +```python +# DON'T DO THIS +from prowler.providers.aws.services.cloudtrail.cloudtrail_service import cloudtrail_service +from prowler.providers.aws.services.s3.s3_service import s3_service + +class cloudtrail_bucket_requires_mfa_delete(Check): + def execute(self): + # WRONG: Directly accessing service data + for trail in cloudtrail_service.trails.values(): + for bucket in s3_service.buckets.values(): + # Direct access violates separation of concerns +``` + +**✅ CORRECT - Client-based communication:** + +```python +# DO THIS INSTEAD +from prowler.providers.aws.services.cloudtrail.cloudtrail_client import cloudtrail_client +from prowler.providers.aws.services.s3.s3_client import s3_client + +class cloudtrail_bucket_requires_mfa_delete(Check): + def execute(self): + # CORRECT: Using client objects for cross-service communication + for trail in cloudtrail_client.trails.values(): + trail_bucket = trail.s3_bucket + for bucket in s3_client.buckets.values(): + if trail_bucket == bucket.name: + # Use bucket properties through s3_client + if bucket.mfa_delete: + # Implementation logic +``` + +### Real-World Example: CloudTrail + S3 Integration + +This example demonstrates how CloudTrail checks validate S3 bucket configurations: + +```python +from prowler.lib.check.models import Check, Check_Report_AWS +from prowler.providers.aws.services.cloudtrail.cloudtrail_client import cloudtrail_client +from prowler.providers.aws.services.s3.s3_client import s3_client + +class cloudtrail_bucket_requires_mfa_delete(Check): + def execute(self): + findings = [] + if cloudtrail_client.trails is not None: + for trail in cloudtrail_client.trails.values(): + if trail.is_logging: + trail_bucket_is_in_account = False + trail_bucket = trail.s3_bucket + + # Cross-service communication: CloudTrail check uses S3 client + for bucket in s3_client.buckets.values(): + if trail_bucket == bucket.name: + trail_bucket_is_in_account = True + if bucket.mfa_delete: + report.status = "PASS" + report.status_extended = f"Trail {trail.name} bucket ({trail_bucket}) has MFA delete enabled." + + # Handle cross-account scenarios + if not trail_bucket_is_in_account: + report.status = "MANUAL" + report.status_extended = f"Trail {trail.name} bucket ({trail_bucket}) is a cross-account bucket or out of Prowler's audit scope, please check it manually." + + findings.append(report) + return findings +``` + +**Key Benefits:** + +- **CloudTrail service** only contains CloudTrail-specific data (trails, configurations) +- **S3 service** only contains S3-specific data (buckets, policies, ACLs) +- **Check logic** orchestrates between services using their public client interfaces +- **Cross-account detection** is handled gracefully when resources span accounts + +### Service Consolidation Guidelines + +**When to combine services in the same file:** + +Implement multiple services as **separate classes in the same file** when two services are **practically the same** or one is a **direct extension** of another. + +**Example: S3 and S3Control** + +S3Control is an extension of S3 that provides account-level controls and access points. Both are implemented in `s3_service.py`: + +```python +# File: prowler/providers/aws/services/s3/s3_service.py + +class S3(AWSService): + """Standard S3 service for bucket operations""" + def __init__(self, provider): + super().__init__(__class__.__name__, provider) + self.buckets = {} + self.regions_with_buckets = [] + + # S3-specific initialization + self._list_buckets(provider) + self._get_bucket_versioning() + # ... other S3-specific operations + +class S3Control(AWSService): + """S3Control service for account-level and access point operations""" + def __init__(self, provider): + super().__init__(__class__.__name__, provider) + self.account_public_access_block = None + self.access_points = {} + + # S3Control-specific initialization + self._get_public_access_block() + self._list_access_points() + # ... other S3Control-specific operations +``` + +**Separate client files:** + +```python +# File: prowler/providers/aws/services/s3/s3_client.py +from prowler.providers.aws.services.s3.s3_service import S3 +s3_client = S3(Provider.get_global_provider()) + +# File: prowler/providers/aws/services/s3/s3control_client.py +from prowler.providers.aws.services.s3.s3_service import S3Control +s3control_client = S3Control(Provider.get_global_provider()) +``` + +**When NOT to consolidate services:** + +Keep services separate when they: + +- **Operate on different resource types** (EC2 vs RDS) +- **Have different authentication mechanisms** (different API endpoints) +- **Serve different operational domains** (IAM vs CloudTrail) +- **Have different regional behaviors** (global vs regional services) + +### Cross-Service Dependencies Guidelines + +**1. Always use client imports:** + +```python +# Correct pattern +from prowler.providers.aws.services.service_a.service_a_client import service_a_client +from prowler.providers.aws.services.service_b.service_b_client import service_b_client +``` + +**2. Handle missing resources gracefully:** + +```python +# Handle cross-service scenarios +resource_found_in_account = False +for external_resource in other_service_client.resources.values(): + if target_resource_id == external_resource.id: + resource_found_in_account = True + # Process found resource + break + +if not resource_found_in_account: + # Handle cross-account or missing resource scenarios + report.status = "MANUAL" + report.status_extended = "Resource is cross-account or out of audit scope" +``` + +**3. Document cross-service dependencies:** + +```python +class check_with_dependencies(Check): + """ + Check Description + + Dependencies: + - service_a_client: For primary resource information + - service_b_client: For related resource validation + - service_c_client: For policy analysis + """ +``` + +## Regional Service Implementation + +When implementing services for regional providers (like AWS, Azure, GCP), special considerations are needed to handle resource discovery across multiple geographic locations. This section provides a complete guide using AWS as the reference example. + +### Regional vs Non-Regional Services + +**Regional Services:** Require iteration across multiple geographic locations where resources may exist (e.g., EC2 instances, VPC, RDS databases). + +**Non-Regional/Global Services:** Operate at a global or tenant level without regional concepts (e.g., IAM users, Route53 hosted zones). + +### AWS Regional Implementation Example + +AWS is the perfect example of a regional provider. Here's how Prowler handles AWS's regional architecture: + + +```python +# File: prowler/providers/aws/services/ec2/ec2_service.py +class EC2(AWSService): + def __init__(self, provider): + super().__init__(__class__.__name__, provider) + self.instances = {} + self.security_groups = {} + + # Regional resource discovery across all AWS regions + self.__threading_call__(self._describe_instances) + self.__threading_call__(self._describe_security_groups) + + def _describe_instances(self, regional_client): + """Discover EC2 instances in a specific region""" + try: + describe_instances_paginator = regional_client.get_paginator("describe_instances") + for page in describe_instances_paginator.paginate(): + for reservation in page["Reservations"]: + for instance in reservation["Instances"]: + # Each instance includes its region + self.instances[instance["InstanceId"]] = Instance( + id=instance["InstanceId"], + region=regional_client.region, + state=instance["State"]["Name"], + # ... other properties + ) + except Exception as error: + logger.error(f"Failed to describe instances in {regional_client.region}: {error}") +``` + +#### Regional Check Execution + +```python +# File: prowler/providers/aws/services/ec2/ec2_instance_public_ip/ec2_instance_public_ip.py +class ec2_instance_public_ip(Check): + def execute(self): + findings = [] + + # Automatically iterates across ALL AWS regions where instances exist + for instance in ec2_client.instances.values(): + report = Check_Report_AWS(metadata=self.metadata(), resource=instance) + report.region = instance.region # Critical: region attribution + report.resource_arn = f"arn:aws:ec2:{instance.region}:{instance.account_id}:instance/{instance.id}" + + if instance.public_ip: + report.status = "FAIL" + report.status_extended = f"Instance {instance.id} in {instance.region} has public IP {instance.public_ip}" + else: + report.status = "PASS" + report.status_extended = f"Instance {instance.id} in {instance.region} does not have a public IP" + + findings.append(report) + + return findings +``` + +#### Key AWS Regional Features + +**Region-Specific ARNs:** + +``` +arn:aws:ec2:us-east-1:123456789012:instance/i-1234567890abcdef0 +arn:aws:s3:eu-west-1:123456789012:bucket/my-bucket +arn:aws:rds:ap-southeast-2:123456789012:db:my-database +``` + +**Parallel Processing:** + +- Each region processed independently in separate threads +- Failed regions don't affect other regions +- User can filter specific regions: `-f us-east-1` + +**Global vs Regional Services:** + +- **Regional**: EC2, RDS, VPC (require region iteration) +- **Global**: IAM, Route53, CloudFront (single `us-east-1` call) + +This architecture allows Prowler to efficiently scan AWS accounts with resources spread across multiple regions while maintaining performance and error isolation. + +### Regional Service Best Practices + +1. **Use Threading for Regional Discovery**: Leverage the `__threading_call__` method to parallelize resource discovery across regions +2. **Store Region Information**: Always include region metadata in resource objects for proper attribution +3. **Handle Regional Failures Gracefully**: Ensure that failures in one region don't affect others +4. **Optimize for Performance**: Use paginated calls and efficient data structures for large-scale resource discovery +5. **Support Region Filtering**: Allow users to limit scans to specific regions for focused audits + +## Best Practices + +- When available in the provider, use threading or parallelization utilities for all methods that can be parallelized by to maximize performance and reduce scan time. +- Define a Pydantic `BaseModel` for every resource you manage, and use these models for all resource data handling. +- Log every major step (start, success, error) in resource discovery and attribute collection for traceability and debugging; include as much context as possible. +- Catch and log all exceptions, providing detailed context (region, subscription, resource, error type, line number) to aid troubleshooting. +- Use consistent naming for resource containers, unique identifiers, and model attributes to improve code readability and maintainability. +- Add docstrings to every method and comments to explain any service-specific logic, especially where provider APIs behave differently or have quirks. +- Collect and store resource tags and additional attributes to support richer checks and reporting. +- Leverage shared utility helpers for session setup, identifier parsing, and other cross-cutting concerns to avoid code duplication. This kind of code is typically stored in a `lib` folder in the service folder. +- Keep code modular, maintainable, and well-documented for ease of extension and troubleshooting. +- **Each service should contain only information unique to that specific service** - use client objects for cross-service communication. +- **Handle cross-account and missing resources gracefully** when checks span multiple services. diff --git a/docs/developer-guide/unit-testing.md b/docs/developer-guide/unit-testing.mdx similarity index 96% rename from docs/developer-guide/unit-testing.md rename to docs/developer-guide/unit-testing.mdx index ab4e4e3bf7..8713988440 100644 --- a/docs/developer-guide/unit-testing.md +++ b/docs/developer-guide/unit-testing.mdx @@ -1,4 +1,6 @@ -# Unit Tests for Prowler Checks +--- +title: 'Unit Tests for Prowler Checks' +--- Unit tests for Prowler checks vary based on the provider being evaluated. @@ -39,7 +41,7 @@ To execute the Prowler test suite, install the necessary dependencies listed in ### Prerequisites -If you have not installed Prowler yet, refer to the [developer guide introduction](./introduction.md#getting-the-code-and-installing-all-dependencies). +If you have not installed Prowler yet, refer to the [developer guide introduction](/developer-guide/introduction#getting-the-code-and-installing-all-dependencies). ### Executing Tests @@ -57,16 +59,18 @@ Other Commands for Running Tests - Running tests for a provider check: `pytest -n auto -vvv -s -x tests/providers//services//` -???+ note - Refer to the [pytest documentation](https://docs.pytest.org/en/7.1.x/getting-started.html) for more details. + +Refer to the [pytest documentation](https://docs.pytest.org/en/7.1.x/getting-started.html) for more details. + ## AWS Testing Approaches For AWS provider, different testing approaches apply based on API coverage based on several criteria. -???+ note - Prowler leverages and contributes to the[Moto](https://github.com/getmoto/moto) library for mocking AWS infrastructure in tests. + +Prowler leverages and contributes to the[Moto](https://github.com/getmoto/moto) library for mocking AWS infrastructure in tests. + - AWS API Calls Covered by [Moto](https://github.com/getmoto/moto): - Service Tests: `@mock_aws` - Checks Tests: `@mock_aws` @@ -205,12 +209,14 @@ class Test_iam_password_policy_uppercase: If the IAM service required for testing is not supported by the Moto library, use [MagicMock](https://docs.python.org/3/library/unittest.mock.html#unittest.mock.MagicMock) to inject objects into the service client. -???+ warning - As stated above, direct service instantiation must be avoided to prevent actual AWS API calls. + +As stated above, direct service instantiation must be avoided to prevent actual AWS API calls. -???+ note - The example below demonstrates the IAM GetAccountPasswordPolicy API, which is covered by Moto, but is used for instructional purposes only. + + +The example below demonstrates the IAM GetAccountPasswordPolicy API, which is covered by Moto, but is used for instructional purposes only. + #### Mocking Service Objects Using MagicMock The following code demonstrates how to use MagicMock to create service objects. @@ -377,13 +383,15 @@ class Test_iam_password_policy_uppercase: # Refer to the previous section for the check test, as the implementation remains unchanged. ``` -???+ note - This example does not use Moto to simplify the setup. - However, if additional `moto` decorators are applied alongside the patch, Moto will automatically intercept the call to `orig(self, operation_name, kwarg)`. + +This example does not use Moto to simplify the setup. +However, if additional `moto` decorators are applied alongside the patch, Moto will automatically intercept the call to `orig(self, operation_name, kwarg)`. -???+ note - The source of the above implementation can be found here:[Patch Other Services with Moto](https://docs.getmoto.org/en/latest/docs/services/patching\_other\_services.html) + + +The source of the above implementation can be found here:[Patch Other Services with Moto](https://docs.getmoto.org/en/latest/docs/services/patching\_other\_services.html) + #### Mocking Several Services Since the provider is being mocked, multiple attributes can be configured to customize its behavior: @@ -488,7 +496,7 @@ will cause that the service will be initialised twice: Later, when importing `_client.py` at `.py`, Python uses the mocked instance since the patch was applied at the correct reference point. -In the [next section](./unit-testing.md#mocking-the-service-and-the-service-client-at-the-service-client-level) we will explore an improved approach to mock objects. +In the [next section](/developer-guide/unit-testing#mocking-the-service-and-the-service-client-at-the-service-client-level) we will explore an improved approach to mock objects. ##### Mocking the Service and the Service Client at the Service Client Level @@ -642,9 +650,10 @@ class Test_compute_project_os_login_enabled: The testing of Google Cloud Services follows the same principles as the one of Google Cloud checks. While all API calls must be mocked, attribute setup for API calls in this scenario is defined in the fixtures file, specifically within the [fixtures file](https://github.com/prowler-cloud/prowler/blob/master/tests/providers/gcp/gcp_fixtures.py) in the `mock_api_client` function. -???+ important - Every method within a service must be tested to ensure full coverage and accurate validation. + +Every method within a service must be tested to ensure full coverage and accurate validation. + The following example presents a real testing class, but includes additional comments for educational purposes, explaining key concepts and implementation details. ```python title="BigQuery Service Test" @@ -907,9 +916,12 @@ class Test_app_ensure_http_is_redirected_to_https: The testing of Azure Services follows the same principles as the one of Google Cloud checks. All API calls are still mocked, but for methods that initialize attributes via an API call, use the [patch](https://docs.python.org/3/library/unittest.mock.html#unittest.mock.patch) decorator at the beginning of the class to ensure proper mocking. -???+ important "Remember" - Every method within a service must be tested to ensure full coverage and accurate validation. + +**Remember** +Every method within a service must be tested to ensure full coverage and accurate validation. + + The following example presents a real testing class, but includes additional comments for educational purposes, explaining key concepts and implementation details. ```python title="AppInsights Service Test" diff --git a/docs/docs.json b/docs/docs.json new file mode 100644 index 0000000000..7d3f795482 --- /dev/null +++ b/docs/docs.json @@ -0,0 +1,433 @@ +{ + "$schema": "https://mintlify.com/docs.json", + "theme": "mint", + "name": "Prowler Documentation", + "colors": { + "primary": "#000000", + "light": "#10B981", + "dark": "#10B981" + }, + "favicon": "/favicon.ico", + "logo": { + "dark": "/images/prowler-logo-white.png", + "light": "/images/prowler-logo-black.png" + }, + "navigation": { + "tabs": [ + { + "tab": "Getting Started", + "groups": [ + { + "group": "Welcome", + "pages": [ + "introduction" + ] + }, + { + "group": "Prowler Cloud", + "pages": [ + "getting-started/products/prowler-cloud", + "getting-started/products/prowler-cloud-pricing", + "getting-started/products/prowler-cloud-aws-marketplace", + "getting-started/goto/prowler-cloud", + "getting-started/goto/prowler-api-reference" + ] + }, + { + "group": "Prowler CLI", + "pages": [ + "getting-started/products/prowler-cli", + "getting-started/installation/prowler-cli", + "getting-started/basic-usage/prowler-cli" + ] + }, + { + "group": "Prowler App", + "pages": [ + "getting-started/products/prowler-app", + "getting-started/installation/prowler-app", + "getting-started/basic-usage/prowler-app" + ] + }, + { + "group": "Prowler Lighthouse AI", + "pages": [ + "user-guide/tutorials/prowler-app-lighthouse" + ] + }, + { + "group": "Prowler MCP Server", + "pages": [ + "getting-started/products/prowler-mcp", + "getting-started/installation/prowler-mcp", + "getting-started/basic-usage/prowler-mcp", + "getting-started/basic-usage/prowler-mcp-tools" + ] + }, + { + "group": "Prowler Hub", + "pages": [ + "getting-started/products/prowler-hub", + "getting-started/goto/prowler-hub" + ] + }, + { + "group": "Prowler vs. Others", + "pages": [ + "getting-started/comparison/index", + "getting-started/comparison/awssecurityhub", + "getting-started/comparison/gcp", + "getting-started/comparison/microsoftdefender", + "getting-started/comparison/microsoftsentinel" + ] + } + ] + }, + { + "tab": "Guides", + "groups": [ + { + "group": "Prowler Cloud/App", + "pages": [ + "user-guide/tutorials/prowler-app", + { + "group": "Authentication", + "pages": [ + "user-guide/tutorials/prowler-app-social-login", + "user-guide/tutorials/prowler-app-sso" + ] + }, + "user-guide/tutorials/prowler-app-rbac", + "user-guide/tutorials/prowler-app-api-keys", + "user-guide/tutorials/prowler-app-mute-findings", + { + "group": "Integrations", + "expanded": true, + "pages": [ + "user-guide/tutorials/prowler-app-s3-integration", + "user-guide/tutorials/prowler-app-security-hub-integration", + "user-guide/tutorials/prowler-app-jira-integration" + ] + }, + "user-guide/tutorials/prowler-app-lighthouse", + "user-guide/tutorials/prowler-cloud-public-ips", + { + "group": "Tutorials", + "pages": [ + "user-guide/tutorials/prowler-app-sso-entra", + "user-guide/tutorials/bulk-provider-provisioning", + "user-guide/tutorials/aws-organizations-bulk-provisioning" + ] + } + ] + }, + { + "group": "CLI", + "pages": [ + "user-guide/cli/tutorials/misc", + "user-guide/cli/tutorials/reporting", + "user-guide/cli/tutorials/compliance", + "user-guide/cli/tutorials/dashboard", + "user-guide/cli/tutorials/configuration_file", + "user-guide/cli/tutorials/logging", + "user-guide/cli/tutorials/mutelist", + { + "group": "Integrations", + "pages": [ + "user-guide/providers/aws/securityhub", + "user-guide/cli/tutorials/integrations", + "user-guide/providers/aws/s3" + ] + }, + "user-guide/cli/tutorials/fixer", + "user-guide/cli/tutorials/check-aliases", + "user-guide/cli/tutorials/custom-checks-metadata", + "user-guide/cli/tutorials/pentesting", + "user-guide/cli/tutorials/scan-unused-services", + "user-guide/cli/tutorials/quick-inventory", + { + "group": "Tutorials", + "pages": [ + "user-guide/cli/tutorials/parallel-execution" + ] + } + ] + }, + { + "group": "Providers", + "pages": [ + { + "group": "AWS", + "pages": [ + "user-guide/providers/aws/getting-started-aws", + "user-guide/providers/aws/authentication", + "user-guide/providers/aws/role-assumption", + "user-guide/providers/aws/organizations", + "user-guide/providers/aws/regions-and-partitions", + "user-guide/providers/aws/tag-based-scan", + "user-guide/providers/aws/resource-arn-based-scan", + "user-guide/providers/aws/boto3-configuration", + "user-guide/providers/aws/threat-detection", + "user-guide/providers/aws/cloudshell", + "user-guide/providers/aws/multiaccount" + ] + }, + { + "group": "Azure", + "pages": [ + "user-guide/providers/azure/getting-started-azure", + "user-guide/providers/azure/authentication", + "user-guide/providers/azure/use-non-default-cloud", + "user-guide/providers/azure/subscriptions", + "user-guide/providers/azure/create-prowler-service-principal" + ] + }, + { + "group": "Google Cloud", + "pages": [ + "user-guide/providers/gcp/getting-started-gcp", + "user-guide/providers/gcp/authentication", + "user-guide/providers/gcp/projects", + "user-guide/providers/gcp/organization", + "user-guide/providers/gcp/retry-configuration" + ] + }, + { + "group": "Kubernetes", + "pages": [ + "user-guide/providers/kubernetes/getting-started-k8s", + "user-guide/providers/kubernetes/misc" + ] + }, + { + "group": "Microsoft 365", + "pages": [ + "user-guide/providers/microsoft365/getting-started-m365", + "user-guide/providers/microsoft365/authentication", + "user-guide/providers/microsoft365/use-of-powershell" + ] + }, + { + "group": "GitHub", + "pages": [ + "user-guide/providers/github/getting-started-github", + "user-guide/providers/github/authentication" + ] + }, + { + "group": "IaC", + "pages": [ + "user-guide/providers/iac/getting-started-iac", + "user-guide/providers/iac/authentication" + ] + }, + { + "group": "MongoDB Atlas", + "pages": [ + "user-guide/providers/mongodbatlas/getting-started-mongodbatlas", + "user-guide/providers/mongodbatlas/authentication" + ] + }, + { + "group": "LLM", + "pages": [ + "user-guide/providers/llm/getting-started-llm" + ] + }, + { + "group": "Oracle Cloud Infrastructure", + "pages": [ + "user-guide/providers/oci/getting-started-oci", + "user-guide/providers/oci/authentication" + ] + } + ] + }, + { + "group": "Compliance", + "pages": [ + "user-guide/compliance/tutorials/threatscore" + ] + } + ] + }, + { + "tab": "Developer Guide", + "groups": [ + { + "group": "Concepts", + "pages": [ + "developer-guide/introduction", + "developer-guide/provider", + "developer-guide/services", + "developer-guide/checks", + "developer-guide/outputs", + "developer-guide/integrations", + "developer-guide/security-compliance-framework", + "developer-guide/lighthouse" + ] + }, + { + "group": "Providers", + "pages": [ + "developer-guide/aws-details", + "developer-guide/azure-details", + "developer-guide/gcp-details", + "developer-guide/kubernetes-details", + "developer-guide/m365-details", + "developer-guide/github-details", + "developer-guide/llm-details" + ] + }, + { + "group": "Miscellaneous", + "pages": [ + "developer-guide/documentation", + { + "group": "Testing", + "pages": [ + "developer-guide/unit-testing", + "developer-guide/integration-testing" + ] + }, + "developer-guide/debugging", + "developer-guide/configurable-checks", + "developer-guide/renaming-checks", + "developer-guide/check-metadata-guidelines" + ] + } + ] + }, + { + "tab": "Security", + "pages": [ + "security" + ] + }, + { + "tab": "Contact Us", + "pages": [ + "contact" + ] + }, + { + "tab": "Troubleshooting", + "pages": [ + "troubleshooting" + ] + }, + { + "tab": "About Us", + "icon": "/favicon.ico", + "href": "https://prowler.com/about#team" + }, + { + "tab": "Changelog", + "icon": "github", + "href": "https://github.com/prowler-cloud/prowler/releases" + }, + { + "tab": "Public Roadmap", + "href": "https://roadmap.prowler.com/" + } + ], + "global": { + "anchors": [ + { + "anchor": "GitHub", + "href": "https://github.com/prowler-cloud/prowler", + "icon": "github" + }, + { + "anchor": "Slack", + "href": "https://goto.prowler.com/slack", + "icon": "slack" + }, + { + "anchor": "YouTube", + "href": "https://www.youtube.com/@prowlercloud", + "icon": "youtube" + } + ] + } + }, + "navbar": { + "links": [ + { + "label": "Prowler Hub", + "href": "https://hub.prowler.com" + }, + { + "label": "Prowler Cloud", + "href": "https://cloud.prowler.com", + "style": "primary" + } + ] + }, + "analytics": { + "ga4": { + "measurementId": "G-KBKV70W5Y2" + } + }, + "feedback": { + "thumbsRating": true, + "suggestEdit": true, + "raiseIssue": true + }, + "footer": { + "socials": { + "x-twitter": "https://x.com/prowlercloud", + "github": "https://github.com/prowler-cloud/prowler", + "linkedin": "https://www.linkedin.com/company/prowler-security", + "youtube": "https://www.youtube.com/@prowlercloud", + "slack": "https://goto.prowler.com/slack", + "website": "https://prowler.com" + } + }, + "redirects": [ + { + "source": "/projects/prowler-open-source/en/latest/tutorials/prowler-app-lighthouse", + "destination": "/user-guide/tutorials/prowler-app-lighthouse" + }, + { + "source": "/projects/prowler-open-source/en/latest/developer-guide/introduction", + "destination": "/developer-guide/introduction" + }, + { + "source": "/projects/prowler-open-source/en/latest/tutorials/aws/getting-started-aws", + "destination": "/user-guide/providers/aws/getting-started-aws" + }, + { + "source": "/projects/prowler-open-source/en/latest/tutorials/azure/getting-started-azure", + "destination": "/user-guide/providers/azure/getting-started-azure" + }, + { + "source": "/projects/prowler-open-source/en/latest/tutorials/gcp/getting-started-gcp", + "destination": "/user-guide/providers/gcp/getting-started-gcp" + }, + { + "source": "/projects/prowler-open-source/en/latest/tutorials/microsoft365/getting-started-m365", + "destination": "/user-guide/providers/microsoft365/getting-started-m365" + }, + { + "source": "/projects/prowler-open-source/en/latest/tutorials/github/getting-started-github", + "destination": "/user-guide/providers/github/getting-started-github" + }, + { + "source": "/projects/prowler-open-source/en/latest/tutorials/prowler-app-sso", + "destination": "/user-guide/tutorials/prowler-app-sso" + }, + { + "source": "/projects/prowler-open-source/en/latest", + "destination": "/introduction" + }, + { + "source": "/projects/prowler-saas/en/latest/:slug*", + "destination": "https://docs.prowler.pro/en/latest/:slug*" + }, + { + "source": "/projects/prowler-open-source/en/latest/tutorials/:slug*", + "destination": "/user-guide/tutorials/:slug*" + } + ] +} diff --git a/docs/getting-started/basic-usage/prowler-app.mdx b/docs/getting-started/basic-usage/prowler-app.mdx new file mode 100644 index 0000000000..26d7d8ee2e --- /dev/null +++ b/docs/getting-started/basic-usage/prowler-app.mdx @@ -0,0 +1,70 @@ +--- +title: 'Basic Usage' +--- + +## Access Prowler App + +After [installation](/getting-started/installation/prowler-app), navigate to [http://localhost:3000](http://localhost:3000) and sign up with email and password. + +Sign Up Button +Sign Up + + +**User creation and default tenant behavior** + + +When creating a new user, the behavior depends on whether an invitation is provided: + +- **Without an invitation**: + + - A new tenant is automatically created. + - The new user is assigned to this tenant. + - A set of **RBAC admin permissions** is generated and assigned to the user for the newly-created tenant. + +- **With an invitation**: The user is added to the specified tenant with the permissions defined in the invitation. + +This mechanism ensures that the first user in a newly created tenant has administrative permissions within that tenant. + + +## Log In + +Access Prowler App by logging in with **email and password**. + +Log In + +## Add Cloud Provider + +Configure a cloud provider for scanning: + +1. Navigate to `Settings > Cloud Providers` and click `Add Account`. +2. Select the cloud provider. +3. Enter the provider's identifier (Optional: Add an alias): + - **AWS**: Account ID + - **GCP**: Project ID + - **Azure**: Subscription ID + - **Kubernetes**: Cluster ID + - **M365**: Domain ID +4. Follow the guided instructions to add and authenticate your credentials. + +## Start a Scan + +Once credentials are successfully added and validated, Prowler initiates a scan of your cloud environment. + +Click `Go to Scans` to monitor progress. + +## View Results + +Review findings during scan execution in the following sections: + +- **Overview** – Provides a high-level summary of your scans. + Overview + +- **Compliance** – Displays compliance insights based on security frameworks. + Compliance + +> For detailed usage instructions, refer to the [Prowler App Guide](/user-guide/tutorials/prowler-app). + + +Prowler will automatically scan all configured providers every **24 hours**, ensuring your cloud environment stays continuously monitored. + + diff --git a/docs/basic-usage/prowler-cli.md b/docs/getting-started/basic-usage/prowler-cli.mdx similarity index 75% rename from docs/basic-usage/prowler-cli.md rename to docs/getting-started/basic-usage/prowler-cli.mdx index 0b50d16564..ae8d7cd947 100644 --- a/docs/basic-usage/prowler-cli.md +++ b/docs/getting-started/basic-usage/prowler-cli.mdx @@ -1,18 +1,24 @@ +--- +title: 'Basic Usage' +--- + ## Running Prowler Running Prowler requires specifying the provider (e.g `aws`, `gcp`, `azure`, `kubernetes`, `m365`, `github`, `iac` or `mongodbatlas`): -???+ note - If no provider is specified, AWS is used by default for backward compatibility with Prowler v2. + +If no provider is specified, AWS is used by default for backward compatibility with Prowler v2. + ```console prowler ``` -![Prowler Execution](../img/short-display.png) +![Prowler Execution](/images/short-display.png) -???+ note - Running the `prowler` command without options will uses environment variable credentials. Refer to the Authentication section of each provider for credential configuration details. + +Running the `prowler` command without options will uses environment variable credentials. Refer to the Authentication section of each provider for credential configuration details. + ## Verbose Output If you prefer the former verbose output, use: `--verbose`. This allows seeing more info while Prowler is running, minimal output is displayed unless verbosity is enabled. @@ -26,7 +32,7 @@ prowler -M csv json-asff json-ocsf html ``` The HTML report is saved in the output directory, alongside other reports. It will look like this: -![Prowler Execution](../img/html-output.png) +![Prowler Execution](/images/html-output.png) ## Listing Available Checks and Services @@ -58,7 +64,7 @@ prowler kubernetes --excluded-services controllermanager ``` ## Additional Options -Explore more advanced time-saving execution methods in the [Miscellaneous](../tutorials/misc.md) section. +Explore more advanced time-saving execution methods in the [Miscellaneous](/user-guide/cli/tutorials/misc) section. Access the help menu and view all available options with `-h`/`--help`: @@ -74,10 +80,11 @@ Use a custom AWS profile with `-p`/`--profile` and/or specific AWS regions with prowler aws --profile custom-profile -f us-east-1 eu-south-2 ``` -???+ note - By default, `prowler` will scan all AWS regions. + +By default, `prowler` will scan all AWS regions. -See more details about AWS Authentication in the [Authentication Section](../tutorials/aws/authentication.md) section. + +See more details about AWS Authentication in the [Authentication Section](/user-guide/providers/aws/authentication) section. ## Azure @@ -97,7 +104,7 @@ prowler azure --browser-auth --tenant-id "XXXXXXXX" prowler azure --managed-identity-auth ``` -See more details about Azure Authentication in the [Authentication Section](../tutorials/azure/authentication.md) +See more details about Azure Authentication in the [Authentication Section](/user-guide/providers/azure/authentication) By default, Prowler scans all accessible subscriptions. Scan specific subscriptions using the following flag (using az cli auth as example): @@ -154,9 +161,10 @@ Prowler enables security scanning of Kubernetes clusters, supporting both **in-c ```console prowler kubernetes --kubeconfig-file path ``` - ???+ note + If no `--kubeconfig-file` is provided, Prowler will use the default KubeConfig file location (`~/.kube/config`). + - **In-Cluster Execution** To run Prowler inside the cluster, apply the provided YAML configuration to deploy a job in a new namespace: @@ -170,9 +178,10 @@ Prowler enables security scanning of Kubernetes clusters, supporting both **in-c kubectl logs prowler-XXXXX --namespace prowler-ns ``` - ???+ note + By default, Prowler scans all namespaces in the active Kubernetes context. Use the `--context`flag to specify the context to be scanned and `--namespaces` to restrict scanning to specific namespaces. + ## Microsoft 365 Microsoft 365 requires specifying the auth method: @@ -190,7 +199,7 @@ prowler m365 --browser-auth --tenant-id "XXXXXXXX" ``` -See more details about M365 Authentication in the [Authentication Section](../tutorials/microsoft365/authentication.md) section. +See more details about M365 Authentication in the [Authentication Section](/user-guide/providers/microsoft365/authentication) section. ## GitHub @@ -212,7 +221,7 @@ Prowler enables security scanning of your **GitHub account**, including **Reposi prowler github --github-app-id app_id --github-app-key $APP_KEY_CONTENT ``` - ???+ note + If no login method is explicitly provided, Prowler will automatically attempt to authenticate using environment variables in the following order of precedence: 1. `GITHUB_PERSONAL_ACCESS_TOKEN` @@ -220,6 +229,7 @@ Prowler enables security scanning of your **GitHub account**, including **Reposi 3. `GITHUB_APP_ID` and `GITHUB_APP_KEY_PATH` 4. `GITHUB_APP_ID` and `GITHUB_APP_KEY` + ## Infrastructure as Code (IaC) Prowler's Infrastructure as Code (IaC) provider enables you to scan local or remote infrastructure code for security and compliance issues using [Trivy](https://trivy.dev/). This provider supports a wide range of IaC frameworks, allowing you to assess your code before deployment. @@ -246,14 +256,15 @@ prowler iac --scan-path ./my-iac-directory --frameworks terraform kubernetes prowler iac --scan-path ./my-iac-directory --exclude-path ./my-iac-directory/test,./my-iac-directory/examples ``` -???+ note - - `--scan-path` and `--scan-repository-url` are mutually exclusive; only one can be specified at a time. - - For remote repository scans, authentication can be provided via CLI flags or environment variables (`GITHUB_OAUTH_APP_TOKEN`, `GITHUB_USERNAME`, `GITHUB_PERSONAL_ACCESS_TOKEN`). CLI flags take precedence. - - The IaC provider does not require cloud authentication for local scans. - - It is ideal for CI/CD pipelines and local development environments. - - For more details on supported scanners, see the [Trivy documentation](https://trivy.dev/latest/docs/scanner/vulnerability/) + +- `--scan-path` and `--scan-repository-url` are mutually exclusive; only one can be specified at a time. +- For remote repository scans, authentication can be provided via CLI flags or environment variables (`GITHUB_OAUTH_APP_TOKEN`, `GITHUB_USERNAME`, `GITHUB_PERSONAL_ACCESS_TOKEN`). CLI flags take precedence. +- The IaC provider does not require cloud authentication for local scans. +- It is ideal for CI/CD pipelines and local development environments. +- For more details on supported scanners, see the [Trivy documentation](https://trivy.dev/latest/docs/scanner/vulnerability/) -See more details about IaC scanning in the [IaC Tutorial](../tutorials/iac/getting-started-iac.md) section. + +See more details about IaC scanning in the [IaC Tutorial](/user-guide/providers/iac/getting-started-iac) section. ## MongoDB Atlas @@ -278,4 +289,30 @@ You can filter scans to specific organizations or projects: prowler mongodbatlas --atlas-project-id ``` -See more details about MongoDB Atlas Authentication in [MongoDB Atlas Authentication](../tutorials/mongodbatlas/authentication.md) +See more details about MongoDB Atlas Authentication in [MongoDB Atlas Authentication](/user-guide/providers/mongodbatlas/authentication) + +## Oracle Cloud + +Prowler allows you to scan your Oracle Cloud deployments for security and compliance issues. + +You have two options to authenticate: + +1. OCI Config File Authentication: this config file can be generated using the OCI CLI with the `oci session authenticate` command or created manually using the OCI Console. For more details, see the [OCI Authentication Guide](/user-guide/providers/oci/authentication#oci-session-authentication). + + ```console + prowler oci + ``` + + You can add different profiles to the config file to scan different tenancies or regions. In order to scan a specific profile, use the `--profile` flag: + + ```console + prowler oci --profile + ``` + +2. Instance Principal Authentication: when running Prowler on an OCI Compute instance, you can use Instance Principal authentication. For more details, see the [OCI Authentication Guide](/user-guide/providers/oci/authentication#instance-principal-authentication). + + ```console + prowler oci --use-instance-principal + ``` + +See more details about Oracle Cloud Authentication in [Oracle Cloud Authentication](/user-guide/providers/oci/authentication) diff --git a/docs/getting-started/basic-usage/prowler-mcp-tools.mdx b/docs/getting-started/basic-usage/prowler-mcp-tools.mdx new file mode 100644 index 0000000000..dc984c9537 --- /dev/null +++ b/docs/getting-started/basic-usage/prowler-mcp-tools.mdx @@ -0,0 +1,122 @@ +--- +title: "Tools Reference" +--- + +Complete reference guide for all tools available in the Prowler MCP Server. Tools are organized by namespace. + +## Tool Categories Summary + +| Category | Tool Count | Authentication Required | +|----------|------------|------------------------| +| Prowler Hub | 10 tools | No | +| Prowler Documentation | 2 tools | No | +| Prowler Cloud/App | 28 tools | Yes | + +## Tool Naming Convention + +All tools follow a consistent naming pattern with prefixes: + +- `prowler_hub_*` - Prowler Hub catalog and compliance tools +- `prowler_docs_*` - Prowler documentation search and retrieval +- `prowler_app_*` - Prowler Cloud and App (Self-Managed) management tools + +## Prowler Hub Tools + +Access Prowler's security check catalog and compliance frameworks. **No authentication required.** + +### Check Discovery + +- **`prowler_hub_get_checks`** - List security checks with advanced filtering options +- **`prowler_hub_get_check_filters`** - Return available filter values for checks (providers, services, severities, categories, compliances) +- **`prowler_hub_search_checks`** - Full-text search across check metadata +- **`prowler_hub_get_check_raw_metadata`** - Fetch raw check metadata in JSON format + +### Check Code + +- **`prowler_hub_get_check_code`** - Fetch the Python implementation code for a security check +- **`prowler_hub_get_check_fixer`** - Fetch the automated fixer code for a check (if available) + +### Compliance Frameworks + +- **`prowler_hub_get_compliance_frameworks`** - List and filter compliance frameworks +- **`prowler_hub_search_compliance_frameworks`** - Full-text search across compliance frameworks + +### Provider Information + +- **`prowler_hub_list_providers`** - List Prowler official providers and their services +- **`prowler_hub_get_artifacts_count`** - Get total count of checks and frameworks in Prowler Hub + +## Prowler Documentation Tools + +Search and access official Prowler documentation. **No authentication required.** + +- **`prowler_docs_search`** - Search the official Prowler documentation using full-text search +- **`prowler_docs_get_document`** - Retrieve the full markdown content of a specific documentation file + +## Prowler Cloud/App Tools + +Manage Prowler Cloud or Prowler App (Self-Managed) features. **Requires authentication.** + + +These tools require a valid API key. See the [Configuration Guide](/getting-started/basic-usage/prowler-mcp) for authentication setup. + + +### Findings Management + +- **`prowler_app_list_findings`** - List security findings with advanced filtering +- **`prowler_app_get_finding`** - Get detailed information about a specific finding +- **`prowler_app_get_latest_findings`** - Retrieve latest findings from the most recent scans +- **`prowler_app_get_findings_metadata`** - Get unique metadata values from filtered findings +- **`prowler_app_get_latest_findings_metadata`** - Get metadata from latest findings across all providers + +### Provider Management + +- **`prowler_app_list_providers`** - List all providers with filtering options +- **`prowler_app_create_provider`** - Create a new provider in the current tenant +- **`prowler_app_get_provider`** - Get detailed information about a specific provider +- **`prowler_app_update_provider`** - Update provider details (alias, etc.) +- **`prowler_app_delete_provider`** - Delete a specific provider +- **`prowler_app_test_provider_connection`** - Test provider connection status + +### Provider Secrets Management + +- **`prowler_app_list_provider_secrets`** - List all provider secrets with filtering +- **`prowler_app_add_provider_secret`** - Add or update credentials for a provider +- **`prowler_app_get_provider_secret`** - Get detailed information about a provider secret +- **`prowler_app_update_provider_secret`** - Update provider secret details +- **`prowler_app_delete_provider_secret`** - Delete a provider secret + +### Scan Management + +- **`prowler_app_list_scans`** - List all scans with filtering options +- **`prowler_app_create_scan`** - Trigger a manual scan for a specific provider +- **`prowler_app_get_scan`** - Get detailed information about a specific scan +- **`prowler_app_update_scan`** - Update scan details +- **`prowler_app_get_scan_compliance_report`** - Download compliance report as CSV +- **`prowler_app_get_scan_report`** - Download ZIP file containing complete scan report + +### Schedule Management + +- **`prowler_app_schedules_daily_scan`** - Create a daily scheduled scan for a provider + +### Processor Management + +- **`prowler_app_processors_list`** - List all processors with filtering +- **`prowler_app_processors_create`** - Create a new processor (currently only mute lists supported) +- **`prowler_app_processors_retrieve`** - Get processor details by ID +- **`prowler_app_processors_partial_update`** - Update processor configuration +- **`prowler_app_processors_destroy`** - Delete a processor + +## Usage Tips + +- Use natural language to interact with the tools through your AI assistant +- Tools can be combined for complex workflows +- Filter options are available on most list tools +- Authentication is only required for Prowler Cloud/App tools + +## Additional Resources + +- [MCP Protocol Specification](https://modelcontextprotocol.io) +- [Prowler API Documentation](https://api.prowler.com/api/v1/docs) +- [Prowler Hub API](https://hub.prowler.com/api/docs) +- [GitHub Repository](https://github.com/prowler-cloud/prowler) diff --git a/docs/getting-started/basic-usage/prowler-mcp.mdx b/docs/getting-started/basic-usage/prowler-mcp.mdx new file mode 100644 index 0000000000..b6d59093e7 --- /dev/null +++ b/docs/getting-started/basic-usage/prowler-mcp.mdx @@ -0,0 +1,291 @@ +--- +title: "Configuration" +--- + +Configure your MCP client to connect to Prowler MCP Server. + +## Step 1: Get Your API Key + + +**Authentication is optional**: Prowler Hub and Prowler Documentation features work without authentication. An API key is only required for Prowler Cloud and Prowler App (Self-Managed) features. + + +To use Prowler Cloud or Prowler App (Self-Managed) features. To get the API key, please refer to the [API Keys](/user-guide/tutorials/prowler-app-api-keys) guide. + + +Keep the API key secure. Never share it publicly or commit it to version control. + + +## Step 2: Configure Your MCP Host/Client + +Choose the configuration based on your deployment: + +- **HTTP Mode**: Prowler Cloud MCP Server or self-hosted Prowler MCP Server. +- **STDIO Mode**: Local installation only (runs as subprocess of your MCP client). + +### HTTP Mode + + + + **Configuration:** + ```json + { + "mcpServers": { + "prowler": { + "url": "https://mcp.prowler.com/mcp", // or your self-hosted Prowler MCP Server URL + "headers": { + "Authorization": "Bearer " + } + } + } + } + ``` + + + + **Configuration:** + ```json + { + "mcpServers": { + "prowler": { + "command": "npx", + "args": [ + "mcp-remote", + "https://mcp.prowler.com/mcp", // or your self-hosted Prowler MCP Server URL + "--header", + "Authorization: Bearer ${PROWLER_APP_API_KEY}" + ], + "env": { + "PROWLER_APP_API_KEY": "" + } + } + } + } + ``` + + The `mcp-remote` tool acts as a bridge for clients that don't support HTTP natively. Learn more at [mcp-remote on npm](https://www.npmjs.com/package/mcp-remote). + + + + + 1. Open Claude Desktop settings + 2. Go to "Developer" tab + 3. Click in "Edit Config" button + 4. Edit the `claude_desktop_config.json` file with your favorite editor + 5. Add the following configuration: + ```json + { + "mcpServers": { + "prowler": { + "command": "npx", + "args": [ + "mcp-remote", + "https://mcp.prowler.com/mcp", + "--header", + "Authorization: Bearer ${PROWLER_APP_API_KEY}" + ], + "env": { + "PROWLER_APP_API_KEY": "" + } + } + } + } + ``` + + + + Run the following command: + ```bash + export PROWLER_APP_API_KEY="" + claude mcp add --transport http prowler https://mcp.prowler.com/mcp --header "Authorization: Bearer $PROWLER_APP_API_KEY" --scope user + ``` + + + + 1. Open Cursor settings + 2. Go to "Tools & MCP" + 3. Click in "New MCP Server" button + 4. Add to the JSON Configuration the following: + ```json + { + "mcpServers": { + "prowler": { + "url": "https://mcp.prowler.com/mcp", + "headers": { + "Authorization": "Bearer " + } + } + } + } + ``` + + + + + +### STDIO Mode + +STDIO mode is only available when running the MCP server locally. + + + + **Run from source or local installation** + + ```json + { + "mcpServers": { + "prowler": { + "command": "uvx", + "args": ["/absolute/path/to/prowler/mcp_server/"], + "env": { + "PROWLER_APP_API_KEY": "", + "PROWLER_API_BASE_URL": "https://api.prowler.com" + } + } + } + } + ``` + + + Replace `/absolute/path/to/prowler/mcp_server/` with the actual path. The `PROWLER_API_BASE_URL` is optional and defaults to Prowler Cloud API. + + + + + + **Run with Docker image** + + ```json + { + "mcpServers": { + "prowler": { + "command": "docker", + "args": [ + "run", + "--rm", + "-i", + "--env", + "PROWLER_APP_API_KEY=", + "--env", + "PROWLER_API_BASE_URL=https://api.prowler.com", + "prowlercloud/prowler-mcp" + ] + } + } + } + ``` + + + The `PROWLER_API_BASE_URL` is optional and defaults to Prowler Cloud API. + + + + + +## Step 3: Start Using Prowler MCP + +Restart your MCP client and start asking questions: +- *"Show me all critical findings from my AWS accounts"* +- *"What does the S3 bucket public access check do?"* +- *"Onboard this new AWS account in my Prowler Organization"* + +## Authentication Methods + +Prowler MCP Server supports two authentication methods to connect to Prowler Cloud or Prowler App (Self-Managed): + +### API Key (Recommended) + +Use your Prowler API key directly in the Bearer token: + +``` +Authorization: Bearer +``` + +This is the recommended method for most users. + +### JWT Token + +Alternatively, obtain a JWT token from Prowler: + +```bash +curl -X POST https://api.prowler.com/api/v1/tokens \ + -H "Content-Type: application/vnd.api+json" \ + -H "Accept: application/vnd.api+json" \ + -d '{ + "data": { + "type": "tokens", + "attributes": { + "email": "your-email@example.com", + "password": "your-password" + } + } + }' +``` + +Use the returned JWT token in place of the API key: + +``` +Authorization: Bearer eyJhbGciOiJIUzI1NiIs... +``` + + +JWT tokens are only valid for 30 minutes. You need to generate a new token if you want to continue using the MCP server. + + +## Troubleshooting + +### Server Not Detected + +- Restart your MCP client after configuration changes +- Check the configuration file syntax (valid JSON) +- Review client logs for specific error messages +- Verify the server URL is correct + +### Authentication Failures + +**Error: Unauthorized (401)** +- Verify your API key is correct +- Ensure the key hasn't expired +- Check you're using the right API endpoint + +### Connection Issues + +**Cannot Reach Server:** +- Verify the server URL is correct +- Check network connectivity +- For local servers, ensure the server is running +- Check firewall settings + +## Security Best Practices + +1. **Protect Your API Key** + - Never commit API keys to version control. + - Use environment variables or secure vaults. + - Rotate keys regularly. + +2. **Network Security** + - Use HTTPS for production deployments. + - Restrict network access to the MCP server. + - Consider VPN for remote access. + +3. **Least Privilege** + - API key gives the permission of the user who created the key, make sure to use the key with the minimal required permissions. + - Review the tools that are gonna be used and how they are gonna be used to avoid prompt injections or unintended behavior. + +## Next Steps + +Now that your MCP server is configured: + + + + Explore all available tools + + + +## Getting Help + +Need assistance with configuration? + +- Search for existing [GitHub issues](https://github.com/prowler-cloud/prowler/issues) +- Ask for help in our [Slack community](https://goto.prowler.com/slack) +- Report a new issue on [GitHub](https://github.com/prowler-cloud/prowler/issues/new) diff --git a/docs/getting-started/comparison/awssecurityhub.mdx b/docs/getting-started/comparison/awssecurityhub.mdx new file mode 100644 index 0000000000..4b5d743dbe --- /dev/null +++ b/docs/getting-started/comparison/awssecurityhub.mdx @@ -0,0 +1,94 @@ +--- +title: 'AWS Security Hub' +--- + +AWS Security Hub remains a managed service designed for centralizing security alerts and compliance status within AWS environments. It integrates with various AWS security services and provides a consolidated view of security findings. + +## Key Features and Strengths + +- **Centralized Dashboard for AWS:** Provides a single pane of glass to monitor and manage security findings from multiple AWS services like GuardDuty, Inspector, and Config. + +- **Compliance Checks:** Automatically checks for compliance against standards like CIS and PCI DSS within AWS environments. + +- **AWS Native Automation:** Offers seamless automation for incident response using AWS Lambda and CloudWatch Events, reducing the time to react to security issues. + +- **User-Friendly Interface:** Accessible via the AWS Management Console, offering a streamlined experience for managing security across AWS accounts. + +## Limitations + +- **AWS-Centric:** Limited to AWS environments, with no direct support for multi-cloud or hybrid environments. + +- **Dependency on AWS Config:** Some of its checks depend on AWS Config, which may not be enabled in all regions or accounts. + +- **Vendor Lock-In:** Tightly coupled with AWS, making it less suitable for organizations with a cloud-agnostic strategy. + +## Prowler + +Prowler is an open-source, multi-cloud security tool that offers extensive customization and flexibility, making it ideal for organizations with complex or multi-cloud environments. Here are the updated features and advantages: + +## Main Advantages of Prowler + +- **Multi-Region and Multi-Account Scanning by Default:** + - Prowler is inherently multi-region and can scan multiple AWS accounts without requiring additional configuration or enabling specific services like AWS Config. + +- **Minimal Setup Requirements:** + - All Prowler needs is a role with appropriate permissions to start scanning. There’s no need to enable specific services or configure complex setups. + +- **Versatile Execution Environment:** + - Prowler can be run from various environments, including a local workstation, container, AWS CloudShell, or even from another AWS account or cloud provider by assuming a role. This flexibility makes it easy to integrate into different operational workflows. + +- **Flexible Results Storage and Sharing:** + - Prowler results can be stored directly into an S3 bucket, allowing for quick analysis, or locally for easy sharing and discussion. This flexibility is particularly useful for collaborative security assessments. + +- **Customizable Reporting and Analysis:** + - Prowler supports exporting results in multiple formats, including JSON, CSV, OCSF format, and static HTML reports. It also supports integration with Amazon QuickSight for in-depth analysis and offers a SaaS model with resource-based pricing, making it adaptable to different organizational needs. + +- **Security Hub Integration for Cost-Effective Operations:** + - Prowler can send results directly into Security Hub in any AWS account, including only failed findings. This selective reporting can make Security Hub more cost-effective by reducing the volume of data processed. + +- **Custom Checks and Compliance Frameworks:** + - Users can write custom checks, remediations, and compliance frameworks in minutes, tailoring the tool to their specific security policies and operational needs. + +- **Extensive Compliance Support:** + - Prowler supports over 27 compliance frameworks out of the box for AWS, providing comprehensive coverage across various regulatory requirements and best practices. + +- **Kubernetes and Multi-Cloud Support:** + - Prowler extends its scanning capabilities beyond AWS, offering support for Kubernetes clusters (including EKS), as well as environments in Google Cloud Platform (GCP) and Azure. This multi-cloud capability is essential for organizations with diverse cloud footprints. + +- **All-Region Checks:** + - Prowler runs all checks in all regions, regardless of AWS Config resource type support, ensuring comprehensive coverage across your entire AWS environment. + +## Comparison Summary + +### Scope and Environment + +- **Security Hub** is ideal for AWS-centric environments needing a managed service for monitoring and automating security across AWS resources. +- **Prowler** is better suited for organizations operating in multi-cloud or hybrid environments, offering flexibility, customization, and support for multiple cloud providers including AWS, Azure, GCP, and Kubernetes. + +### Setup and Maintenance + +- **Security Hub** requires enabling and configuring AWS services by region, per account, and can become more than one person's full-time role – including Config. Security Hub operates only within the AWS ecosystem. +- **Prowler** requires minimal setup, only needing appropriate permissions, and can be executed from various environments, making it more versatile in different operational contexts. + +### Customization and Flexibility + +- **Security Hub** offers predefined compliance checks and automation within AWS but is less flexible in terms of customization. +- **Prowler** allows for highly customizable checks, remediation actions, and compliance frameworks, with the ability to adapt quickly to organizational needs and regulatory changes. + +### Cost Efficiency + +- **Security Hub** may involve additional costs for processing and storing findings. +- **Prowler** can optimize costs by selectively sending failed findings to Security Hub and storing results locally or in S3, which can be more cost-effective. + +### Multi-Cloud and Multi-Region Support + +- **Security Hub** is confined to AWS, with region-specific checks depending on AWS Config. +- **Prowler** is inherently multi-region and multi-cloud, offering consistent and comprehensive checks across different cloud environments and regions. + +## Conclusion + +For a CISO or security professional evaluating these tools, the decision between AWS Security Hub and Prowler will depend on the organization’s cloud strategy, compliance needs, and the level of flexibility required: + +- If the organization is heavily invested in AWS and prefers a managed, integrated security service that offers ease of use and automation within the AWS ecosystem, **AWS Security Hub** is the more appropriate choice. + +- If the organization operates in a multi-cloud environment or requires a highly customizable tool that can run comprehensive, multi-region scans across AWS, Azure, GCP, and Kubernetes, **Prowler** provides a more powerful and flexible solution, especially for those needing to adapt quickly to evolving security and compliance requirements. diff --git a/docs/getting-started/comparison/gcp.mdx b/docs/getting-started/comparison/gcp.mdx new file mode 100644 index 0000000000..00f61091d7 --- /dev/null +++ b/docs/getting-started/comparison/gcp.mdx @@ -0,0 +1,97 @@ +--- +title: 'GCP Cloud Security Command Center (Cloud SCC)' +--- + +Google Cloud Security Command Center (Cloud SCC) is a centralized security and risk management platform for Google Cloud Platform (GCP). It provides visibility into assets, vulnerabilities, and threats across GCP environments, helping organizations to manage and improve their security posture. + +## Key Features and Strengths + +- **Centralized Security Visibility:** Cloud SCC provides a single pane of glass to monitor the security and risk status across your GCP resources. It aggregates findings from various GCP security services, such as Security Health Analytics, Web Security Scanner, and Event Threat Detection. + +- **Asset Inventory and Classification:** Cloud SCC offers comprehensive asset discovery and classification across GCP, giving security teams a detailed inventory of their cloud resources, including their configurations and security states. + +- **Threat Detection and Monitoring:** The platform integrates with GCP’s threat detection tools, such as Google’s Event Threat Detection, which analyzes logs for suspicious activities and potential threats. + +- **Compliance Monitoring:** Cloud SCC helps monitor compliance with various regulatory standards by continuously assessing your GCP resources against best practices and security benchmarks. + +- **Automated Remediation:** Cloud SCC can trigger automated responses to security findings through integrations with Google Cloud Functions or other orchestration tools, helping to mitigate risks quickly. + +- **Native GCP Integration:** Cloud SCC is deeply integrated with the GCP ecosystem, offering seamless operation within Google Cloud environments and leveraging Google's extensive security expertise. + +## Limitations + +- **GCP-Centric:** While Cloud SCC is powerful within the GCP ecosystem, it is primarily focused on GCP and does not natively extend to multi-cloud environments without additional tools or connectors. + +- **Cost Considerations:** As a managed service within GCP, costs can scale with the amount of data ingested and the complexity of the environment, especially as additional features or higher volumes of data are utilized. + +- **Dependency on GCP Services:** Cloud SCC's capabilities depend on other GCP services being enabled, such as Security Health Analytics and Web Security Scanner, which may increase overall complexity and cost. + +## Prowler + +Prowler is an open-source, multi-cloud security tool designed to perform detailed security assessments and compliance checks across diverse cloud environments, including AWS, Azure, GCP, and Kubernetes. Here are the key advantages of Prowler when compared to GCP Cloud SCC: + +## Main Advantages of Prowler + +- **Multi-Region and Multi-Account Scanning by Default:** + - Prowler inherently supports multi-region and multi-account scanning across multiple cloud providers, including GCP, AWS, Azure, and Kubernetes. It does not require additional configuration to perform these scans, making it immediately useful for organizations operating in multiple cloud environments. + +- **Minimal Setup Requirements:** + - Prowler requires only appropriate roles and permissions to start scanning. It doesn’t necessitate enabling specific services within GCP, which can simplify the setup process and reduce dependencies. + +- **Versatile Execution Environment:** + - Prowler can be run from various environments, such as a local workstation, container, Google Cloud Shell, or even other cloud providers by assuming a role. This versatility allows for flexible deployment and integration into existing security operations. + +- **Flexible Results Storage and Sharing:** + - Prowler results can be stored in an S3 bucket for AWS, Google Cloud Storage (GCS) for GCP, or locally, allowing for quick analysis and easy sharing. This flexibility is particularly advantageous for multi-cloud security assessments and collaborative security processes. + +- **Customizable Reporting and Analysis:** + - Prowler supports exporting results in multiple formats, including JSON, CSV, OCSF format, and static HTML reports. These reports can be tailored to specific needs and easily integrated with other security tools or dashboards, providing comprehensive insights across all cloud environments. + +- **SIEM Integration and Cost Efficiency:** + - Prowler can be configured to send findings directly into SIEM systems, including those integrated with GCP or other platforms. By sending only failed findings or selected results, Prowler helps manage costs associated with data ingestion and analysis in SIEM platforms. + +- **Custom Checks and Compliance Frameworks:** + - Prowler allows for the creation of custom security checks, remediation actions, and compliance frameworks, providing flexibility that can be adapted to the unique security policies and regulatory requirements of an organization. + +- **Extensive Compliance Support:** + - Prowler supports over 27 compliance frameworks out of the box, with capabilities to extend these frameworks to GCP environments as well as other cloud platforms. This broad compliance coverage ensures that organizations can maintain adherence to various regulatory requirements. + +- **Kubernetes and Multi-Cloud Support:** + - Prowler is designed to support security assessments across cloud environments, including Kubernetes clusters and GCP. This multi-cloud capability is essential for organizations that operate across diverse cloud platforms and require consistent security posture management. + +- **All-Region Checks:** + - Prowler runs all checks in all regions by default, ensuring comprehensive coverage across an organization’s cloud resources, regardless of the region or cloud provider. + +## Comparison Summary + +### Scope and Environment + +- **GCP Cloud SCC** is ideal for organizations primarily using GCP, offering a centralized platform for managing security and compliance within the GCP ecosystem. +- **Prowler** excels in multi-cloud environments, offering flexibility and comprehensive security checks across AWS, Azure, GCP, and Kubernetes without being confined to a single cloud provider. + +### Setup and Flexibility + +- **GCP Cloud SCC** requires enabling various GCP services and may involve more complex setup, especially for multi-region or multi-account scenarios within GCP. +- **Prowler** requires minimal setup and can be deployed quickly across different cloud environments, offering a more straightforward approach to multi-cloud security management. + +### Customization and Compliance + +- **GCP Cloud SCC** provides predefined compliance checks within the GCP environment but may require additional tools or customization for broader or more specific requirements. +- **Prowler** allows for extensive customization of security checks, compliance frameworks, and reporting, providing a flexible solution that can be tailored to an organization’s specific needs across various cloud platforms. + +### Cost Efficiency + +- **GCP Cloud SCC** costs can scale with the volume of data processed and the number of enabled services, which may be significant in large or complex environments. SCC pricing is confusing to understand, and starts at $0.071 per vCPU hour for some tiers and depending on the scan service. Take a look at the pricing model [here](https://cloud.google.com/security-command-center/pricing), godspeed. +- **Prowler** helps manage costs by allowing selective reporting, such as sending only failed findings to SIEMs, and storing results in cost-effective ways, such as local storage or cloud buckets. Prowler is always $0.001 per resource per day - no per account charge. + +### Multi-Cloud and Multi-Region Support + +- **GCP Cloud SCC** is focused on GCP and may require additional tools for comprehensive multi-cloud support. +- **Prowler** is inherently multi-cloud, supporting AWS, Azure, GCP, and Kubernetes out of the box, making it an ideal choice for organizations with diverse cloud footprints. + +## Conclusion + +For a CISO evaluating these tools, the decision between GCP Cloud Security Command Center (Cloud SCC) and Prowler hinges on the organization’s cloud strategy, security management needs, and the level of flexibility and multi-cloud support required: + +- If the organization is heavily invested in GCP and needs a centralized platform that integrates seamlessly with GCP services for asset management, threat detection, and compliance monitoring, **GCP Cloud SCC** is likely the better choice. +- If the organization operates in a multi-cloud environment or requires a highly customizable tool for performing detailed security assessments across AWS, Azure, GCP, and Kubernetes, **Prowler** offers a more flexible and cost-effective solution, especially for those needing quick deployment, minimal setup, and the ability to manage security across diverse cloud environments. diff --git a/docs/getting-started/comparison/index.mdx b/docs/getting-started/comparison/index.mdx new file mode 100644 index 0000000000..9cbc494d42 --- /dev/null +++ b/docs/getting-started/comparison/index.mdx @@ -0,0 +1,10 @@ +--- +title: 'Comparison' +--- + +Click to learn more about each cloud security provider and learn how Prowler is differentiated. + +- [AWS Security Hub](/getting-started/comparison/awssecurityhub) +- [Microsoft Sentinel](/getting-started/comparison/microsoftsentinel) +- [Microsoft Defender](/getting-started/comparison/microsoftdefender) +- [Google Cloud Security Command Center](/getting-started/comparison/gcp) diff --git a/docs/getting-started/comparison/microsoftdefender.mdx b/docs/getting-started/comparison/microsoftdefender.mdx new file mode 100644 index 0000000000..76b61ba9a6 --- /dev/null +++ b/docs/getting-started/comparison/microsoftdefender.mdx @@ -0,0 +1,101 @@ +--- +title: 'Microsoft Defender for Cloud' +--- + +**Use open-source scanning to validate and extend Microsoft Defender for Cloud** + +--- + +## **Overview** + +If you're using Microsoft Defender for Cloud to monitor your Azure infrastructure, Prowler can complement it with fully transparent, customizable scans across Azure, AWS, GCP, and Kubernetes. Prowler helps you validate policies, automate compliance, and gain deeper visibility—all from the CLI, API or our Prowler UI. + +You can run Prowler alongside Defender for Cloud to: + +* Double-check security posture with open-source checks. +* Customize rules for your organization’s policies. +* Bring your own, or community contributed policies. +* Automate multi-cloud scans in CI/CD or scheduled jobs. + +--- + +## **Why use Prowler with Defender for Cloud** + +Microsoft Defender for Cloud offers centralized dashboards, alerting, and some cross-cloud coverage. Prowler provides full transparency and control over what’s being checked and how those checks work—no vendor lock-in, no surprises. + +Use them together to get: + +* More confidence in your security posture +* Checks you can inspect, modify, and version +* CLI-first, portable scanning across clouds +* Open-source tooling that integrates easily into pipelines and audits + +--- + +## **Quickstart** + +Here’s how to install Prowler and run a scan in your Azure account. + +### **1\. Install Prowler** + +``` +git clone https://github.com/prowler-cloud/prowler +cd prowler +./install.sh +``` + +### **2\. Authenticate with Azure** + +Make sure you're signed in and select your subscription: + +``` +az login +export AZURE_SUBSCRIPTION_ID=$(az account show --query id -o tsv) +``` + +### **3\. Run a scan** + +``` +./prowler -p Azure -f az-aks -f az-general +``` + +This will run checks focused on Azure Kubernetes Service (AKS) and general Azure best practices. + +### **4\. Review results** + +``` +cat output/prowler-output-*.json +open output/prowler-output-*.html +``` + +You can export findings in JSON, CSV, JUnit, HTML, or AWS Security Hub–compatible formats. + +--- + +## **Compare capabilities** + +| Feature | Microsoft Defender for Cloud | Prowler | +| ----- | ----- | ----- | +| Azure-native posture management | ✅ | ✅ | +| AWS, GCP, and Kubernetes support | ⚠ī¸ (limited) | ✅ | +| Custom policy creation | ❌ | ✅ | +| CLI-first, scriptable | ❌ | ✅ | +| Open source | ❌ | ✅ | +| Compliance mappings (CIS, NIST, etc.) | ✅ (limited control) | ✅ (customizable) | +| Exportable detections | ❌ | ✅ | + +--- + +## **Common use cases** + +**✅ Validate policies** + Run Prowler to confirm your Azure policies are configured as expected and compliant with frameworks like CIS or NIST. + +**✅ Automate compliance scans** + Schedule regular Prowler scans in your CI/CD pipeline or infrastructure monitoring workflows. Generate reports for auditors or internal reviews. + +**✅ Extend detection coverage** + If Defender for Cloud doesn’t cover all the services or resources in your environment, Prowler’s checks fill in the gaps. + +**✅ Build custom checks** + Security is never one-size-fits-all. Prowler lets you write your own checks for organization-specific policies. diff --git a/docs/getting-started/comparison/microsoftsentinel.mdx b/docs/getting-started/comparison/microsoftsentinel.mdx new file mode 100644 index 0000000000..a6ff00dbfc --- /dev/null +++ b/docs/getting-started/comparison/microsoftsentinel.mdx @@ -0,0 +1,93 @@ +--- +title: 'Microsoft Sentinel' +--- + +Microsoft Sentinel is a scalable, cloud-native security information and event management (SIEM) and security orchestration automated response (SOAR) solution. It's designed to collect, detect, investigate, and respond to threats across the enterprise, primarily within the Azure cloud environment but also extending to on-premises and other cloud environments through various connectors. + +## Key Features and Strengths + +- **SIEM and SOAR Capabilities:** Microsoft Sentinel combines SIEM and SOAR functionalities, allowing it to collect and analyze large volumes of data from various sources and automate responses to detected threats. + +- **Native Azure Integration:** As part of the Azure ecosystem, Sentinel integrates seamlessly with Azure services, providing deep visibility and analytics for Azure resources. + +- **Advanced Threat Detection:** Sentinel uses AI and machine learning to detect potential threats and anomalous activities, leveraging Microsoft's extensive threat intelligence network. + +- **Scalability and Flexibility:** Being cloud-native, Sentinel scales automatically to handle increasing data volumes and complexity without requiring extensive infrastructure management. + +- **Customizable Dashboards and Analytics:** Sentinel offers customizable dashboards and analytics, allowing security teams to tailor their views and queries to specific needs. + +- **Multi-Source Data Ingestion:** While focused on Azure, Sentinel can ingest data from multiple sources, including AWS, GCP, on-premises environments, and third-party security products. + +## Limitations + +- **Azure-Centric:** While it supports multi-cloud environments, its primary focus and strengths are within the Azure ecosystem. Integration with other cloud platforms and on-premises environments may require additional connectors and configurations. + +- **Cost Considerations:** As a SIEM tool, Sentinel can become expensive, particularly as data ingestion and analysis volumes grow. The cost model is based on data volume, which can add up quickly in large environments. + +- **Complexity in Customization:** Although Sentinel offers advanced customization, setting up and fine-tuning these customizations can require significant expertise and effort, particularly in multi-cloud environments. + +## Prowler + +Prowler is an open-source, multi-cloud security tool that offers extensive flexibility and customization, making it ideal for organizations that need to maintain a strong security posture across diverse cloud environments. Here are the key advantages of Prowler, particularly when compared to Microsoft Sentinel: + +## Main Advantages of Prowler + +- **Multi-Region and Multi-Account Scanning by Default:** + - Prowler is inherently multi-region and multi-account, requiring no additional configuration to scan across these environments. This capability is available out of the box without needing to enable specific services or create complex setups. + +- **Minimal Setup Requirements:** + - Prowler requires only a role with appropriate permissions to begin scanning. There’s no need for extensive setup, making it easier and quicker to deploy across various environments. + +- **Versatile Execution Environment:** + - Prowler can be run from a local workstation, container, AWS CloudShell, or even from other cloud providers like Azure or GCP by assuming a role. This versatility allows security teams to integrate Prowler into a wide range of operational workflows without being tied to a single cloud environment. + +- **Flexible Results Storage and Sharing:** + - Prowler results can be stored directly into an S3 bucket, allowing for quick analysis or locally for easy sharing and collaboration. This flexibility is particularly useful for multi-cloud security assessments and incident response. + +- **Customizable Reporting and Analysis:** + - Prowler supports exporting results in multiple formats, including JSON, CSV, OCSF format, and static HTML reports. Additionally, it can integrate with Amazon QuickSight for advanced analytics, and offers a SaaS model with resource-based pricing, making it adaptable to various organizational needs. + +- **SIEM Integration and Cost Efficiency:** + - While Microsoft Sentinel has a built-in SIEM functionality, Prowler can send results directly into SIEM systems, including Microsoft Sentinel. By sending only failed findings, Prowler can help optimize costs associated with data ingestion and storage in SIEM platforms. + +- **Custom Checks and Compliance Frameworks:** + - Prowler enables users to write custom checks, remediations, and compliance frameworks quickly, allowing organizations to adapt the tool to their specific security policies and regulatory requirements. + +- **Extensive Compliance Support:** + - Prowler supports over 27 compliance frameworks out of the box, providing comprehensive coverage for AWS environments, which can be extended to multi-cloud scenarios. + +- **Kubernetes and Multi-Cloud Support:** + - Prowler is designed to support security assessments beyond AWS, including Kubernetes clusters (including EKS) and environments in Azure and GCP. This capability is critical for organizations that operate across multiple cloud platforms and require consistent security posture management. + +- **All-Region Checks:** + - Prowler runs all checks in all regions by default, ensuring comprehensive coverage without the limitations that may be imposed by region-specific configurations or services. + +## Comparison Summary + +### Scope and Environment +- **Microsoft Sentinel** is an advanced SIEM/SOAR tool optimized for Azure environments, with support for multi-cloud and on-premises systems through connectors. +- **Prowler** is a flexible, multi-cloud security tool that excels in environments where organizations need to manage security across AWS, Azure, GCP, and Kubernetes with minimal setup and high customizability. + +### Setup and Flexibility +- **Microsoft Sentinel** requires more setup, especially when integrating with non-Azure environments, and its cost scales with data ingestion. +- **Prowler** requires minimal setup and can be easily deployed in any cloud or on-premises environment. Its ability to run from various environments and store results in flexible formats makes it particularly adaptable. + +### Customization and Compliance +- **Microsoft Sentinel** offers powerful but complex customization options, primarily within the Azure ecosystem. +- **Prowler** provides straightforward customization of security checks, remediation actions, and compliance frameworks, with broad support for multiple compliance standards out of the box. + +### Cost Efficiency +- **Microsoft Sentinel** can become costly as data volumes grow, particularly in large or multi-cloud environments. +- **Prowler** helps control costs by enabling selective reporting (e.g., sending only failed findings to SIEMs like Sentinel) and storing results in cost-effective ways, such as S3 or locally. + +### Multi-Cloud and Multi-Region Support +- **Microsoft Sentinel** supports multi-cloud environments but may require additional configuration and connectors. +- **Prowler** is designed for multi-cloud environments from the ground up, with inherent support for AWS, Azure, GCP, Kubernetes, and all regions, making it an ideal tool for organizations with diverse cloud footprints. + +## Conclusion + +For a CISO or security professional evaluating these tools, the decision between Microsoft Sentinel and Prowler hinges on the organization's cloud strategy, SIEM needs, and the level of customization and flexibility required: + +- If the organization is heavily invested in Azure and needs an integrated SIEM/SOAR solution with advanced threat detection, analytics, and automation capabilities, **Microsoft Sentinel** is likely the better choice. + +- If the organization operates in a multi-cloud environment or requires a highly customizable tool for performing detailed security assessments across AWS, Azure, GCP, and Kubernetes, **Prowler** offers a more flexible and cost-effective solution, especially for those needing quick deployment, minimal setup, and the ability to manage security across diverse cloud environments. diff --git a/docs/getting-started/goto/prowler-api-reference.mdx b/docs/getting-started/goto/prowler-api-reference.mdx new file mode 100644 index 0000000000..f8ca1dc3da --- /dev/null +++ b/docs/getting-started/goto/prowler-api-reference.mdx @@ -0,0 +1,4 @@ +--- +title: "Prowler API Reference" +url: "https://api.prowler.com/api/v1/docs" +--- diff --git a/docs/getting-started/goto/prowler-cloud.mdx b/docs/getting-started/goto/prowler-cloud.mdx new file mode 100644 index 0000000000..964c34ddab --- /dev/null +++ b/docs/getting-started/goto/prowler-cloud.mdx @@ -0,0 +1,4 @@ +--- +title: "Go to Cloud" +url: "https://cloud.prowler.com" +--- diff --git a/docs/getting-started/goto/prowler-hub.mdx b/docs/getting-started/goto/prowler-hub.mdx new file mode 100644 index 0000000000..b9862b582d --- /dev/null +++ b/docs/getting-started/goto/prowler-hub.mdx @@ -0,0 +1,4 @@ +--- +title: "Go to Hub" +url: "https://hub.prowler.com" +--- diff --git a/docs/getting-started/goto/prowler-mcp.mdx b/docs/getting-started/goto/prowler-mcp.mdx new file mode 100644 index 0000000000..5706ca1396 --- /dev/null +++ b/docs/getting-started/goto/prowler-mcp.mdx @@ -0,0 +1,5 @@ +--- +title: "Prowler MCP" +url: "https://github.com/prowler-cloud/prowler/tree/master/mcp_server" +tag: "new!" +--- diff --git a/docs/installation/prowler-app.md b/docs/getting-started/installation/prowler-app.mdx similarity index 52% rename from docs/installation/prowler-app.md rename to docs/getting-started/installation/prowler-app.mdx index 4a5b7eb639..f404180791 100644 --- a/docs/installation/prowler-app.md +++ b/docs/getting-started/installation/prowler-app.mdx @@ -1,57 +1,48 @@ +--- +title: "Installation" +--- + ### Installation Prowler App supports multiple installation methods based on your environment. -Refer to the [Prowler App Tutorial](../tutorials/prowler-app.md) for detailed usage instructions. +Refer to the [Prowler App Tutorial](/user-guide/tutorials/prowler-app) for detailed usage instructions. -???+ warning - Prowler configuration is based in `.env` files. Every version of Prowler can have differences on that file, so, please, use the file that corresponds with that version or repository branch or tag. - -=== "Docker Compose" + + Prowler configuration is based in `.env` files. Every version of Prowler can have differences on that file, so, please, use the file that corresponds with that version or repository branch or tag. + + + _Requirements_: - * `Docker Compose` installed: https://docs.docker.com/compose/install/. + - `Docker Compose` installed: https://docs.docker.com/compose/install/. _Commands_: - ``` bash - curl -LO https://raw.githubusercontent.com/prowler-cloud/prowler/refs/heads/master/docker-compose.yml - curl -LO https://raw.githubusercontent.com/prowler-cloud/prowler/refs/heads/master/.env + ```bash + VERSION=$(curl -s https://api.github.com/repos/prowler-cloud/prowler/releases/latest | jq -r .tag_name) + curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${VERSION}/docker-compose.yml" + curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${VERSION}/.env" docker compose up -d ``` > Containers are built for `linux/amd64`. If your workstation's architecture is different, please set `DOCKER_DEFAULT_PLATFORM=linux/amd64` in your environment or use the `--platform linux/amd64` flag in the docker command. - - > Enjoy Prowler App at http://localhost:3000 by signing up with your email and password. - - ???+ note - You can change the environment variables in the `.env` file. Note that it is not recommended to use the default values in production environments. - - ???+ note - For a secure setup, leave empty or remove `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY` in `.env` before first start. When absent, the API auto‑generates a unique key pair and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate, delete the stored key files and restart the API. - - ???+ note - There is a development mode available, you can use the file https://github.com/prowler-cloud/prowler/blob/master/docker-compose-dev.yml to run the app in development mode. - - ???+ warning - Google and GitHub authentication is only available in [Prowler Cloud](https://prowler.com). - -=== "GitHub" - + + _Requirements_: - * `git` installed. - * `poetry` installed: [poetry installation](https://python-poetry.org/docs/#installation). - * `npm` installed: [npm installation](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). - * `Docker Compose` installed: https://docs.docker.com/compose/install/. - - ???+ warning - Make sure to have `api/.env` and `ui/.env.local` files with the required environment variables. You can find the required environment variables in the [`api/.env.template`](https://github.com/prowler-cloud/prowler/blob/master/api/.env.example) and [`ui/.env.template`](https://github.com/prowler-cloud/prowler/blob/master/ui/.env.template) files. + - `git` installed. + - `poetry` installed: [poetry installation](https://python-poetry.org/docs/#installation). + - `npm` installed: [npm installation](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). + - `Docker Compose` installed: https://docs.docker.com/compose/install/. + + Make sure to have `api/.env` and `ui/.env.local` files with the required environment variables. You can find the required environment variables in the [`api/.env.template`](https://github.com/prowler-cloud/prowler/blob/master/api/.env.example) and [`ui/.env.template`](https://github.com/prowler-cloud/prowler/blob/master/ui/.env.template) files. + _Commands to run the API_: - ``` bash + ```bash git clone https://github.com/prowler-cloud/prowler \ cd prowler/api \ poetry install \ @@ -64,17 +55,16 @@ Refer to the [Prowler App Tutorial](../tutorials/prowler-app.md) for detailed us gunicorn -c config/guniconf.py config.wsgi:application ``` - ???+ important - Starting from Poetry v2.0.0, `poetry shell` has been deprecated in favor of `poetry env activate`. - - If your poetry version is below 2.0.0 you must keep using `poetry shell` to activate your environment. - In case you have any doubts, consult the Poetry environment activation guide: https://python-poetry.org/docs/managing-environments/#activating-the-environment + + Starting from Poetry v2.0.0, `poetry shell` has been deprecated in favor of `poetry env activate`. + If your poetry version is below 2.0.0 you must keep using `poetry shell` to activate your environment. In case you have any doubts, consult the Poetry environment activation guide: https://python-poetry.org/docs/managing-environments/#activating-the-environment + > Now, you can access the API documentation at http://localhost:8080/api/v1/docs. _Commands to run the API Worker_: - ``` bash + ```bash git clone https://github.com/prowler-cloud/prowler \ cd prowler/api \ poetry install \ @@ -87,7 +77,7 @@ Refer to the [Prowler App Tutorial](../tutorials/prowler-app.md) for detailed us _Commands to run the API Scheduler_: - ``` bash + ```bash git clone https://github.com/prowler-cloud/prowler \ cd prowler/api \ poetry install \ @@ -100,7 +90,7 @@ Refer to the [Prowler App Tutorial](../tutorials/prowler-app.md) for detailed us _Commands to run the UI_: - ``` bash + ```bash git clone https://github.com/prowler-cloud/prowler \ cd prowler/ui \ npm install \ @@ -110,9 +100,11 @@ Refer to the [Prowler App Tutorial](../tutorials/prowler-app.md) for detailed us > Enjoy Prowler App at http://localhost:3000 by signing up with your email and password. - ???+ warning - Google and GitHub authentication is only available in [Prowler Cloud](https://prowler.com). - + + Google and GitHub authentication is only available in [Prowler Cloud](https://prowler.com). + + + ### Update Prowler App @@ -135,9 +127,11 @@ docker compose pull --policy always The `--policy always` flag ensures that Docker pulls the latest images even if they already exist locally. + + **What Gets Preserved During Upgrade** -???+ note "What Gets Preserved During Upgrade" - Everything is preserved, nothing will be deleted after the update. + Everything is preserved, nothing will be deleted after the update. + ### Troubleshooting @@ -158,7 +152,6 @@ docker compose pull docker compose up -d ``` - ### Container versions The available versions of Prowler CLI are the following: @@ -174,6 +167,5 @@ The available versions of Prowler CLI are the following: The container images are available here: - Prowler App: - - - [DockerHub - Prowler UI](https://hub.docker.com/r/prowlercloud/prowler-ui/tags) - - [DockerHub - Prowler API](https://hub.docker.com/r/prowlercloud/prowler-api/tags) + - [DockerHub - Prowler UI](https://hub.docker.com/r/prowlercloud/prowler-ui/tags) + - [DockerHub - Prowler API](https://hub.docker.com/r/prowlercloud/prowler-api/tags) diff --git a/docs/installation/prowler-cli.md b/docs/getting-started/installation/prowler-cli.mdx similarity index 83% rename from docs/installation/prowler-cli.md rename to docs/getting-started/installation/prowler-cli.mdx index 663b6bebaf..a3c87c8ce7 100644 --- a/docs/installation/prowler-cli.md +++ b/docs/getting-started/installation/prowler-cli.mdx @@ -1,10 +1,13 @@ +--- +title: 'Installation' +--- ## Installation Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). Install it as a Python package with `Python >= 3.9, <= 3.12`: -=== "pipx" - + + [pipx](https://pipx.pypa.io/stable/) installs Python applications in isolated environments. Use `pipx` for global installation. _Requirements_: @@ -19,17 +22,11 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). pipx install prowler prowler -v ``` - - Upgrade Prowler to the latest version: - - ``` bash - pipx upgrade prowler - ``` - -=== "pip" - - ???+ warning - This method modifies the chosen installation environment. Consider using [pipx](https://docs.prowler.com/projects/prowler-open-source/en/latest/#__tabbed_1_1) for global installation. + + + + This method modifies the chosen installation environment. Consider using [pipx](https://docs.prowler.com/projects/prowler-open-source/en/latest/#__tabbed_1_1) for global installation. + _Requirements_: @@ -49,9 +46,8 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). ``` bash pip install --upgrade prowler ``` - -=== "Docker" - + + _Requirements_: * Have `docker` installed: https://docs.docker.com/get-docker/. @@ -69,9 +65,8 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). --env AWS_SECRET_ACCESS_KEY \ --env AWS_SESSION_TOKEN toniblyx/prowler:latest ``` - -=== "GitHub" - + + _Requirements for Developers_: * `git` @@ -86,11 +81,12 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). poetry install poetry run python prowler-cli.py -v ``` - ???+ note - If you want to clone Prowler from Windows, use `git config core.longpaths true` to allow long file paths. - -=== "Amazon Linux 2" + + If you want to clone Prowler from Windows, use `git config core.longpaths true` to allow long file paths. + + + _Requirements_: * `Python >= 3.9, <= 3.12` @@ -104,9 +100,8 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). pipx install prowler prowler -v ``` - -=== "Ubuntu" - + + _Requirements_: * `Ubuntu 23.04` or above, if you are using an older version of Ubuntu check [pipx installation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#__tabbed_1_1) and ensure you have `Python >= 3.9, <= 3.12`. @@ -122,9 +117,8 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). pipx install prowler prowler -v ``` - -=== "Brew" - + + _Requirements_: * `Brew` installed in your Mac or Linux @@ -136,9 +130,8 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). brew install prowler prowler -v ``` - -=== "AWS CloudShell" - + + After the migration of AWS CloudShell from Amazon Linux 2 to Amazon Linux 2023 [[1]](https://aws.amazon.com/about-aws/whats-new/2023/12/aws-cloudshell-migrated-al2023/) [[2]](https://docs.aws.amazon.com/cloudshell/latest/userguide/cloudshell-AL2023-migration.html), there is no longer a need to manually compile Python 3.9 as it is already included in AL2023. Prowler can thus be easily installed following the generic method of installation via pip. Follow the steps below to successfully execute Prowler v4 in AWS CloudShell: _Requirements_: @@ -158,11 +151,11 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). prowler aws ``` - ???+ note - To download the results from AWS CloudShell, select Actions -> Download File and add the full path of each file. For the CSV file it will be something like `/tmp/output/prowler-output-123456789012-20221220191331.csv` - -=== "Azure CloudShell" - + + To download the results from AWS CloudShell, select Actions -> Download File and add the full path of each file. For the CSV file it will be something like `/tmp/output/prowler-output-123456789012-20221220191331.csv` + + + _Requirements_: * Open Azure CloudShell `bash`. @@ -176,18 +169,8 @@ Prowler is available as a project in [PyPI](https://pypi.org/project/prowler/). cd /tmp prowler azure --az-cli-auth ``` - - - - - - - - - - - - + + ## Container versions The available versions of Prowler CLI are the following: diff --git a/docs/getting-started/installation/prowler-mcp.mdx b/docs/getting-started/installation/prowler-mcp.mdx new file mode 100644 index 0000000000..8c3fd2f955 --- /dev/null +++ b/docs/getting-started/installation/prowler-mcp.mdx @@ -0,0 +1,276 @@ +--- +title: "Installation" +--- + +There are **two ways** to use Prowler MCP Server: + + + + **No installation required** - Just configuration + + Use `https://mcp.prowler.com/mcp` + + + **Local installation** - Full control + + Install via Docker, PyPI, or source code + + + + +For "Option 1: Managed by Prowler", go directly to the [Configuration Guide](/getting-started/basic-usage/prowler-mcp#hosted-server-configuration-recommended) to set up your Claude Desktop, Cursor, or other MCP client. +**This guide is focused on local installation, "Option 2: Run Locally"**. + +## Installation Methods + +Choose one of the following installation methods: + + + + ### Pull from Docker Hub + + The easiest way to run locally is using the official Docker image: + + ```bash + docker pull prowlercloud/prowler-mcp + ``` + + ### Run the Container + + ```bash + # STDIO mode (for local MCP clients) + docker run --rm -i prowlercloud/prowler-mcp + + # HTTP mode (for remote access) + docker run --rm -p 8000:8000 \ + prowlercloud/prowler-mcp \ + --transport http --host 0.0.0.0 --port 8000 + ``` + + ### With Environment Variables + + ```bash + docker run --rm -i \ + -e PROWLER_APP_API_KEY="pk_your_api_key" \ + -e PROWLER_API_BASE_URL="https://api.prowler.com" \ + prowlercloud/prowler-mcp + ``` + + + **Docker Hub:** [prowlercloud/prowler-mcp](https://hub.docker.com/r/prowlercloud/prowler-mcp) + + + + + + ### Install via pip + + + **Coming Soon** - PyPI package will be available shortly + + + + + ### Install uv + + If `uv` is not installed, install it first. Visit [uv documentation](https://docs.astral.sh/uv/) for more installation options. + + ### Clone the Repository + + ```bash + git clone https://github.com/prowler-cloud/prowler.git + cd prowler/mcp_server + ``` + + ### Install Dependencies + + The MCP server uses `uv` for dependency management. Install dependencies with: + + ```bash + uv sync + ``` + + ### Verify Installation + + Test that the server is properly installed: + + ```bash + uv run prowler-mcp --help + ``` + + The help message with available command-line options should appear. + + + **For Development:** This method is recommended if you're developing or contributing to the MCP server. + + + + + + ### Prerequisites + + Ensure Docker is installed on your system. Visit [Docker documentation](https://docs.docker.com/get-docker/) for more installation options. + + ### Clone the Repository + + ```bash + git clone https://github.com/prowler-cloud/prowler.git + cd prowler/mcp_server + ``` + + ### Build the Docker Image + + ```bash + docker build -t prowler-mcp . + ``` + + This creates a Docker image named `prowler-mcp` with all necessary dependencies. + + ### Verify Installation + + Test that the Docker image was built successfully: + + ```bash + docker run --rm prowler-mcp --help + ``` + + The help message with available command-line options should appear. + + + + +--- + +## Command Line Options + +The Prowler MCP Server supports the following command-line arguments: + +```bash +prowler-mcp [--transport {stdio,http}] [--host HOST] [--port PORT] +``` + +| Argument | Values | Default | Description | +|----------|--------|---------|-------------| +| `--transport` | `stdio`, `http` | `stdio` | Transport method for MCP communication | +| `--host` | Any valid hostname/IP | `127.0.0.1` | Host to bind to (HTTP mode only) | +| `--port` | Port number | `8000` | Port to bind to (HTTP mode only) | + +### Examples + +```bash +# Default STDIO mode for local MCP client integration +prowler-mcp + +# Explicit STDIO mode +prowler-mcp --transport stdio + +# HTTP mode on default host and port (127.0.0.1:8000) +prowler-mcp --transport http + +# HTTP mode accessible from any network interface +prowler-mcp --transport http --host 0.0.0.0 + +# HTTP mode with custom port +prowler-mcp --transport http --host 0.0.0.0 --port 8080 +``` + +## Environment Variables + +Configure the server using environment variables: + +| Variable | Description | Required | Default | +|----------|-------------|----------|---------| +| `PROWLER_APP_API_KEY` | Prowler API key | Only for STDIO mode | - | +| `PROWLER_API_BASE_URL` | Custom Prowler API endpoint | No | `https://api.prowler.com` | +| `PROWLER_MCP_TRANSPORT_MODE` | Default transport mode (overwritten by `--transport` argument) | No | `stdio` | + + +```bash macOS/Linux +export PROWLER_APP_API_KEY="pk_your_api_key_here" +export PROWLER_API_BASE_URL="https://api.prowler.com" +export PROWLER_MCP_TRANSPORT_MODE="http" +``` + +```bash Windows PowerShell +$env:PROWLER_APP_API_KEY="pk_your_api_key_here" +$env:PROWLER_API_BASE_URL="https://api.prowler.com" +$env:PROWLER_MCP_TRANSPORT_MODE="http" +``` + + + +Never commit your API key to version control. Use environment variables or secure secret management solutions. + + +### Using Environment Files + +For convenience, create a `.env` file in the `mcp_server` directory: + +```bash .env +PROWLER_APP_API_KEY=pk_your_api_key_here +PROWLER_API_BASE_URL=https://api.prowler.com +PROWLER_MCP_TRANSPORT_MODE=stdio +``` + +When using Docker, pass the environment file: + +```bash +docker run --rm --env-file .env -it prowler-mcp +``` + +## Running from Any Location + +Run the MCP server from anywhere using `uvx`: + +```bash +uvx /path/to/prowler/mcp_server/ +``` + +This is particularly useful when configuring MCP clients that need to launch the server from a specific path. + +## Production Deployment + +For production deployments that require customization, it is recommended to use the ASGI application that can be found in `prowler_mcp_server.server`. This can be run with uvicorn: + +```bash +uvicorn prowler_mcp_server.server:app --host 0.0.0.0 --port 8000 +``` + +For more details on production deployment options, see the [FastMCP production deployment guide](https://gofastmcp.com/deployment/http#production-deployment) and [uvicorn settings](https://www.uvicorn.org/settings/). + +### Entrypoint Script + +The source tree includes `entrypoint.sh` to simplify switching between the +standard CLI runner and the ASGI app. The first argument selects the mode and +any additional flags are passed straight through: + +```bash +# Default CLI experience (prowler-mcp console script) +./entrypoint.sh main --transport http --host 0.0.0.0 + +# ASGI app via uvicorn +./entrypoint.sh uvicorn --host 0.0.0.0 --port 9000 +``` + +Omitting the mode defaults to `main`, matching the `prowler-mcp` console script. +When `uvicorn` mode is selected, the script exports `PROWLER_MCP_TRANSPORT_MODE=http` automatically. + +This is the default entrypoint for the Docker container. + +## Next Steps + +Now that you have the Prowler MCP Server installed, proceed to configure your MCP client: + + + + Configure Claude Desktop, Cursor, or other MCP clients + + + +## Getting Help + +If you encounter issues during installation: + +- Search for existing [GitHub issues](https://github.com/prowler-cloud/prowler/issues) +- Ask for help in our [Slack community](https://goto.prowler.com/slack) +- Report a new issue on [GitHub](https://github.com/prowler-cloud/prowler/issues/new) diff --git a/docs/products/img/dashboard.png b/docs/getting-started/products/img/dashboard.png similarity index 100% rename from docs/products/img/dashboard.png rename to docs/getting-started/products/img/dashboard.png diff --git a/docs/products/img/overview.png b/docs/getting-started/products/img/overview.png similarity index 100% rename from docs/products/img/overview.png rename to docs/getting-started/products/img/overview.png diff --git a/docs/products/img/prowler-app-architecture.png b/docs/getting-started/products/img/prowler-app-architecture.png similarity index 100% rename from docs/products/img/prowler-app-architecture.png rename to docs/getting-started/products/img/prowler-app-architecture.png diff --git a/docs/products/prowler-app.md b/docs/getting-started/products/prowler-app.mdx similarity index 84% rename from docs/products/prowler-app.md rename to docs/getting-started/products/prowler-app.mdx index fb2eb1686b..f21e0222cd 100644 --- a/docs/products/prowler-app.md +++ b/docs/getting-started/products/prowler-app.mdx @@ -1,9 +1,13 @@ +--- +title: 'Overview' +--- + Prowler App is a web application that simplifies running Prowler. It provides: - **User-friendly interface** for configuring and executing scans - Dashboard to **view results** and manage **security findings** -![Prowler App](img/overview.png) +![Prowler App](/images/products/overview.png) ## Components @@ -19,4 +23,4 @@ Supporting infrastructure includes: - **Celery Workers**: Asynchronous execution of Prowler scans - **Valkey**: In-memory database serving as message broker for Celery workers -![Prowler App Architecture](img/prowler-app-architecture.png) +![Prowler App Architecture](/images/products/prowler-app-architecture.png) diff --git a/docs/products/prowler-cli.md b/docs/getting-started/products/prowler-cli.mdx similarity index 90% rename from docs/products/prowler-cli.md rename to docs/getting-started/products/prowler-cli.mdx index 633827b01e..ac4f37c291 100644 --- a/docs/products/prowler-cli.md +++ b/docs/getting-started/products/prowler-cli.mdx @@ -1,16 +1,20 @@ +--- +title: 'Overview' +--- + Prowler CLI is a command-line interface for running Prowler scans from the terminal. ```console prowler ``` -![Prowler CLI Execution](../img/short-display.png) +![Prowler CLI Execution](/images/short-display.png) ## Prowler Dashboard ```console prowler dashboard ``` -![Prowler Dashboard](img/dashboard.png) +![Prowler Dashboard](/images/products/dashboard.png) Prowler includes hundreds of security controls aligned with widely recognized industry frameworks and standards, including: diff --git a/docs/getting-started/products/prowler-cloud-aws-marketplace.mdx b/docs/getting-started/products/prowler-cloud-aws-marketplace.mdx new file mode 100644 index 0000000000..5e4c7036c7 --- /dev/null +++ b/docs/getting-started/products/prowler-cloud-aws-marketplace.mdx @@ -0,0 +1,82 @@ +--- +title: "AWS Marketplace" +--- + +This section contains the instructions to subscribe to **Prowler Cloud** through the **AWS Marketplace**. + +## How to subscribe + +To get to the **Prowler Cloud** product listing in the AWS Marketplace, and click the `View purchase options` button: + +1. Use this link to be taken directly to the [Prowler Cloud Marketplace Listing](https://aws.amazon.com/marketplace/pp/prodview-6ochhig5kxpok): + + ![](/images/aws-marketplace/marketplace-listing.png) + +2. Then, scroll down to the "Purchase Details" section and click the `Subscribe` button: + + ![](/images/aws-marketplace/marketplace-subscribe.png) + +## Set up your account + +After you have subscribed to the **Prowler Cloud** product, you will need to set up your **Prowler Cloud** account: + +1. Click the `Set up your account` button: + + ![](/images/aws-marketplace/marketplace-message.png) + +2. You will be redirected to **Prowler Cloud Sign In** page. You can sign in with an exsiting account or sign up with a new account.: + + ![](/images/aws-marketplace/marketplace-sign-up.png) + +3. Once you have successfully authenticated, you should be automatically redirected to the **Prowler Cloud** [Billing](https://cloud.prowler.com/billing) page where you should now see that your account has the `AWS Marketplace Subscription` badge. + + ![](/images/aws-marketplace/marketplace-my-account.png) + +If you have any issues signing up, please contact us at support@prowler.com. + + +## Billing + +You will be charged monthly based on resources scanned and monitored depending on usage in **Prowler Cloud**. For more information on billing, please see the [Prowler Cloud Pricing FAQ](https://prowler.com/pricing/). + +**Note:** Your **Prowler Cloud** bills can be seen at [AWS Billing](https://us-east-1.console.aws.amazon.com/billing/home#/bills). + +## Subscription + +If you subscribe to Prowler Cloud through the AWS Marketplace it is not necessary to subscribe from different AWS accounts to use Prowler Cloud for those accounts. + +In Prowler Cloud you only need to subscribe from one of your AWS accounts through the AWS Marketplace and add multiple provider accounts once you are in the Prowler Cloud console. We will send usage metrics to the AWS Marketplace regardless of the number of accounts you add in our platform, so the AWS Marketplace will bill you based on those usage metrics. + +## Troubleshooting + +### SEPA Payment Method Issues + +If AWS Marketplace notifies that payment failed due to an issue with the payment method, this typically occurs when a SEPA bank account is set as the default payment method. AWS Marketplace does not support SEPA bank accounts for product subscriptions, even when the account includes valid alternative payment methods. This is because AWS Marketplace invoices are issued by AWS Inc., a US entity. SEPA accounts do not recognize these invoices as valid, causing subscription failures. + +To successfully subscribe to AWS Marketplace products with a SEPA account configuration: + +1. Switch default payment method to credit card +2. Complete subscription +3. Switch the default payment method back to the bank account + + +**Renewal Considerations** + +This issue will recur during subscription renewals. AWS service teams recommend maintaining credit card as the default payment method to prevent future disruptions. Update payment methods at https://console.aws.amazon.com/billing/home#/paymentmethods. + + + + +**AWS Marketplace Statement** + +The AWS Marketplace team acknowledges this limitation: "We apologize for these additional steps, and please know we are fully aware of this situation, and our internal teams are working on simplifying this process." + + + +### Credit and Debit Card Storage Restrictions (AISPL Customers in India) + +AWS Marketplace no longer supports payments using credit or debit cards stored on file for Amazon Internet Services Private Limited (AISPL) customers. This restriction stems from Reserve Bank of India (RBI) regulations regarding payment aggregators, which prohibit the storage of card data. As explained in [this AWS blog post](https://aws.amazon.com/blogs/awsmarketplace/restriction-on-credit-and-debit-card-purchases-for-aispl-customers-using-aws-marketplace/): + +> AWS Marketplace can no longer support payments using credit or debit cards stored on file. [The Reserve Bank of India (RBI) has issued a notice regarding regulation of payment aggregators](https://www.rbi.org.in/Scripts/NotificationUser.aspx?Id=11822&Mode=0), which restricts the storage of card data. If you are currently using credit or debit card as your default payment instrument, your ability to use AWS Marketplace products will be restricted. However, you can switch your default payment instrument to Pay By Invoice to avoid disruption or restore your original experience. + +To maintain uninterrupted access to AWS Marketplace products, change the default payment instrument from stored card data to Pay By Invoice billing. diff --git a/docs/getting-started/products/prowler-cloud-pricing.mdx b/docs/getting-started/products/prowler-cloud-pricing.mdx new file mode 100644 index 0000000000..fc7d2104df --- /dev/null +++ b/docs/getting-started/products/prowler-cloud-pricing.mdx @@ -0,0 +1,4 @@ +--- +title: "Pricing" +url: "https://prowler.com/pricing" +--- diff --git a/docs/getting-started/products/prowler-cloud.mdx b/docs/getting-started/products/prowler-cloud.mdx new file mode 100644 index 0000000000..b8b3431a89 --- /dev/null +++ b/docs/getting-started/products/prowler-cloud.mdx @@ -0,0 +1,24 @@ +--- +title: "Overview" +--- + +[Prowler Cloud](https://prowler.com) makes Cloud Security easy and enables your team to build trust in their deployed services and applications. + +Prowler Cloud Automates scanning single or multiple accounts and has all of the benefits of Prowler Open Source, plus hands-off continuous monitoring, auto-scaling workers for faster execution, integrations, personalized support options and out of the box social authentication. + +![](/images/products/overview.png) + + + +With 100% consistency across our open source policies and APIs. Prowler Cloud provides the following added benefits: + +
    +
  • Immediate sign-up and account provisioning, including a trial period with zero billing details needed at registration.
    • +
  • Simple, transparent pricing, with cloud account sizing and exact pricing being available to view in your account settings.
    • +
  • SOC2 Security processes around the deployment, management, data protection and security updates of your prowler environment.
    • +
  • Helpers for smooth onboarding of cloud environments (Eg. Automatic account ID and ExternalID for AWS assumed roles, Known Static IP for Kubernetes access).
    • +
  • Zero touch third party notifications to Slack, Jira, and more.
    • +
+ +The team who built [Prowler](https://github.com/prowler-cloud/prowler), has helped thousands of companies get Cloud Security under control, is now making it easier by taking +[Prowler](https://github.com/prowler-cloud/prowler) to the [Cloud](https://prowler.com)! diff --git a/docs/getting-started/products/prowler-hub.mdx b/docs/getting-started/products/prowler-hub.mdx new file mode 100644 index 0000000000..a8dde7549f --- /dev/null +++ b/docs/getting-started/products/prowler-hub.mdx @@ -0,0 +1,17 @@ +--- +title: "Overview" +--- + +**Prowler Hub** is our growing public library of versioned checks, cloud service artifacts, and compliance frameworks with its mappings. It’s searchable, explainable, and built to serve the community. + +**Why this matters**: Every engineer has asked, “What does this check actually do?” Prowler Hub answers that question in one place, lets you pin to a specific version, and pulls definitions into your own tools or dashboards. + +![](/images/products/prowler-hub.webp) + + + +Prowler Hub also provides a fully documented public API that you can integrate into your internal tools, dashboards, or automation workflows. + +📚 Explore the API docs at: https://hub.prowler.com/api/docs + +Whether you’re customizing policies, managing compliance, or enhancing visibility, Prowler Hub is built to support your security operations. \ No newline at end of file diff --git a/docs/getting-started/products/prowler-mcp.mdx b/docs/getting-started/products/prowler-mcp.mdx new file mode 100644 index 0000000000..94a5466edb --- /dev/null +++ b/docs/getting-started/products/prowler-mcp.mdx @@ -0,0 +1,187 @@ +--- +title: "Overview" +--- + +**Prowler MCP Server** brings the entire Prowler ecosystem to AI assistants through the Model Context Protocol (MCP). It enables seamless integration with AI tools like Claude Desktop, Cursor, and other MCP clients, allowing interaction with Prowler's security capabilities through natural language. + + +**Preview Feature**: This MCP server is currently under active development. Features and functionality may change. We welcome your feedback—please report any issues on [GitHub](https://github.com/prowler-cloud/prowler/issues) or join our [Slack community](https://goto.prowler.com/slack) to discuss and share your thoughts. + + +## What is the Model Context Protocol? + +The [Model Context Protocol (MCP)](https://modelcontextprotocol.io) is an open standard developed by Anthropic that enables AI assistants to securely connect to external data sources and tools. It functions as a universal adapter enabling AI assistants to interact with various services through a standardized interface. + +## Key Capabilities + +The Prowler MCP Server provides three main integration points: + +### 1. Prowler Cloud and Prowler App (Self-Managed) + +Full access to Prowler Cloud platform and self-managed Prowler App for: +- **Provider Management**: Create, configure, and manage cloud providers (AWS, Azure, GCP, etc.). +- **Scan Orchestration**: Trigger on-demand scans and schedule recurring security assessments. +- **Findings Analysis**: Query, filter, and analyze security findings across all your cloud environments. +- **Compliance Reporting**: Generate compliance reports for various frameworks (CIS, PCI-DSS, HIPAA, etc.). +- **Secrets Management**: Securely manage provider credentials and connection details. +- **Processor Configuration**: Set up the [Prowler Mutelist](/user-guide/tutorials/prowler-app-mute-findings) to mute findings. + +### 2. Prowler Hub + +Access to Prowler's comprehensive security knowledge base: +- **Security Checks Catalog**: Browse and search **over 1000 security checks** across multiple cloud providers. +- **Check Implementation**: View the Python code that powers each security check. +- **Automated Fixers**: Access remediation scripts for common security issues. +- **Compliance Frameworks**: Explore mappings to **over 70 compliance standards and frameworks**. +- **Provider Services**: View available services and checks for each cloud provider. + +### 3. Prowler Documentation + +Search and retrieve official Prowler documentation: +- **Intelligent Search**: Full-text search across all Prowler documentation. +- **Contextual Results**: Get relevant documentation pages with highlighted snippets. +- **Document Retrieval**: Access complete markdown content of any documentation file. + +## MCP Server Architecture + +The following diagram illustrates the Prowler MCP Server architecture and its integration points: + +Prowler MCP Server Schema +Prowler MCP Server Schema + +The architecture shows how AI assistants connect through the MCP protocol to access Prowler's three main components: Prowler Cloud/App for security operations, Prowler Hub for security knowledge, and Prowler Documentation for guidance and reference. + +## Use Cases + +The Prowler MCP Server enables powerful workflows through AI assistants: + +**Security Operations** +- "Show me all critical findings from my AWS production accounts" +- "What is my compliance status for the PCI standards accross all my AWS accounts according to the latest Prowler scan results?" +- "Register my new AWS account in Prowler and run an scheduled scan every day" + +**Security Research** +- "Explain what the S3 bucket public access check does" +- "Find all checks related to encryption at rest" +- "What is the latest version of the CIS that Prowler is covering per provider?" + +**Documentation & Learning** +- "How do I configure Prowler to scan my GCP organization?" +- "What authentication methods does Prowler support for Azure?" +- "How can I contribute with a new security check to Prowler?" + +### Example: Creating a custom dashboard with Prowler extracted data + +In the next example you can see how to create a dashboard using Prowler MCP Server and Claude Desktop. + +**Used Prompt:** +``` +Generate me a security dashboard for the Prowler open source project using live data from Prowler MCP tools. + +REQUIREMENTS: +1. Fetch real-time data from Prowler Findings using MCP tools +2. Create a single self-contained HTML file and display it +3. Dashboard must be production-ready with modern design + +DATA TO FETCH: +Use these MCP tools in this order: +1. Prowler app list providers - To get all available configured provider in the account +2. Prowler app get latest findings - To get findings information, if there are so many you can use the filter_fields to get less information, or pagination to get in different batches +3. For most critical findings you can get more context and remediation with Prowler Hub to get remediations for example + +DESIGN REQUIREMENTS: +- Dark theme (gradient background: #0a0e27 to #131830) +- Card-based layout with glassmorphism effects +- Color scheme: + * Primary green + * Secondary purple +- Modern, professional look +- Animated "LIVE DATA" indicator (pulsing green badge) +- Hover effects on all cards (lift, glow, border color change) +- Responsive grid layout +- Mobile-responsive breakpoints at 768px +- Single HTML file with all CSS and JavaScript embedded +- No external dependencies + +SPECIFIC DETAILS TO INCLUDE: +- Show actual counts from the data (don't hardcode numbers) +- Add timestamp showing when dashboard was generated +- Link to GitHub repository: https://github.com/prowler-cloud/prowler + +OUTPUT: +Generate the complete HTML file and display it +``` + +**Video:** + + + +## Deployment Options + +Prowler MCP Server can be used in three ways: + +### 1. Prowler Cloud MCP Server + +**Use Prowler's managed MCP server at `https://mcp.prowler.com/mcp`** + +- No installation required. +- Managed and maintained by Prowler team. +- Authentication to Prowler Cloud or Prowler App (self-managed) via API key or JWT token. + +### 2. Local STDIO Mode + +**Run the server locally on your machine** + +- Runs as a subprocess of your MCP client. +- Possibility to connect to a self-hosted Prowler App (e.g. self-hosted Prowler App). +- Authentication to Prowler Cloud or Prowler App (self-managed) via environment variables. +- Requires Python 3.12+ or Docker. + +### 3. Self-Hosted HTTP Mode + +**Deploy your own remote MCP server** + +- Full control over deployment. +- Possibility to connect to a self-hosted Prowler App (e.g. self-hosted Prowler App). +- Authentication to Prowler App (self-managed) via API key or JWT token. +- Requires Python 3.12+ or Docker. + +## Requirements + +Requirements vary based on deployment option: + +**For Prowler Cloud MCP Server:** +- Prowler Cloud account and API key (only for Prowler Cloud/App features) + +**For self-hosted STDIO/HTTP Mode:** +- Python 3.12+ or Docker +- Network access to: + - `https://hub.prowler.com` (for Prowler Hub) + - `https://docs.prowler.com` (for Prowler Documentation) + - Prowler Cloud API or self-hosted Prowler App API (for Prowler Cloud/App features) + + +**No Authentication Required**: Prowler Hub and Prowler Documentation features work without authentication in both deployment options. A Prowler API key is only required to access Prowler Cloud or Prowler App (Self-Managed) features. + + +## Next Steps + + + + Install the Prowler MCP Server using uv or Docker + + + Configure your MCP client to connect to the server + + + + + Explore all available tools and capabilities + diff --git a/docs/images/AAD-permissions.png b/docs/images/AAD-permissions.png new file mode 100644 index 0000000000..f2f37e354d Binary files /dev/null and b/docs/images/AAD-permissions.png differ diff --git a/docs/images/add-account.png b/docs/images/add-account.png new file mode 100644 index 0000000000..35333883b5 Binary files /dev/null and b/docs/images/add-account.png differ diff --git a/docs/images/add-provider.png b/docs/images/add-provider.png new file mode 100644 index 0000000000..4e986e3f1a Binary files /dev/null and b/docs/images/add-provider.png differ diff --git a/docs/images/add-reader-role.gif b/docs/images/add-reader-role.gif new file mode 100644 index 0000000000..4868abb773 Binary files /dev/null and b/docs/images/add-reader-role.gif differ diff --git a/docs/images/add-sub-to-management-group.gif b/docs/images/add-sub-to-management-group.gif new file mode 100644 index 0000000000..d63049d58b Binary files /dev/null and b/docs/images/add-sub-to-management-group.gif differ diff --git a/docs/images/architecture.png b/docs/images/architecture.png new file mode 100644 index 0000000000..aeac3b136a Binary files /dev/null and b/docs/images/architecture.png differ diff --git a/docs/images/aws-credentials.png b/docs/images/aws-credentials.png new file mode 100644 index 0000000000..71098075db Binary files /dev/null and b/docs/images/aws-credentials.png differ diff --git a/docs/images/aws-marketplace/discover-product.png b/docs/images/aws-marketplace/discover-product.png new file mode 100644 index 0000000000..b78f0c6fc4 Binary files /dev/null and b/docs/images/aws-marketplace/discover-product.png differ diff --git a/docs/images/aws-marketplace/marketplace-billing.png b/docs/images/aws-marketplace/marketplace-billing.png new file mode 100644 index 0000000000..f54d232880 Binary files /dev/null and b/docs/images/aws-marketplace/marketplace-billing.png differ diff --git a/docs/images/aws-marketplace/marketplace-listing.png b/docs/images/aws-marketplace/marketplace-listing.png new file mode 100644 index 0000000000..aae03ee438 Binary files /dev/null and b/docs/images/aws-marketplace/marketplace-listing.png differ diff --git a/docs/images/aws-marketplace/marketplace-message.png b/docs/images/aws-marketplace/marketplace-message.png new file mode 100644 index 0000000000..7b4581273c Binary files /dev/null and b/docs/images/aws-marketplace/marketplace-message.png differ diff --git a/docs/images/aws-marketplace/marketplace-my-account.png b/docs/images/aws-marketplace/marketplace-my-account.png new file mode 100644 index 0000000000..85fc08f932 Binary files /dev/null and b/docs/images/aws-marketplace/marketplace-my-account.png differ diff --git a/docs/images/aws-marketplace/marketplace-sign-up.png b/docs/images/aws-marketplace/marketplace-sign-up.png new file mode 100644 index 0000000000..249bbb6c56 Binary files /dev/null and b/docs/images/aws-marketplace/marketplace-sign-up.png differ diff --git a/docs/images/aws-marketplace/marketplace-subscribe.png b/docs/images/aws-marketplace/marketplace-subscribe.png new file mode 100644 index 0000000000..033e7f156d Binary files /dev/null and b/docs/images/aws-marketplace/marketplace-subscribe.png differ diff --git a/docs/images/aws-marketplace/marketplace-unsubscribe-error.png b/docs/images/aws-marketplace/marketplace-unsubscribe-error.png new file mode 100644 index 0000000000..97c81abe71 Binary files /dev/null and b/docs/images/aws-marketplace/marketplace-unsubscribe-error.png differ diff --git a/docs/images/aws-marketplace/my-account.png b/docs/images/aws-marketplace/my-account.png new file mode 100644 index 0000000000..3476a7e5aa Binary files /dev/null and b/docs/images/aws-marketplace/my-account.png differ diff --git a/docs/images/aws-marketplace/subscription-in-use.png b/docs/images/aws-marketplace/subscription-in-use.png new file mode 100644 index 0000000000..7797170f6b Binary files /dev/null and b/docs/images/aws-marketplace/subscription-in-use.png differ diff --git a/docs/images/aws-marketplace/subscription-pending.png b/docs/images/aws-marketplace/subscription-pending.png new file mode 100644 index 0000000000..bb608afd68 Binary files /dev/null and b/docs/images/aws-marketplace/subscription-pending.png differ diff --git a/docs/images/aws-role.png b/docs/images/aws-role.png new file mode 100644 index 0000000000..bd63511fb6 Binary files /dev/null and b/docs/images/aws-role.png differ diff --git a/docs/images/azure-credentials.png b/docs/images/azure-credentials.png new file mode 100644 index 0000000000..2f5448d0b9 Binary files /dev/null and b/docs/images/azure-credentials.png differ diff --git a/docs/tutorials/img/add-cloud-provider.png b/docs/images/cli/add-cloud-provider.png similarity index 100% rename from docs/tutorials/img/add-cloud-provider.png rename to docs/images/cli/add-cloud-provider.png diff --git a/docs/images/cli/api-keys/create.png b/docs/images/cli/api-keys/create.png new file mode 100644 index 0000000000..54218c88f8 Binary files /dev/null and b/docs/images/cli/api-keys/create.png differ diff --git a/docs/images/cli/api-keys/created.png b/docs/images/cli/api-keys/created.png new file mode 100644 index 0000000000..a64b6faa74 Binary files /dev/null and b/docs/images/cli/api-keys/created.png differ diff --git a/docs/images/cli/api-keys/list.png b/docs/images/cli/api-keys/list.png new file mode 100644 index 0000000000..3c6020acfc Binary files /dev/null and b/docs/images/cli/api-keys/list.png differ diff --git a/docs/images/cli/api-keys/management.png b/docs/images/cli/api-keys/management.png new file mode 100644 index 0000000000..85ebab8d02 Binary files /dev/null and b/docs/images/cli/api-keys/management.png differ diff --git a/docs/images/cli/api-keys/update.png b/docs/images/cli/api-keys/update.png new file mode 100644 index 0000000000..81cc574801 Binary files /dev/null and b/docs/images/cli/api-keys/update.png differ diff --git a/docs/tutorials/img/bulk-provider-provisioning.png b/docs/images/cli/bulk-provider-provisioning.png similarity index 100% rename from docs/tutorials/img/bulk-provider-provisioning.png rename to docs/images/cli/bulk-provider-provisioning.png diff --git a/docs/tutorials/img/cloud-providers-page.png b/docs/images/cli/cloud-providers-page.png similarity index 100% rename from docs/tutorials/img/cloud-providers-page.png rename to docs/images/cli/cloud-providers-page.png diff --git a/docs/tutorials/img/compliance/compliance-cis-sample1.png b/docs/images/cli/compliance/compliance-cis-sample1.png similarity index 100% rename from docs/tutorials/img/compliance/compliance-cis-sample1.png rename to docs/images/cli/compliance/compliance-cis-sample1.png diff --git a/docs/tutorials/img/compliance/compliance.png b/docs/images/cli/compliance/compliance.png similarity index 100% rename from docs/tutorials/img/compliance/compliance.png rename to docs/images/cli/compliance/compliance.png diff --git a/docs/tutorials/img/create-slack-app.png b/docs/images/cli/create-slack-app.png similarity index 100% rename from docs/tutorials/img/create-slack-app.png rename to docs/images/cli/create-slack-app.png diff --git a/docs/tutorials/img/create-sp.gif b/docs/images/cli/create-sp.gif similarity index 100% rename from docs/tutorials/img/create-sp.gif rename to docs/images/cli/create-sp.gif diff --git a/docs/tutorials/img/dashboard/dashboard-banner.png b/docs/images/cli/dashboard/dashboard-banner.png similarity index 100% rename from docs/tutorials/img/dashboard/dashboard-banner.png rename to docs/images/cli/dashboard/dashboard-banner.png diff --git a/docs/tutorials/img/dashboard/dashboard-compliance.png b/docs/images/cli/dashboard/dashboard-compliance.png similarity index 100% rename from docs/tutorials/img/dashboard/dashboard-compliance.png rename to docs/images/cli/dashboard/dashboard-compliance.png diff --git a/docs/tutorials/img/dashboard/dashboard-files-scanned.png b/docs/images/cli/dashboard/dashboard-files-scanned.png similarity index 100% rename from docs/tutorials/img/dashboard/dashboard-files-scanned.png rename to docs/images/cli/dashboard/dashboard-files-scanned.png diff --git a/docs/tutorials/img/dashboard/dashboard-overview.png b/docs/images/cli/dashboard/dashboard-overview.png similarity index 100% rename from docs/tutorials/img/dashboard/dashboard-overview.png rename to docs/images/cli/dashboard/dashboard-overview.png diff --git a/docs/tutorials/img/dashboard/dropdown.png b/docs/images/cli/dashboard/dropdown.png similarity index 100% rename from docs/tutorials/img/dashboard/dropdown.png rename to docs/images/cli/dashboard/dropdown.png diff --git a/docs/tutorials/img/fixer.png b/docs/images/cli/fixer.png similarity index 100% rename from docs/tutorials/img/fixer.png rename to docs/images/cli/fixer.png diff --git a/docs/tutorials/img/gcp-auth-methods.png b/docs/images/cli/gcp-auth-methods.png similarity index 100% rename from docs/tutorials/img/gcp-auth-methods.png rename to docs/images/cli/gcp-auth-methods.png diff --git a/docs/tutorials/img/gcp-service-account-creds.png b/docs/images/cli/gcp-service-account-creds.png similarity index 100% rename from docs/tutorials/img/gcp-service-account-creds.png rename to docs/images/cli/gcp-service-account-creds.png diff --git a/docs/tutorials/img/github-app-credentials.png b/docs/images/cli/github-app-credentials.png similarity index 100% rename from docs/tutorials/img/github-app-credentials.png rename to docs/images/cli/github-app-credentials.png diff --git a/docs/tutorials/img/github-auth-methods.png b/docs/images/cli/github-auth-methods.png similarity index 100% rename from docs/tutorials/img/github-auth-methods.png rename to docs/images/cli/github-auth-methods.png diff --git a/docs/tutorials/img/github-oauth-credentials.png b/docs/images/cli/github-oauth-credentials.png similarity index 100% rename from docs/tutorials/img/github-oauth-credentials.png rename to docs/images/cli/github-oauth-credentials.png diff --git a/docs/tutorials/img/github-pat-credentials.png b/docs/images/cli/github-pat-credentials.png similarity index 100% rename from docs/tutorials/img/github-pat-credentials.png rename to docs/images/cli/github-pat-credentials.png diff --git a/docs/tutorials/img/install-in-slack-workspace.png b/docs/images/cli/install-in-slack-workspace.png similarity index 100% rename from docs/tutorials/img/install-in-slack-workspace.png rename to docs/images/cli/install-in-slack-workspace.png diff --git a/docs/tutorials/img/integrate-slack-app.png b/docs/images/cli/integrate-slack-app.png similarity index 100% rename from docs/tutorials/img/integrate-slack-app.png rename to docs/images/cli/integrate-slack-app.png diff --git a/docs/tutorials/img/jira/connection-settings.png b/docs/images/cli/jira/connection-settings.png similarity index 100% rename from docs/tutorials/img/jira/connection-settings.png rename to docs/images/cli/jira/connection-settings.png diff --git a/docs/tutorials/img/jira/integrations-tab.png b/docs/images/cli/jira/integrations-tab.png similarity index 100% rename from docs/tutorials/img/jira/integrations-tab.png rename to docs/images/cli/jira/integrations-tab.png diff --git a/docs/tutorials/img/jira/send-to-jira-modal.png b/docs/images/cli/jira/send-to-jira-modal.png similarity index 100% rename from docs/tutorials/img/jira/send-to-jira-modal.png rename to docs/images/cli/jira/send-to-jira-modal.png diff --git a/docs/tutorials/img/lighthouse-architecture.png b/docs/images/cli/lighthouse-architecture.png similarity index 100% rename from docs/tutorials/img/lighthouse-architecture.png rename to docs/images/cli/lighthouse-architecture.png diff --git a/docs/tutorials/img/lighthouse-config.png b/docs/images/cli/lighthouse-config.png similarity index 100% rename from docs/tutorials/img/lighthouse-config.png rename to docs/images/cli/lighthouse-config.png diff --git a/docs/tutorials/img/lighthouse-feature1.png b/docs/images/cli/lighthouse-feature1.png similarity index 100% rename from docs/tutorials/img/lighthouse-feature1.png rename to docs/images/cli/lighthouse-feature1.png diff --git a/docs/tutorials/img/lighthouse-feature2.png b/docs/images/cli/lighthouse-feature2.png similarity index 100% rename from docs/tutorials/img/lighthouse-feature2.png rename to docs/images/cli/lighthouse-feature2.png diff --git a/docs/tutorials/img/lighthouse-feature3.png b/docs/images/cli/lighthouse-feature3.png similarity index 100% rename from docs/tutorials/img/lighthouse-feature3.png rename to docs/images/cli/lighthouse-feature3.png diff --git a/docs/tutorials/img/lighthouse-intro.png b/docs/images/cli/lighthouse-intro.png similarity index 100% rename from docs/tutorials/img/lighthouse-intro.png rename to docs/images/cli/lighthouse-intro.png diff --git a/docs/tutorials/img/mutelist-keys.png b/docs/images/cli/mutelist-keys.png similarity index 100% rename from docs/tutorials/img/mutelist-keys.png rename to docs/images/cli/mutelist-keys.png diff --git a/docs/tutorials/img/mutelist-row.png b/docs/images/cli/mutelist-row.png similarity index 100% rename from docs/tutorials/img/mutelist-row.png rename to docs/images/cli/mutelist-row.png diff --git a/docs/tutorials/img/rbac/invitation_details.png b/docs/images/cli/rbac/invitation_details.png similarity index 100% rename from docs/tutorials/img/rbac/invitation_details.png rename to docs/images/cli/rbac/invitation_details.png diff --git a/docs/tutorials/img/rbac/invitation_details_1.png b/docs/images/cli/rbac/invitation_details_1.png similarity index 100% rename from docs/tutorials/img/rbac/invitation_details_1.png rename to docs/images/cli/rbac/invitation_details_1.png diff --git a/docs/tutorials/img/rbac/invitation_edit.png b/docs/images/cli/rbac/invitation_edit.png similarity index 100% rename from docs/tutorials/img/rbac/invitation_edit.png rename to docs/images/cli/rbac/invitation_edit.png diff --git a/docs/tutorials/img/rbac/invitation_edit_1.png b/docs/images/cli/rbac/invitation_edit_1.png similarity index 100% rename from docs/tutorials/img/rbac/invitation_edit_1.png rename to docs/images/cli/rbac/invitation_edit_1.png diff --git a/docs/tutorials/img/rbac/invitation_info.png b/docs/images/cli/rbac/invitation_info.png similarity index 100% rename from docs/tutorials/img/rbac/invitation_info.png rename to docs/images/cli/rbac/invitation_info.png diff --git a/docs/tutorials/img/rbac/invitation_revoke.png b/docs/images/cli/rbac/invitation_revoke.png similarity index 100% rename from docs/tutorials/img/rbac/invitation_revoke.png rename to docs/images/cli/rbac/invitation_revoke.png diff --git a/docs/tutorials/img/rbac/invitation_sign-up.png b/docs/images/cli/rbac/invitation_sign-up.png similarity index 100% rename from docs/tutorials/img/rbac/invitation_sign-up.png rename to docs/images/cli/rbac/invitation_sign-up.png diff --git a/docs/tutorials/img/rbac/invite.png b/docs/images/cli/rbac/invite.png similarity index 100% rename from docs/tutorials/img/rbac/invite.png rename to docs/images/cli/rbac/invite.png diff --git a/docs/tutorials/img/rbac/membership.png b/docs/images/cli/rbac/membership.png similarity index 100% rename from docs/tutorials/img/rbac/membership.png rename to docs/images/cli/rbac/membership.png diff --git a/docs/tutorials/img/rbac/provider_group.png b/docs/images/cli/rbac/provider_group.png similarity index 100% rename from docs/tutorials/img/rbac/provider_group.png rename to docs/images/cli/rbac/provider_group.png diff --git a/docs/tutorials/img/rbac/provider_group_edit.png b/docs/images/cli/rbac/provider_group_edit.png similarity index 100% rename from docs/tutorials/img/rbac/provider_group_edit.png rename to docs/images/cli/rbac/provider_group_edit.png diff --git a/docs/tutorials/img/rbac/provider_group_edit_1.png b/docs/images/cli/rbac/provider_group_edit_1.png similarity index 100% rename from docs/tutorials/img/rbac/provider_group_edit_1.png rename to docs/images/cli/rbac/provider_group_edit_1.png diff --git a/docs/tutorials/img/rbac/provider_group_remove.png b/docs/images/cli/rbac/provider_group_remove.png similarity index 100% rename from docs/tutorials/img/rbac/provider_group_remove.png rename to docs/images/cli/rbac/provider_group_remove.png diff --git a/docs/tutorials/img/rbac/role_create.png b/docs/images/cli/rbac/role_create.png similarity index 100% rename from docs/tutorials/img/rbac/role_create.png rename to docs/images/cli/rbac/role_create.png diff --git a/docs/tutorials/img/rbac/role_create_1.png b/docs/images/cli/rbac/role_create_1.png similarity index 100% rename from docs/tutorials/img/rbac/role_create_1.png rename to docs/images/cli/rbac/role_create_1.png diff --git a/docs/tutorials/img/rbac/role_edit.png b/docs/images/cli/rbac/role_edit.png similarity index 100% rename from docs/tutorials/img/rbac/role_edit.png rename to docs/images/cli/rbac/role_edit.png diff --git a/docs/tutorials/img/rbac/role_edit_details.png b/docs/images/cli/rbac/role_edit_details.png similarity index 100% rename from docs/tutorials/img/rbac/role_edit_details.png rename to docs/images/cli/rbac/role_edit_details.png diff --git a/docs/tutorials/img/rbac/role_remove.png b/docs/images/cli/rbac/role_remove.png similarity index 100% rename from docs/tutorials/img/rbac/role_remove.png rename to docs/images/cli/rbac/role_remove.png diff --git a/docs/tutorials/img/rbac/user_edit.png b/docs/images/cli/rbac/user_edit.png similarity index 100% rename from docs/tutorials/img/rbac/user_edit.png rename to docs/images/cli/rbac/user_edit.png diff --git a/docs/tutorials/img/rbac/user_edit_details.png b/docs/images/cli/rbac/user_edit_details.png similarity index 100% rename from docs/tutorials/img/rbac/user_edit_details.png rename to docs/images/cli/rbac/user_edit_details.png diff --git a/docs/tutorials/img/rbac/user_remove.png b/docs/images/cli/rbac/user_remove.png similarity index 100% rename from docs/tutorials/img/rbac/user_remove.png rename to docs/images/cli/rbac/user_remove.png diff --git a/docs/tutorials/img/reporting/html-output.png b/docs/images/cli/reporting/html-output.png similarity index 100% rename from docs/tutorials/img/reporting/html-output.png rename to docs/images/cli/reporting/html-output.png diff --git a/docs/tutorials/img/s3/s3-cross-account.png b/docs/images/cli/s3/s3-cross-account.png similarity index 100% rename from docs/tutorials/img/s3/s3-cross-account.png rename to docs/images/cli/s3/s3-cross-account.png diff --git a/docs/tutorials/img/s3/s3-integration-ui-1.png b/docs/images/cli/s3/s3-integration-ui-1.png similarity index 100% rename from docs/tutorials/img/s3/s3-integration-ui-1.png rename to docs/images/cli/s3/s3-integration-ui-1.png diff --git a/docs/tutorials/img/s3/s3-integration-ui-2.png b/docs/images/cli/s3/s3-integration-ui-2.png similarity index 100% rename from docs/tutorials/img/s3/s3-integration-ui-2.png rename to docs/images/cli/s3/s3-integration-ui-2.png diff --git a/docs/tutorials/img/s3/s3-integration-ui-3.png b/docs/images/cli/s3/s3-integration-ui-3.png similarity index 100% rename from docs/tutorials/img/s3/s3-integration-ui-3.png rename to docs/images/cli/s3/s3-integration-ui-3.png diff --git a/docs/tutorials/img/s3/s3-integration-ui-4.png b/docs/images/cli/s3/s3-integration-ui-4.png similarity index 100% rename from docs/tutorials/img/s3/s3-integration-ui-4.png rename to docs/images/cli/s3/s3-integration-ui-4.png diff --git a/docs/tutorials/img/s3/s3-integration-ui-5.png b/docs/images/cli/s3/s3-integration-ui-5.png similarity index 100% rename from docs/tutorials/img/s3/s3-integration-ui-5.png rename to docs/images/cli/s3/s3-integration-ui-5.png diff --git a/docs/tutorials/img/s3/s3-integration-ui-6.png b/docs/images/cli/s3/s3-integration-ui-6.png similarity index 100% rename from docs/tutorials/img/s3/s3-integration-ui-6.png rename to docs/images/cli/s3/s3-integration-ui-6.png diff --git a/docs/tutorials/img/s3/s3-integration-ui-7.png b/docs/images/cli/s3/s3-integration-ui-7.png similarity index 100% rename from docs/tutorials/img/s3/s3-integration-ui-7.png rename to docs/images/cli/s3/s3-integration-ui-7.png diff --git a/docs/tutorials/img/s3/s3-multiple-accounts.png b/docs/images/cli/s3/s3-multiple-accounts.png similarity index 100% rename from docs/tutorials/img/s3/s3-multiple-accounts.png rename to docs/images/cli/s3/s3-multiple-accounts.png diff --git a/docs/tutorials/img/s3/s3-output-folder.png b/docs/images/cli/s3/s3-output-folder.png similarity index 100% rename from docs/tutorials/img/s3/s3-output-folder.png rename to docs/images/cli/s3/s3-output-folder.png diff --git a/docs/tutorials/img/s3/s3-same-account.png b/docs/images/cli/s3/s3-same-account.png similarity index 100% rename from docs/tutorials/img/s3/s3-same-account.png rename to docs/images/cli/s3/s3-same-account.png diff --git a/docs/tutorials/img/saml/app-catalog-browse-prowler-add.png b/docs/images/cli/saml/app-catalog-browse-prowler-add.png similarity index 100% rename from docs/tutorials/img/saml/app-catalog-browse-prowler-add.png rename to docs/images/cli/saml/app-catalog-browse-prowler-add.png diff --git a/docs/tutorials/img/saml/app-catalog-browse-prowler-configure.png b/docs/images/cli/saml/app-catalog-browse-prowler-configure.png similarity index 100% rename from docs/tutorials/img/saml/app-catalog-browse-prowler-configure.png rename to docs/images/cli/saml/app-catalog-browse-prowler-configure.png diff --git a/docs/tutorials/img/saml/app-catalog-browse-prowler.png b/docs/images/cli/saml/app-catalog-browse-prowler.png similarity index 100% rename from docs/tutorials/img/saml/app-catalog-browse-prowler.png rename to docs/images/cli/saml/app-catalog-browse-prowler.png diff --git a/docs/tutorials/img/saml/app-catalog-browse.png b/docs/images/cli/saml/app-catalog-browse.png similarity index 100% rename from docs/tutorials/img/saml/app-catalog-browse.png rename to docs/images/cli/saml/app-catalog-browse.png diff --git a/docs/tutorials/img/saml/idp_config.png b/docs/images/cli/saml/idp_config.png similarity index 100% rename from docs/tutorials/img/saml/idp_config.png rename to docs/images/cli/saml/idp_config.png diff --git a/docs/tutorials/img/saml/saml-signin-1.png b/docs/images/cli/saml/saml-signin-1.png similarity index 100% rename from docs/tutorials/img/saml/saml-signin-1.png rename to docs/images/cli/saml/saml-signin-1.png diff --git a/docs/tutorials/img/saml/saml-signin-2.png b/docs/images/cli/saml/saml-signin-2.png similarity index 100% rename from docs/tutorials/img/saml/saml-signin-2.png rename to docs/images/cli/saml/saml-signin-2.png diff --git a/docs/tutorials/img/saml/saml-sso-azure-1.png b/docs/images/cli/saml/saml-sso-azure-1.png similarity index 100% rename from docs/tutorials/img/saml/saml-sso-azure-1.png rename to docs/images/cli/saml/saml-sso-azure-1.png diff --git a/docs/tutorials/img/saml/saml-sso-azure-2.png b/docs/images/cli/saml/saml-sso-azure-2.png similarity index 100% rename from docs/tutorials/img/saml/saml-sso-azure-2.png rename to docs/images/cli/saml/saml-sso-azure-2.png diff --git a/docs/tutorials/img/saml/saml-sso-azure-3.png b/docs/images/cli/saml/saml-sso-azure-3.png similarity index 100% rename from docs/tutorials/img/saml/saml-sso-azure-3.png rename to docs/images/cli/saml/saml-sso-azure-3.png diff --git a/docs/tutorials/img/saml/saml-sso-azure-4.png b/docs/images/cli/saml/saml-sso-azure-4.png similarity index 100% rename from docs/tutorials/img/saml/saml-sso-azure-4.png rename to docs/images/cli/saml/saml-sso-azure-4.png diff --git a/docs/tutorials/img/saml/saml-sso-azure-5.png b/docs/images/cli/saml/saml-sso-azure-5.png similarity index 100% rename from docs/tutorials/img/saml/saml-sso-azure-5.png rename to docs/images/cli/saml/saml-sso-azure-5.png diff --git a/docs/tutorials/img/saml/saml-sso-azure-6.png b/docs/images/cli/saml/saml-sso-azure-6.png similarity index 100% rename from docs/tutorials/img/saml/saml-sso-azure-6.png rename to docs/images/cli/saml/saml-sso-azure-6.png diff --git a/docs/tutorials/img/saml/saml-sso-azure-7.png b/docs/images/cli/saml/saml-sso-azure-7.png similarity index 100% rename from docs/tutorials/img/saml/saml-sso-azure-7.png rename to docs/images/cli/saml/saml-sso-azure-7.png diff --git a/docs/tutorials/img/saml/saml-sso-azure-8.png b/docs/images/cli/saml/saml-sso-azure-8.png similarity index 100% rename from docs/tutorials/img/saml/saml-sso-azure-8.png rename to docs/images/cli/saml/saml-sso-azure-8.png diff --git a/docs/tutorials/img/saml/saml-sso-azure-9.png b/docs/images/cli/saml/saml-sso-azure-9.png similarity index 100% rename from docs/tutorials/img/saml/saml-sso-azure-9.png rename to docs/images/cli/saml/saml-sso-azure-9.png diff --git a/docs/tutorials/img/saml/saml-step-1.png b/docs/images/cli/saml/saml-step-1.png similarity index 100% rename from docs/tutorials/img/saml/saml-step-1.png rename to docs/images/cli/saml/saml-step-1.png diff --git a/docs/tutorials/img/saml/saml-step-2.png b/docs/images/cli/saml/saml-step-2.png similarity index 100% rename from docs/tutorials/img/saml/saml-step-2.png rename to docs/images/cli/saml/saml-step-2.png diff --git a/docs/tutorials/img/saml/saml-step-3.png b/docs/images/cli/saml/saml-step-3.png similarity index 100% rename from docs/tutorials/img/saml/saml-step-3.png rename to docs/images/cli/saml/saml-step-3.png diff --git a/docs/tutorials/img/saml/saml-step-4.png b/docs/images/cli/saml/saml-step-4.png similarity index 100% rename from docs/tutorials/img/saml/saml-step-4.png rename to docs/images/cli/saml/saml-step-4.png diff --git a/docs/tutorials/img/saml/saml-step-remove.png b/docs/images/cli/saml/saml-step-remove.png similarity index 100% rename from docs/tutorials/img/saml/saml-step-remove.png rename to docs/images/cli/saml/saml-step-remove.png diff --git a/docs/tutorials/img/saml/saml_attribute_statements.png b/docs/images/cli/saml/saml_attribute_statements.png similarity index 100% rename from docs/tutorials/img/saml/saml_attribute_statements.png rename to docs/images/cli/saml/saml_attribute_statements.png diff --git a/docs/tutorials/img/security-hub/create-integration.png b/docs/images/cli/security-hub/create-integration.png similarity index 100% rename from docs/tutorials/img/security-hub/create-integration.png rename to docs/images/cli/security-hub/create-integration.png diff --git a/docs/tutorials/img/security-hub/integration-settings.png b/docs/images/cli/security-hub/integration-settings.png similarity index 100% rename from docs/tutorials/img/security-hub/integration-settings.png rename to docs/images/cli/security-hub/integration-settings.png diff --git a/docs/tutorials/img/security-hub/integrations-tab.png b/docs/images/cli/security-hub/integrations-tab.png similarity index 100% rename from docs/tutorials/img/security-hub/integrations-tab.png rename to docs/images/cli/security-hub/integrations-tab.png diff --git a/docs/tutorials/img/slack-app-token.png b/docs/images/cli/slack-app-token.png similarity index 100% rename from docs/tutorials/img/slack-app-token.png rename to docs/images/cli/slack-app-token.png diff --git a/docs/tutorials/img/slack-prowler-message.png b/docs/images/cli/slack-prowler-message.png similarity index 100% rename from docs/tutorials/img/slack-prowler-message.png rename to docs/images/cli/slack-prowler-message.png diff --git a/docs/tutorials/img/social-login/social_login_buttons.png b/docs/images/cli/social-login/social_login_buttons.png similarity index 100% rename from docs/tutorials/img/social-login/social_login_buttons.png rename to docs/images/cli/social-login/social_login_buttons.png diff --git a/docs/tutorials/img/social-login/social_login_buttons_disabled.png b/docs/images/cli/social-login/social_login_buttons_disabled.png similarity index 100% rename from docs/tutorials/img/social-login/social_login_buttons_disabled.png rename to docs/images/cli/social-login/social_login_buttons_disabled.png diff --git a/docs/images/compliance.png b/docs/images/compliance.png new file mode 100644 index 0000000000..b08fcc3651 Binary files /dev/null and b/docs/images/compliance.png differ diff --git a/docs/images/compliance_download.png b/docs/images/compliance_download.png new file mode 100644 index 0000000000..32aed141b3 Binary files /dev/null and b/docs/images/compliance_download.png differ diff --git a/docs/images/compliance_section.png b/docs/images/compliance_section.png new file mode 100644 index 0000000000..2b64031550 Binary files /dev/null and b/docs/images/compliance_section.png differ diff --git a/docs/images/connect-aws-credentials.png b/docs/images/connect-aws-credentials.png new file mode 100644 index 0000000000..7ea1f193c3 Binary files /dev/null and b/docs/images/connect-aws-credentials.png differ diff --git a/docs/images/connect-aws-role.png b/docs/images/connect-aws-role.png new file mode 100644 index 0000000000..78c8f6f390 Binary files /dev/null and b/docs/images/connect-aws-role.png differ diff --git a/docs/images/create-management-group.gif b/docs/images/create-management-group.gif new file mode 100644 index 0000000000..0088e78495 Binary files /dev/null and b/docs/images/create-management-group.gif differ diff --git a/docs/images/download_output.png b/docs/images/download_output.png new file mode 100644 index 0000000000..452851b84d Binary files /dev/null and b/docs/images/download_output.png differ diff --git a/docs/images/extended-display.png b/docs/images/extended-display.png new file mode 100644 index 0000000000..4e97d278ff Binary files /dev/null and b/docs/images/extended-display.png differ diff --git a/docs/images/findings.png b/docs/images/findings.png new file mode 100644 index 0000000000..e2ea9c56ce Binary files /dev/null and b/docs/images/findings.png differ diff --git a/docs/images/gcp-credentials.png b/docs/images/gcp-credentials.png new file mode 100644 index 0000000000..4f6dae3b1e Binary files /dev/null and b/docs/images/gcp-credentials.png differ diff --git a/docs/images/html-output.png b/docs/images/html-output.png new file mode 100644 index 0000000000..00a8b60812 Binary files /dev/null and b/docs/images/html-output.png differ diff --git a/docs/images/issues.png b/docs/images/issues.png new file mode 100644 index 0000000000..009bc0d447 Binary files /dev/null and b/docs/images/issues.png differ diff --git a/docs/images/kubernetes-credentials.png b/docs/images/kubernetes-credentials.png new file mode 100644 index 0000000000..b461ed218a Binary files /dev/null and b/docs/images/kubernetes-credentials.png differ diff --git a/docs/images/log-in.png b/docs/images/log-in.png new file mode 100644 index 0000000000..9e6d410abe Binary files /dev/null and b/docs/images/log-in.png differ diff --git a/docs/tutorials/microsoft365/img/m365-credentials.png b/docs/images/m365-credentials.png similarity index 100% rename from docs/tutorials/microsoft365/img/m365-credentials.png rename to docs/images/m365-credentials.png diff --git a/docs/images/mutelist-ui-1.png b/docs/images/mutelist-ui-1.png new file mode 100644 index 0000000000..8114e64fdf Binary files /dev/null and b/docs/images/mutelist-ui-1.png differ diff --git a/docs/images/mutelist-ui-2.png b/docs/images/mutelist-ui-2.png new file mode 100644 index 0000000000..847f7c1b16 Binary files /dev/null and b/docs/images/mutelist-ui-2.png differ diff --git a/docs/images/mutelist-ui-3.png b/docs/images/mutelist-ui-3.png new file mode 100644 index 0000000000..da1951c533 Binary files /dev/null and b/docs/images/mutelist-ui-3.png differ diff --git a/docs/images/mutelist-ui-4.png b/docs/images/mutelist-ui-4.png new file mode 100644 index 0000000000..82ce685e51 Binary files /dev/null and b/docs/images/mutelist-ui-4.png differ diff --git a/docs/images/mutelist-ui-5.png b/docs/images/mutelist-ui-5.png new file mode 100644 index 0000000000..128bf30731 Binary files /dev/null and b/docs/images/mutelist-ui-5.png differ diff --git a/docs/images/mutelist-ui-6.png b/docs/images/mutelist-ui-6.png new file mode 100644 index 0000000000..659eccb61e Binary files /dev/null and b/docs/images/mutelist-ui-6.png differ diff --git a/docs/images/mutelist-ui-7.png b/docs/images/mutelist-ui-7.png new file mode 100644 index 0000000000..e6352c97e9 Binary files /dev/null and b/docs/images/mutelist-ui-7.png differ diff --git a/docs/images/mutelist-ui-8.png b/docs/images/mutelist-ui-8.png new file mode 100644 index 0000000000..54e2110edb Binary files /dev/null and b/docs/images/mutelist-ui-8.png differ diff --git a/docs/images/mutelist-ui-9.png b/docs/images/mutelist-ui-9.png new file mode 100644 index 0000000000..cba2ece1a3 Binary files /dev/null and b/docs/images/mutelist-ui-9.png differ diff --git a/docs/images/nacho.png b/docs/images/nacho.png new file mode 100644 index 0000000000..549d6b6c6d Binary files /dev/null and b/docs/images/nacho.png differ diff --git a/docs/images/output_folder.png b/docs/images/output_folder.png new file mode 100644 index 0000000000..11ae4d9925 Binary files /dev/null and b/docs/images/output_folder.png differ diff --git a/docs/images/pepe.png b/docs/images/pepe.png new file mode 100644 index 0000000000..a9aed2be15 Binary files /dev/null and b/docs/images/pepe.png differ diff --git a/docs/images/products/dashboard.png b/docs/images/products/dashboard.png new file mode 100644 index 0000000000..2392debbb7 Binary files /dev/null and b/docs/images/products/dashboard.png differ diff --git a/docs/images/products/overview.png b/docs/images/products/overview.png new file mode 100644 index 0000000000..724ffc9588 Binary files /dev/null and b/docs/images/products/overview.png differ diff --git a/docs/images/products/prowler-app-architecture.png b/docs/images/products/prowler-app-architecture.png new file mode 100644 index 0000000000..889bc0da88 Binary files /dev/null and b/docs/images/products/prowler-app-architecture.png differ diff --git a/docs/images/products/prowler-hub.webp b/docs/images/products/prowler-hub.webp new file mode 100644 index 0000000000..b489ff9012 Binary files /dev/null and b/docs/images/products/prowler-hub.webp differ diff --git a/docs/images/provider-added.png b/docs/images/provider-added.png new file mode 100644 index 0000000000..cfb94d2cab Binary files /dev/null and b/docs/images/provider-added.png differ diff --git a/docs/tutorials/gcp/img/access-console.png b/docs/images/providers/access-console.png similarity index 100% rename from docs/tutorials/gcp/img/access-console.png rename to docs/images/providers/access-console.png diff --git a/docs/tutorials/mongodbatlas/img/access-manager.png b/docs/images/providers/access-manager.png similarity index 100% rename from docs/tutorials/mongodbatlas/img/access-manager.png rename to docs/images/providers/access-manager.png diff --git a/docs/tutorials/aws/img/add-account-id.png b/docs/images/providers/add-account-id.png similarity index 100% rename from docs/tutorials/aws/img/add-account-id.png rename to docs/images/providers/add-account-id.png diff --git a/docs/tutorials/azure/img/add-api-permission.png b/docs/images/providers/add-api-permission.png similarity index 100% rename from docs/tutorials/azure/img/add-api-permission.png rename to docs/images/providers/add-api-permission.png diff --git a/docs/tutorials/microsoft365/img/add-app-api-permission.png b/docs/images/providers/add-app-api-permission.png similarity index 100% rename from docs/tutorials/microsoft365/img/add-app-api-permission.png rename to docs/images/providers/add-app-api-permission.png diff --git a/docs/tutorials/microsoft365/img/add-assginments.png b/docs/images/providers/add-assginments.png similarity index 100% rename from docs/tutorials/microsoft365/img/add-assginments.png rename to docs/images/providers/add-assginments.png diff --git a/docs/tutorials/azure/img/add-credentials-azure-prowler-cloud.png b/docs/images/providers/add-credentials-azure-prowler-cloud.png similarity index 100% rename from docs/tutorials/azure/img/add-credentials-azure-prowler-cloud.png rename to docs/images/providers/add-credentials-azure-prowler-cloud.png diff --git a/docs/tutorials/azure/img/add-custom-role-json.png b/docs/images/providers/add-custom-role-json.png similarity index 100% rename from docs/tutorials/azure/img/add-custom-role-json.png rename to docs/images/providers/add-custom-role-json.png diff --git a/docs/tutorials/azure/img/add-custom-role.png b/docs/images/providers/add-custom-role.png similarity index 100% rename from docs/tutorials/azure/img/add-custom-role.png rename to docs/images/providers/add-custom-role.png diff --git a/docs/tutorials/microsoft365/img/add-delegated-api-permission.png b/docs/images/providers/add-delegated-api-permission.png similarity index 100% rename from docs/tutorials/microsoft365/img/add-delegated-api-permission.png rename to docs/images/providers/add-delegated-api-permission.png diff --git a/docs/tutorials/microsoft365/img/add-domain-id.png b/docs/images/providers/add-domain-id.png similarity index 100% rename from docs/tutorials/microsoft365/img/add-domain-id.png rename to docs/images/providers/add-domain-id.png diff --git a/docs/tutorials/github/img/add-github-account-id.png b/docs/images/providers/add-github-account-id.png similarity index 100% rename from docs/tutorials/github/img/add-github-account-id.png rename to docs/images/providers/add-github-account-id.png diff --git a/docs/images/providers/add-iac-repo.png b/docs/images/providers/add-iac-repo.png new file mode 100644 index 0000000000..31981e7fd1 Binary files /dev/null and b/docs/images/providers/add-iac-repo.png differ diff --git a/docs/tutorials/mongodbatlas/img/add-ip.png b/docs/images/providers/add-ip.png similarity index 100% rename from docs/tutorials/mongodbatlas/img/add-ip.png rename to docs/images/providers/add-ip.png diff --git a/docs/tutorials/gcp/img/add-project-id.png b/docs/images/providers/add-project-id.png similarity index 100% rename from docs/tutorials/gcp/img/add-project-id.png rename to docs/images/providers/add-project-id.png diff --git a/docs/tutorials/azure/img/add-reader-role.png b/docs/images/providers/add-reader-role.png similarity index 100% rename from docs/tutorials/azure/img/add-reader-role.png rename to docs/images/providers/add-reader-role.png diff --git a/docs/tutorials/azure/img/add-role-assigment.png b/docs/images/providers/add-role-assigment.png similarity index 100% rename from docs/tutorials/azure/img/add-role-assigment.png rename to docs/images/providers/add-role-assigment.png diff --git a/docs/tutorials/azure/img/add-subscription-id.png b/docs/images/providers/add-subscription-id.png similarity index 100% rename from docs/tutorials/azure/img/add-subscription-id.png rename to docs/images/providers/add-subscription-id.png diff --git a/docs/tutorials/microsoft365/img/api-permissions-page.png b/docs/images/providers/api-permissions-page.png similarity index 100% rename from docs/tutorials/microsoft365/img/api-permissions-page.png rename to docs/images/providers/api-permissions-page.png diff --git a/docs/tutorials/azure/img/api-permissions-result.png b/docs/images/providers/api-permissions-result.png similarity index 100% rename from docs/tutorials/azure/img/api-permissions-result.png rename to docs/images/providers/api-permissions-result.png diff --git a/docs/tutorials/microsoft365/img/app-overview.png b/docs/images/providers/app-overview.png similarity index 100% rename from docs/tutorials/microsoft365/img/app-overview.png rename to docs/images/providers/app-overview.png diff --git a/docs/tutorials/microsoft365/img/app-permissions.png b/docs/images/providers/app-permissions.png similarity index 100% rename from docs/tutorials/microsoft365/img/app-permissions.png rename to docs/images/providers/app-permissions.png diff --git a/docs/tutorials/microsoft365/img/app-registration-menu.png b/docs/images/providers/app-registration-menu.png similarity index 100% rename from docs/tutorials/microsoft365/img/app-registration-menu.png rename to docs/images/providers/app-registration-menu.png diff --git a/docs/tutorials/azure/img/application-permissions-inside-graph.png b/docs/images/providers/application-permissions-inside-graph.png similarity index 100% rename from docs/tutorials/azure/img/application-permissions-inside-graph.png rename to docs/images/providers/application-permissions-inside-graph.png diff --git a/docs/tutorials/microsoft365/img/assign-global-reader-role.png b/docs/images/providers/assign-global-reader-role.png similarity index 100% rename from docs/tutorials/microsoft365/img/assign-global-reader-role.png rename to docs/images/providers/assign-global-reader-role.png diff --git a/docs/tutorials/aws/img/assume-role-overview.png b/docs/images/providers/assume-role-overview.png similarity index 100% rename from docs/tutorials/aws/img/assume-role-overview.png rename to docs/images/providers/assume-role-overview.png diff --git a/docs/tutorials/github/img/auth-github-app.png b/docs/images/providers/auth-github-app.png similarity index 100% rename from docs/tutorials/github/img/auth-github-app.png rename to docs/images/providers/auth-github-app.png diff --git a/docs/tutorials/github/img/auth-oauth.png b/docs/images/providers/auth-oauth.png similarity index 100% rename from docs/tutorials/github/img/auth-oauth.png rename to docs/images/providers/auth-oauth.png diff --git a/docs/tutorials/github/img/auth-pat.png b/docs/images/providers/auth-pat.png similarity index 100% rename from docs/tutorials/github/img/auth-pat.png rename to docs/images/providers/auth-pat.png diff --git a/docs/tutorials/gcp/img/authorize-cloud-shell.png b/docs/images/providers/authorize-cloud-shell.png similarity index 100% rename from docs/tutorials/gcp/img/authorize-cloud-shell.png rename to docs/images/providers/authorize-cloud-shell.png diff --git a/docs/tutorials/aws/img/aws-account-id.png b/docs/images/providers/aws-account-id.png similarity index 100% rename from docs/tutorials/aws/img/aws-account-id.png rename to docs/images/providers/aws-account-id.png diff --git a/docs/tutorials/aws/img/aws-cloudshell.png b/docs/images/providers/aws-cloudshell.png similarity index 100% rename from docs/tutorials/aws/img/aws-cloudshell.png rename to docs/images/providers/aws-cloudshell.png diff --git a/docs/images/providers/certificate-form.png b/docs/images/providers/certificate-form.png new file mode 100644 index 0000000000..b19c079d23 Binary files /dev/null and b/docs/images/providers/certificate-form.png differ diff --git a/docs/tutorials/microsoft365/img/certificates-and-secrets.png b/docs/images/providers/certificates-and-secrets.png similarity index 100% rename from docs/tutorials/microsoft365/img/certificates-and-secrets.png rename to docs/images/providers/certificates-and-secrets.png diff --git a/docs/tutorials/azure/img/click-add-permissions.png b/docs/images/providers/click-add-permissions.png similarity index 100% rename from docs/tutorials/azure/img/click-add-permissions.png rename to docs/images/providers/click-add-permissions.png diff --git a/docs/tutorials/azure/img/click-next-azure.png b/docs/images/providers/click-next-azure.png similarity index 100% rename from docs/tutorials/azure/img/click-next-azure.png rename to docs/images/providers/click-next-azure.png diff --git a/docs/tutorials/microsoft365/img/click-next-m365.png b/docs/images/providers/click-next-m365.png similarity index 100% rename from docs/tutorials/microsoft365/img/click-next-m365.png rename to docs/images/providers/click-next-m365.png diff --git a/docs/tutorials/aws/img/cloudformation-nav.png b/docs/images/providers/cloudformation-nav.png similarity index 100% rename from docs/tutorials/aws/img/cloudformation-nav.png rename to docs/images/providers/cloudformation-nav.png diff --git a/docs/tutorials/aws/img/cloudshell-output.png b/docs/images/providers/cloudshell-output.png similarity index 100% rename from docs/tutorials/aws/img/cloudshell-output.png rename to docs/images/providers/cloudshell-output.png diff --git a/docs/tutorials/aws/img/connect-via-credentials.png b/docs/images/providers/connect-via-credentials.png similarity index 100% rename from docs/tutorials/aws/img/connect-via-credentials.png rename to docs/images/providers/connect-via-credentials.png diff --git a/docs/tutorials/gcp/img/copy-auth-code.png b/docs/images/providers/copy-auth-code.png similarity index 100% rename from docs/tutorials/gcp/img/copy-auth-code.png rename to docs/images/providers/copy-auth-code.png diff --git a/docs/tutorials/mongodbatlas/img/copy-key.png b/docs/images/providers/copy-key.png similarity index 100% rename from docs/tutorials/mongodbatlas/img/copy-key.png rename to docs/images/providers/copy-key.png diff --git a/docs/tutorials/mongodbatlas/img/create-api-key.png b/docs/images/providers/create-api-key.png similarity index 100% rename from docs/tutorials/mongodbatlas/img/create-api-key.png rename to docs/images/providers/create-api-key.png diff --git a/docs/tutorials/aws/img/create-stack.png b/docs/images/providers/create-stack.png similarity index 100% rename from docs/tutorials/aws/img/create-stack.png rename to docs/images/providers/create-stack.png diff --git a/docs/tutorials/microsoft365/img/custom-domain-names.png b/docs/images/providers/custom-domain-names.png similarity index 100% rename from docs/tutorials/microsoft365/img/custom-domain-names.png rename to docs/images/providers/custom-domain-names.png diff --git a/docs/tutorials/microsoft365/img/directory-permission-delegated.png b/docs/images/providers/directory-permission-delegated.png similarity index 100% rename from docs/tutorials/microsoft365/img/directory-permission-delegated.png rename to docs/images/providers/directory-permission-delegated.png diff --git a/docs/tutorials/microsoft365/img/directory-permission.png b/docs/images/providers/directory-permission.png similarity index 100% rename from docs/tutorials/microsoft365/img/directory-permission.png rename to docs/images/providers/directory-permission.png diff --git a/docs/tutorials/azure/img/domain-permission.png b/docs/images/providers/domain-permission.png similarity index 100% rename from docs/tutorials/azure/img/domain-permission.png rename to docs/images/providers/domain-permission.png diff --git a/docs/tutorials/azure/img/download-prowler-role.png b/docs/images/providers/download-prowler-role.png similarity index 100% rename from docs/tutorials/azure/img/download-prowler-role.png rename to docs/images/providers/download-prowler-role.png diff --git a/docs/tutorials/aws/img/download-role-template.png b/docs/images/providers/download-role-template.png similarity index 100% rename from docs/tutorials/aws/img/download-role-template.png rename to docs/images/providers/download-role-template.png diff --git a/docs/tutorials/aws/img/enable-2.png b/docs/images/providers/enable-2.png similarity index 100% rename from docs/tutorials/aws/img/enable-2.png rename to docs/images/providers/enable-2.png diff --git a/docs/tutorials/aws/img/enable-partner-integration-2.png b/docs/images/providers/enable-partner-integration-2.png similarity index 100% rename from docs/tutorials/aws/img/enable-partner-integration-2.png rename to docs/images/providers/enable-partner-integration-2.png diff --git a/docs/tutorials/aws/img/enable-partner-integration-3.png b/docs/images/providers/enable-partner-integration-3.png similarity index 100% rename from docs/tutorials/aws/img/enable-partner-integration-3.png rename to docs/images/providers/enable-partner-integration-3.png diff --git a/docs/tutorials/aws/img/enable-partner-integration-4.png b/docs/images/providers/enable-partner-integration-4.png similarity index 100% rename from docs/tutorials/aws/img/enable-partner-integration-4.png rename to docs/images/providers/enable-partner-integration-4.png diff --git a/docs/tutorials/aws/img/enable-partner-integration.png b/docs/images/providers/enable-partner-integration.png similarity index 100% rename from docs/tutorials/aws/img/enable-partner-integration.png rename to docs/images/providers/enable-partner-integration.png diff --git a/docs/tutorials/aws/img/enable.png b/docs/images/providers/enable.png similarity index 100% rename from docs/tutorials/aws/img/enable.png rename to docs/images/providers/enable.png diff --git a/docs/tutorials/gcp/img/enter-auth-code.png b/docs/images/providers/enter-auth-code.png similarity index 100% rename from docs/tutorials/gcp/img/enter-auth-code.png rename to docs/images/providers/enter-auth-code.png diff --git a/docs/tutorials/gcp/img/enter-credentials-prowler-cloud.png b/docs/images/providers/enter-credentials-prowler-cloud.png similarity index 100% rename from docs/tutorials/gcp/img/enter-credentials-prowler-cloud.png rename to docs/images/providers/enter-credentials-prowler-cloud.png diff --git a/docs/tutorials/microsoft365/img/exchange-permission.png b/docs/images/providers/exchange-permission.png similarity index 100% rename from docs/tutorials/microsoft365/img/exchange-permission.png rename to docs/images/providers/exchange-permission.png diff --git a/docs/tutorials/aws/img/fill-stack-data.png b/docs/images/providers/fill-stack-data.png similarity index 100% rename from docs/tutorials/aws/img/fill-stack-data.png rename to docs/images/providers/fill-stack-data.png diff --git a/docs/tutorials/microsoft365/img/final-permissions-m365.png b/docs/images/providers/final-permissions-m365.png similarity index 100% rename from docs/tutorials/microsoft365/img/final-permissions-m365.png rename to docs/images/providers/final-permissions-m365.png diff --git a/docs/tutorials/microsoft365/img/final-permissions.png b/docs/images/providers/final-permissions.png similarity index 100% rename from docs/tutorials/microsoft365/img/final-permissions.png rename to docs/images/providers/final-permissions.png diff --git a/docs/tutorials/aws/img/finding-details.png b/docs/images/providers/finding-details.png similarity index 100% rename from docs/tutorials/aws/img/finding-details.png rename to docs/images/providers/finding-details.png diff --git a/docs/tutorials/aws/img/findings.png b/docs/images/providers/findings.png similarity index 100% rename from docs/tutorials/aws/img/findings.png rename to docs/images/providers/findings.png diff --git a/docs/tutorials/aws/img/get-external-id-prowler-cloud.png b/docs/images/providers/get-external-id-prowler-cloud.png similarity index 100% rename from docs/tutorials/aws/img/get-external-id-prowler-cloud.png rename to docs/images/providers/get-external-id-prowler-cloud.png diff --git a/docs/tutorials/gcp/img/get-needed-values-auth.png b/docs/images/providers/get-needed-values-auth.png similarity index 100% rename from docs/tutorials/gcp/img/get-needed-values-auth.png rename to docs/images/providers/get-needed-values-auth.png diff --git a/docs/tutorials/aws/img/get-role-arn.png b/docs/images/providers/get-role-arn.png similarity index 100% rename from docs/tutorials/aws/img/get-role-arn.png rename to docs/images/providers/get-role-arn.png diff --git a/docs/tutorials/azure/img/get-subscription-id.png b/docs/images/providers/get-subscription-id.png similarity index 100% rename from docs/tutorials/azure/img/get-subscription-id.png rename to docs/images/providers/get-subscription-id.png diff --git a/docs/tutorials/gcp/img/get-temp-file-credentials.png b/docs/images/providers/get-temp-file-credentials.png similarity index 100% rename from docs/tutorials/gcp/img/get-temp-file-credentials.png rename to docs/images/providers/get-temp-file-credentials.png diff --git a/docs/tutorials/github/img/github-pat-permissions.png b/docs/images/providers/github-pat-permissions.png similarity index 100% rename from docs/tutorials/github/img/github-pat-permissions.png rename to docs/images/providers/github-pat-permissions.png diff --git a/docs/tutorials/microsoft365/img/global-reader-role.png b/docs/images/providers/global-reader-role.png similarity index 100% rename from docs/tutorials/microsoft365/img/global-reader-role.png rename to docs/images/providers/global-reader-role.png diff --git a/docs/tutorials/microsoft365/img/global-reader.png b/docs/images/providers/global-reader.png similarity index 100% rename from docs/tutorials/microsoft365/img/global-reader.png rename to docs/images/providers/global-reader.png diff --git a/docs/tutorials/microsoft365/img/grant-admin-consent-delegated.png b/docs/images/providers/grant-admin-consent-delegated.png similarity index 100% rename from docs/tutorials/microsoft365/img/grant-admin-consent-delegated.png rename to docs/images/providers/grant-admin-consent-delegated.png diff --git a/docs/tutorials/microsoft365/img/grant-admin-consent-for-role.png b/docs/images/providers/grant-admin-consent-for-role.png similarity index 100% rename from docs/tutorials/microsoft365/img/grant-admin-consent-for-role.png rename to docs/images/providers/grant-admin-consent-for-role.png diff --git a/docs/tutorials/microsoft365/img/grant-admin-consent.png b/docs/images/providers/grant-admin-consent.png similarity index 100% rename from docs/tutorials/microsoft365/img/grant-admin-consent.png rename to docs/images/providers/grant-admin-consent.png diff --git a/docs/tutorials/microsoft365/img/grant-external-api-permissions.png b/docs/images/providers/grant-external-api-permissions.png similarity index 100% rename from docs/tutorials/microsoft365/img/grant-external-api-permissions.png rename to docs/images/providers/grant-external-api-permissions.png diff --git a/docs/tutorials/microsoft365/img/here.png b/docs/images/providers/here.png similarity index 100% rename from docs/tutorials/microsoft365/img/here.png rename to docs/images/providers/here.png diff --git a/docs/images/providers/iac-authentication.png b/docs/images/providers/iac-authentication.png new file mode 100644 index 0000000000..c6caad4ae1 Binary files /dev/null and b/docs/images/providers/iac-authentication.png differ diff --git a/docs/images/providers/iac-verify-connection.png b/docs/images/providers/iac-verify-connection.png new file mode 100644 index 0000000000..bc918f6c24 Binary files /dev/null and b/docs/images/providers/iac-verify-connection.png differ diff --git a/docs/tutorials/azure/img/iam-azure-page.png b/docs/images/providers/iam-azure-page.png similarity index 100% rename from docs/tutorials/azure/img/iam-azure-page.png rename to docs/images/providers/iam-azure-page.png diff --git a/docs/tutorials/mongodbatlas/img/ip-access-list.png b/docs/images/providers/ip-access-list.png similarity index 100% rename from docs/tutorials/mongodbatlas/img/ip-access-list.png rename to docs/images/providers/ip-access-list.png diff --git a/docs/tutorials/aws/img/launch-scan-button-prowler-cloud.png b/docs/images/providers/launch-scan-button-prowler-cloud.png similarity index 100% rename from docs/tutorials/aws/img/launch-scan-button-prowler-cloud.png rename to docs/images/providers/launch-scan-button-prowler-cloud.png diff --git a/docs/tutorials/microsoft365/img/launch-scan.png b/docs/images/providers/launch-scan.png similarity index 100% rename from docs/tutorials/microsoft365/img/launch-scan.png rename to docs/images/providers/launch-scan.png diff --git a/docs/images/providers/m365-auth-selection-form.png b/docs/images/providers/m365-auth-selection-form.png new file mode 100644 index 0000000000..4757f44d96 Binary files /dev/null and b/docs/images/providers/m365-auth-selection-form.png differ diff --git a/docs/images/providers/m365-credentials.png b/docs/images/providers/m365-credentials.png new file mode 100644 index 0000000000..ab912f9d8e Binary files /dev/null and b/docs/images/providers/m365-credentials.png differ diff --git a/docs/tutorials/azure/img/member-select-app-prowler.png b/docs/images/providers/member-select-app-prowler.png similarity index 100% rename from docs/tutorials/azure/img/member-select-app-prowler.png rename to docs/images/providers/member-select-app-prowler.png diff --git a/docs/tutorials/microsoft365/img/microsoft-entra-id.png b/docs/images/providers/microsoft-entra-id.png similarity index 100% rename from docs/tutorials/microsoft365/img/microsoft-entra-id.png rename to docs/images/providers/microsoft-entra-id.png diff --git a/docs/tutorials/azure/img/microsoft-graph-detail.png b/docs/images/providers/microsoft-graph-detail.png similarity index 100% rename from docs/tutorials/azure/img/microsoft-graph-detail.png rename to docs/images/providers/microsoft-graph-detail.png diff --git a/docs/tutorials/mongodbatlas/img/modify-permission.png b/docs/images/providers/modify-permission.png similarity index 100% rename from docs/tutorials/mongodbatlas/img/modify-permission.png rename to docs/images/providers/modify-permission.png diff --git a/docs/tutorials/microsoft365/img/new-client-secret.png b/docs/images/providers/new-client-secret.png similarity index 100% rename from docs/tutorials/microsoft365/img/new-client-secret.png rename to docs/images/providers/new-client-secret.png diff --git a/docs/tutorials/microsoft365/img/new-registration.png b/docs/images/providers/new-registration.png similarity index 100% rename from docs/tutorials/microsoft365/img/new-registration.png rename to docs/images/providers/new-registration.png diff --git a/docs/tutorials/aws/img/next-button-prowler-cloud.png b/docs/images/providers/next-button-prowler-cloud.png similarity index 100% rename from docs/tutorials/aws/img/next-button-prowler-cloud.png rename to docs/images/providers/next-button-prowler-cloud.png diff --git a/docs/tutorials/aws/img/next-cloudformation-template.png b/docs/images/providers/next-cloudformation-template.png similarity index 100% rename from docs/tutorials/aws/img/next-cloudformation-template.png rename to docs/images/providers/next-cloudformation-template.png diff --git a/docs/tutorials/gcp/img/open-link-console.png b/docs/images/providers/open-link-console.png similarity index 100% rename from docs/tutorials/gcp/img/open-link-console.png rename to docs/images/providers/open-link-console.png diff --git a/docs/tutorials/mongodbatlas/img/organization-access.png b/docs/images/providers/organization-access.png similarity index 100% rename from docs/tutorials/mongodbatlas/img/organization-access.png rename to docs/images/providers/organization-access.png diff --git a/docs/tutorials/aws/img/paste-role-arn-prowler.png b/docs/images/providers/paste-role-arn-prowler.png similarity index 100% rename from docs/tutorials/aws/img/paste-role-arn-prowler.png rename to docs/images/providers/paste-role-arn-prowler.png diff --git a/docs/tutorials/azure/img/policy-permission.png b/docs/images/providers/policy-permission.png similarity index 100% rename from docs/tutorials/azure/img/policy-permission.png rename to docs/images/providers/policy-permission.png diff --git a/docs/tutorials/gcp/img/project-id-console.png b/docs/images/providers/project-id-console.png similarity index 100% rename from docs/tutorials/gcp/img/project-id-console.png rename to docs/images/providers/project-id-console.png diff --git a/docs/tutorials/azure/img/prowler-app-registration.png b/docs/images/providers/prowler-app-registration.png similarity index 100% rename from docs/tutorials/azure/img/prowler-app-registration.png rename to docs/images/providers/prowler-app-registration.png diff --git a/docs/tutorials/aws/img/prowler-cloud-credentials-next.png b/docs/images/providers/prowler-cloud-credentials-next.png similarity index 100% rename from docs/tutorials/aws/img/prowler-cloud-credentials-next.png rename to docs/images/providers/prowler-cloud-credentials-next.png diff --git a/docs/tutorials/aws/img/prowler-cloud-external-id.png b/docs/images/providers/prowler-cloud-external-id.png similarity index 100% rename from docs/tutorials/aws/img/prowler-cloud-external-id.png rename to docs/images/providers/prowler-cloud-external-id.png diff --git a/docs/tutorials/aws/img/prowler-scan-pre-info.png b/docs/images/providers/prowler-scan-pre-info.png similarity index 100% rename from docs/tutorials/aws/img/prowler-scan-pre-info.png rename to docs/images/providers/prowler-scan-pre-info.png diff --git a/docs/tutorials/aws/img/prowler-scan-role-template.png b/docs/images/providers/prowler-scan-role-template.png similarity index 100% rename from docs/tutorials/aws/img/prowler-scan-role-template.png rename to docs/images/providers/prowler-scan-role-template.png diff --git a/docs/tutorials/azure/img/review-and-assign-last-step.png b/docs/images/providers/review-and-assign-last-step.png similarity index 100% rename from docs/tutorials/azure/img/review-and-assign-last-step.png rename to docs/images/providers/review-and-assign-last-step.png diff --git a/docs/tutorials/azure/img/review-and-create.png b/docs/images/providers/review-and-create.png similarity index 100% rename from docs/tutorials/azure/img/review-and-create.png rename to docs/images/providers/review-and-create.png diff --git a/docs/tutorials/gcp/img/run-gcloud-auth.png b/docs/images/providers/run-gcloud-auth.png similarity index 100% rename from docs/tutorials/gcp/img/run-gcloud-auth.png rename to docs/images/providers/run-gcloud-auth.png diff --git a/docs/tutorials/microsoft365/img/search-default-domain.png b/docs/images/providers/search-default-domain.png similarity index 100% rename from docs/tutorials/microsoft365/img/search-default-domain.png rename to docs/images/providers/search-default-domain.png diff --git a/docs/tutorials/microsoft365/img/search-domain-names.png b/docs/images/providers/search-domain-names.png similarity index 100% rename from docs/tutorials/microsoft365/img/search-domain-names.png rename to docs/images/providers/search-domain-names.png diff --git a/docs/tutorials/microsoft365/img/search-exchange-api.png b/docs/images/providers/search-exchange-api.png similarity index 100% rename from docs/tutorials/microsoft365/img/search-exchange-api.png rename to docs/images/providers/search-exchange-api.png diff --git a/docs/tutorials/azure/img/search-microsoft-entra-id.png b/docs/images/providers/search-microsoft-entra-id.png similarity index 100% rename from docs/tutorials/azure/img/search-microsoft-entra-id.png rename to docs/images/providers/search-microsoft-entra-id.png diff --git a/docs/tutorials/microsoft365/img/search-skype-teams-tenant-admin-api.png b/docs/images/providers/search-skype-teams-tenant-admin-api.png similarity index 100% rename from docs/tutorials/microsoft365/img/search-skype-teams-tenant-admin-api.png rename to docs/images/providers/search-skype-teams-tenant-admin-api.png diff --git a/docs/tutorials/azure/img/search-subscriptions.png b/docs/images/providers/search-subscriptions.png similarity index 100% rename from docs/tutorials/azure/img/search-subscriptions.png rename to docs/images/providers/search-subscriptions.png diff --git a/docs/images/providers/secret-form.png b/docs/images/providers/secret-form.png new file mode 100644 index 0000000000..e202c56aab Binary files /dev/null and b/docs/images/providers/secret-form.png differ diff --git a/docs/tutorials/github/img/select-auth-method.png b/docs/images/providers/select-auth-method.png similarity index 100% rename from docs/tutorials/github/img/select-auth-method.png rename to docs/images/providers/select-auth-method.png diff --git a/docs/tutorials/aws/img/select-aws.png b/docs/images/providers/select-aws.png similarity index 100% rename from docs/tutorials/aws/img/select-aws.png rename to docs/images/providers/select-aws.png diff --git a/docs/tutorials/azure/img/select-azure-prowler-cloud.png b/docs/images/providers/select-azure-prowler-cloud.png similarity index 100% rename from docs/tutorials/azure/img/select-azure-prowler-cloud.png rename to docs/images/providers/select-azure-prowler-cloud.png diff --git a/docs/tutorials/azure/img/select-custom-role-prowler.png b/docs/images/providers/select-custom-role-prowler.png similarity index 100% rename from docs/tutorials/azure/img/select-custom-role-prowler.png rename to docs/images/providers/select-custom-role-prowler.png diff --git a/docs/tutorials/gcp/img/select-gcp.png b/docs/images/providers/select-gcp.png similarity index 100% rename from docs/tutorials/gcp/img/select-gcp.png rename to docs/images/providers/select-gcp.png diff --git a/docs/tutorials/github/img/select-github.png b/docs/images/providers/select-github.png similarity index 100% rename from docs/tutorials/github/img/select-github.png rename to docs/images/providers/select-github.png diff --git a/docs/images/providers/select-iac.png b/docs/images/providers/select-iac.png new file mode 100644 index 0000000000..1dc474cbe8 Binary files /dev/null and b/docs/images/providers/select-iac.png differ diff --git a/docs/tutorials/microsoft365/img/select-m365-prowler-cloud.png b/docs/images/providers/select-m365-prowler-cloud.png similarity index 100% rename from docs/tutorials/microsoft365/img/select-m365-prowler-cloud.png rename to docs/images/providers/select-m365-prowler-cloud.png diff --git a/docs/tutorials/azure/img/select-members-iam.png b/docs/images/providers/select-members-iam.png similarity index 100% rename from docs/tutorials/azure/img/select-members-iam.png rename to docs/images/providers/select-members-iam.png diff --git a/docs/tutorials/aws/img/stack-creation-second-step.png b/docs/images/providers/stack-creation-second-step.png similarity index 100% rename from docs/tutorials/aws/img/stack-creation-second-step.png rename to docs/images/providers/stack-creation-second-step.png diff --git a/docs/tutorials/aws/img/submit-third-page.png b/docs/images/providers/submit-third-page.png similarity index 100% rename from docs/tutorials/aws/img/submit-third-page.png rename to docs/images/providers/submit-third-page.png diff --git a/docs/tutorials/azure/img/subscription-page-azure.png b/docs/images/providers/subscription-page-azure.png similarity index 100% rename from docs/tutorials/azure/img/subscription-page-azure.png rename to docs/images/providers/subscription-page-azure.png diff --git a/docs/tutorials/gcp/img/take-account-email.png b/docs/images/providers/take-account-email.png similarity index 100% rename from docs/tutorials/gcp/img/take-account-email.png rename to docs/images/providers/take-account-email.png diff --git a/docs/tutorials/microsoft365/img/teams-permission.png b/docs/images/providers/teams-permission.png similarity index 100% rename from docs/tutorials/microsoft365/img/teams-permission.png rename to docs/images/providers/teams-permission.png diff --git a/docs/tutorials/aws/img/upload-template-file.png b/docs/images/providers/upload-template-file.png similarity index 100% rename from docs/tutorials/aws/img/upload-template-file.png rename to docs/images/providers/upload-template-file.png diff --git a/docs/tutorials/aws/img/upload-template-from-downloads.png b/docs/images/providers/upload-template-from-downloads.png similarity index 100% rename from docs/tutorials/aws/img/upload-template-from-downloads.png rename to docs/images/providers/upload-template-from-downloads.png diff --git a/docs/tutorials/microsoft365/img/user-domains.png b/docs/images/providers/user-domains.png similarity index 100% rename from docs/tutorials/microsoft365/img/user-domains.png rename to docs/images/providers/user-domains.png diff --git a/docs/tutorials/microsoft365/img/user-info-page.png b/docs/images/providers/user-info-page.png similarity index 100% rename from docs/tutorials/microsoft365/img/user-info-page.png rename to docs/images/providers/user-info-page.png diff --git a/docs/tutorials/azure/img/user-permission.png b/docs/images/providers/user-permission.png similarity index 100% rename from docs/tutorials/azure/img/user-permission.png rename to docs/images/providers/user-permission.png diff --git a/docs/tutorials/microsoft365/img/user-role-page.png b/docs/images/providers/user-role-page.png similarity index 100% rename from docs/tutorials/microsoft365/img/user-role-page.png rename to docs/images/providers/user-role-page.png diff --git a/docs/images/prowler-app/add-cloud-provider.png b/docs/images/prowler-app/add-cloud-provider.png new file mode 100644 index 0000000000..dd42e73047 Binary files /dev/null and b/docs/images/prowler-app/add-cloud-provider.png differ diff --git a/docs/images/prowler-app/bulk-provider-provisioning.png b/docs/images/prowler-app/bulk-provider-provisioning.png new file mode 100644 index 0000000000..c43c746874 Binary files /dev/null and b/docs/images/prowler-app/bulk-provider-provisioning.png differ diff --git a/docs/images/prowler-app/cloud-providers-page.png b/docs/images/prowler-app/cloud-providers-page.png new file mode 100644 index 0000000000..0581e0132f Binary files /dev/null and b/docs/images/prowler-app/cloud-providers-page.png differ diff --git a/docs/images/prowler-app/compliance/compliance-cis-sample1.png b/docs/images/prowler-app/compliance/compliance-cis-sample1.png new file mode 100644 index 0000000000..925e5c80f8 Binary files /dev/null and b/docs/images/prowler-app/compliance/compliance-cis-sample1.png differ diff --git a/docs/images/prowler-app/compliance/compliance.png b/docs/images/prowler-app/compliance/compliance.png new file mode 100644 index 0000000000..de5c27fbe3 Binary files /dev/null and b/docs/images/prowler-app/compliance/compliance.png differ diff --git a/docs/images/prowler-app/create-slack-app.png b/docs/images/prowler-app/create-slack-app.png new file mode 100644 index 0000000000..dc910634ec Binary files /dev/null and b/docs/images/prowler-app/create-slack-app.png differ diff --git a/docs/images/prowler-app/create-sp.gif b/docs/images/prowler-app/create-sp.gif new file mode 100644 index 0000000000..1ed345bb6a Binary files /dev/null and b/docs/images/prowler-app/create-sp.gif differ diff --git a/docs/images/prowler-app/dashboard/dashboard-banner.png b/docs/images/prowler-app/dashboard/dashboard-banner.png new file mode 100644 index 0000000000..3d6dc92304 Binary files /dev/null and b/docs/images/prowler-app/dashboard/dashboard-banner.png differ diff --git a/docs/images/prowler-app/dashboard/dashboard-compliance.png b/docs/images/prowler-app/dashboard/dashboard-compliance.png new file mode 100644 index 0000000000..95fb7179f9 Binary files /dev/null and b/docs/images/prowler-app/dashboard/dashboard-compliance.png differ diff --git a/docs/images/prowler-app/dashboard/dashboard-files-scanned.png b/docs/images/prowler-app/dashboard/dashboard-files-scanned.png new file mode 100644 index 0000000000..2629a8ac77 Binary files /dev/null and b/docs/images/prowler-app/dashboard/dashboard-files-scanned.png differ diff --git a/docs/images/prowler-app/dashboard/dashboard-overview.png b/docs/images/prowler-app/dashboard/dashboard-overview.png new file mode 100644 index 0000000000..a5a470cb0e Binary files /dev/null and b/docs/images/prowler-app/dashboard/dashboard-overview.png differ diff --git a/docs/images/prowler-app/dashboard/dropdown.png b/docs/images/prowler-app/dashboard/dropdown.png new file mode 100644 index 0000000000..b1df77798e Binary files /dev/null and b/docs/images/prowler-app/dashboard/dropdown.png differ diff --git a/docs/images/prowler-app/fixer.png b/docs/images/prowler-app/fixer.png new file mode 100644 index 0000000000..d794485454 Binary files /dev/null and b/docs/images/prowler-app/fixer.png differ diff --git a/docs/images/prowler-app/gcp-auth-methods.png b/docs/images/prowler-app/gcp-auth-methods.png new file mode 100644 index 0000000000..dc8681396e Binary files /dev/null and b/docs/images/prowler-app/gcp-auth-methods.png differ diff --git a/docs/images/prowler-app/gcp-service-account-creds.png b/docs/images/prowler-app/gcp-service-account-creds.png new file mode 100644 index 0000000000..af09776ab1 Binary files /dev/null and b/docs/images/prowler-app/gcp-service-account-creds.png differ diff --git a/docs/images/prowler-app/github-app-credentials.png b/docs/images/prowler-app/github-app-credentials.png new file mode 100644 index 0000000000..1a71db4fbd Binary files /dev/null and b/docs/images/prowler-app/github-app-credentials.png differ diff --git a/docs/images/prowler-app/github-auth-methods.png b/docs/images/prowler-app/github-auth-methods.png new file mode 100644 index 0000000000..17414054aa Binary files /dev/null and b/docs/images/prowler-app/github-auth-methods.png differ diff --git a/docs/images/prowler-app/github-oauth-credentials.png b/docs/images/prowler-app/github-oauth-credentials.png new file mode 100644 index 0000000000..5d84c4bf75 Binary files /dev/null and b/docs/images/prowler-app/github-oauth-credentials.png differ diff --git a/docs/images/prowler-app/github-pat-credentials.png b/docs/images/prowler-app/github-pat-credentials.png new file mode 100644 index 0000000000..044cbb7996 Binary files /dev/null and b/docs/images/prowler-app/github-pat-credentials.png differ diff --git a/docs/images/prowler-app/install-in-slack-workspace.png b/docs/images/prowler-app/install-in-slack-workspace.png new file mode 100644 index 0000000000..649c107cdd Binary files /dev/null and b/docs/images/prowler-app/install-in-slack-workspace.png differ diff --git a/docs/images/prowler-app/integrate-slack-app.png b/docs/images/prowler-app/integrate-slack-app.png new file mode 100644 index 0000000000..beea9ef56b Binary files /dev/null and b/docs/images/prowler-app/integrate-slack-app.png differ diff --git a/docs/images/prowler-app/jira/connection-settings.png b/docs/images/prowler-app/jira/connection-settings.png new file mode 100644 index 0000000000..86ebe73dea Binary files /dev/null and b/docs/images/prowler-app/jira/connection-settings.png differ diff --git a/docs/images/prowler-app/jira/integrations-tab.png b/docs/images/prowler-app/jira/integrations-tab.png new file mode 100644 index 0000000000..e71773fc52 Binary files /dev/null and b/docs/images/prowler-app/jira/integrations-tab.png differ diff --git a/docs/images/prowler-app/jira/send-to-jira-modal.png b/docs/images/prowler-app/jira/send-to-jira-modal.png new file mode 100644 index 0000000000..2cf498926a Binary files /dev/null and b/docs/images/prowler-app/jira/send-to-jira-modal.png differ diff --git a/docs/images/prowler-app/lighthouse-architecture.png b/docs/images/prowler-app/lighthouse-architecture.png new file mode 100644 index 0000000000..63202ce7c7 Binary files /dev/null and b/docs/images/prowler-app/lighthouse-architecture.png differ diff --git a/docs/images/prowler-app/lighthouse-config.png b/docs/images/prowler-app/lighthouse-config.png new file mode 100644 index 0000000000..8bae46f51b Binary files /dev/null and b/docs/images/prowler-app/lighthouse-config.png differ diff --git a/docs/images/prowler-app/lighthouse-feature1.png b/docs/images/prowler-app/lighthouse-feature1.png new file mode 100644 index 0000000000..4568f5752a Binary files /dev/null and b/docs/images/prowler-app/lighthouse-feature1.png differ diff --git a/docs/images/prowler-app/lighthouse-feature2.png b/docs/images/prowler-app/lighthouse-feature2.png new file mode 100644 index 0000000000..01e72bbaf4 Binary files /dev/null and b/docs/images/prowler-app/lighthouse-feature2.png differ diff --git a/docs/images/prowler-app/lighthouse-feature3.png b/docs/images/prowler-app/lighthouse-feature3.png new file mode 100644 index 0000000000..a40177cf18 Binary files /dev/null and b/docs/images/prowler-app/lighthouse-feature3.png differ diff --git a/docs/images/prowler-app/lighthouse-intro.png b/docs/images/prowler-app/lighthouse-intro.png new file mode 100644 index 0000000000..38d3f0819c Binary files /dev/null and b/docs/images/prowler-app/lighthouse-intro.png differ diff --git a/docs/images/prowler-app/mutelist-keys.png b/docs/images/prowler-app/mutelist-keys.png new file mode 100644 index 0000000000..0822344376 Binary files /dev/null and b/docs/images/prowler-app/mutelist-keys.png differ diff --git a/docs/images/prowler-app/mutelist-row.png b/docs/images/prowler-app/mutelist-row.png new file mode 100644 index 0000000000..0ed7d75a7d Binary files /dev/null and b/docs/images/prowler-app/mutelist-row.png differ diff --git a/docs/images/prowler-app/rbac/invitation_details.png b/docs/images/prowler-app/rbac/invitation_details.png new file mode 100644 index 0000000000..656a698308 Binary files /dev/null and b/docs/images/prowler-app/rbac/invitation_details.png differ diff --git a/docs/images/prowler-app/rbac/invitation_details_1.png b/docs/images/prowler-app/rbac/invitation_details_1.png new file mode 100644 index 0000000000..e167db74af Binary files /dev/null and b/docs/images/prowler-app/rbac/invitation_details_1.png differ diff --git a/docs/images/prowler-app/rbac/invitation_edit.png b/docs/images/prowler-app/rbac/invitation_edit.png new file mode 100644 index 0000000000..ef3d81f192 Binary files /dev/null and b/docs/images/prowler-app/rbac/invitation_edit.png differ diff --git a/docs/images/prowler-app/rbac/invitation_edit_1.png b/docs/images/prowler-app/rbac/invitation_edit_1.png new file mode 100644 index 0000000000..6d1a1d2223 Binary files /dev/null and b/docs/images/prowler-app/rbac/invitation_edit_1.png differ diff --git a/docs/images/prowler-app/rbac/invitation_info.png b/docs/images/prowler-app/rbac/invitation_info.png new file mode 100644 index 0000000000..a6ec05f976 Binary files /dev/null and b/docs/images/prowler-app/rbac/invitation_info.png differ diff --git a/docs/images/prowler-app/rbac/invitation_revoke.png b/docs/images/prowler-app/rbac/invitation_revoke.png new file mode 100644 index 0000000000..6c4e042c16 Binary files /dev/null and b/docs/images/prowler-app/rbac/invitation_revoke.png differ diff --git a/docs/images/prowler-app/rbac/invitation_sign-up.png b/docs/images/prowler-app/rbac/invitation_sign-up.png new file mode 100644 index 0000000000..f5b67a7762 Binary files /dev/null and b/docs/images/prowler-app/rbac/invitation_sign-up.png differ diff --git a/docs/images/prowler-app/rbac/invite.png b/docs/images/prowler-app/rbac/invite.png new file mode 100644 index 0000000000..dd60aba314 Binary files /dev/null and b/docs/images/prowler-app/rbac/invite.png differ diff --git a/docs/images/prowler-app/rbac/membership.png b/docs/images/prowler-app/rbac/membership.png new file mode 100644 index 0000000000..e2b96e40f5 Binary files /dev/null and b/docs/images/prowler-app/rbac/membership.png differ diff --git a/docs/images/prowler-app/rbac/provider_group.png b/docs/images/prowler-app/rbac/provider_group.png new file mode 100644 index 0000000000..878306e02f Binary files /dev/null and b/docs/images/prowler-app/rbac/provider_group.png differ diff --git a/docs/images/prowler-app/rbac/provider_group_edit.png b/docs/images/prowler-app/rbac/provider_group_edit.png new file mode 100644 index 0000000000..5de9649578 Binary files /dev/null and b/docs/images/prowler-app/rbac/provider_group_edit.png differ diff --git a/docs/images/prowler-app/rbac/provider_group_edit_1.png b/docs/images/prowler-app/rbac/provider_group_edit_1.png new file mode 100644 index 0000000000..ed4a090eed Binary files /dev/null and b/docs/images/prowler-app/rbac/provider_group_edit_1.png differ diff --git a/docs/images/prowler-app/rbac/provider_group_remove.png b/docs/images/prowler-app/rbac/provider_group_remove.png new file mode 100644 index 0000000000..580a8533cf Binary files /dev/null and b/docs/images/prowler-app/rbac/provider_group_remove.png differ diff --git a/docs/images/prowler-app/rbac/role_create.png b/docs/images/prowler-app/rbac/role_create.png new file mode 100644 index 0000000000..8dc270f209 Binary files /dev/null and b/docs/images/prowler-app/rbac/role_create.png differ diff --git a/docs/images/prowler-app/rbac/role_create_1.png b/docs/images/prowler-app/rbac/role_create_1.png new file mode 100644 index 0000000000..a96b0ac9ef Binary files /dev/null and b/docs/images/prowler-app/rbac/role_create_1.png differ diff --git a/docs/images/prowler-app/rbac/role_edit.png b/docs/images/prowler-app/rbac/role_edit.png new file mode 100644 index 0000000000..775b4b8562 Binary files /dev/null and b/docs/images/prowler-app/rbac/role_edit.png differ diff --git a/docs/images/prowler-app/rbac/role_edit_details.png b/docs/images/prowler-app/rbac/role_edit_details.png new file mode 100644 index 0000000000..6170f210b1 Binary files /dev/null and b/docs/images/prowler-app/rbac/role_edit_details.png differ diff --git a/docs/images/prowler-app/rbac/role_remove.png b/docs/images/prowler-app/rbac/role_remove.png new file mode 100644 index 0000000000..613f770bd4 Binary files /dev/null and b/docs/images/prowler-app/rbac/role_remove.png differ diff --git a/docs/images/prowler-app/rbac/user_edit.png b/docs/images/prowler-app/rbac/user_edit.png new file mode 100644 index 0000000000..421ea93156 Binary files /dev/null and b/docs/images/prowler-app/rbac/user_edit.png differ diff --git a/docs/images/prowler-app/rbac/user_edit_details.png b/docs/images/prowler-app/rbac/user_edit_details.png new file mode 100644 index 0000000000..3e4734f142 Binary files /dev/null and b/docs/images/prowler-app/rbac/user_edit_details.png differ diff --git a/docs/images/prowler-app/rbac/user_remove.png b/docs/images/prowler-app/rbac/user_remove.png new file mode 100644 index 0000000000..4c368ded66 Binary files /dev/null and b/docs/images/prowler-app/rbac/user_remove.png differ diff --git a/docs/images/prowler-app/reporting/html-output.png b/docs/images/prowler-app/reporting/html-output.png new file mode 100644 index 0000000000..00a8b60812 Binary files /dev/null and b/docs/images/prowler-app/reporting/html-output.png differ diff --git a/docs/images/prowler-app/s3/s3-cross-account.png b/docs/images/prowler-app/s3/s3-cross-account.png new file mode 100644 index 0000000000..ad34a303fc Binary files /dev/null and b/docs/images/prowler-app/s3/s3-cross-account.png differ diff --git a/docs/images/prowler-app/s3/s3-integration-ui-1.png b/docs/images/prowler-app/s3/s3-integration-ui-1.png new file mode 100644 index 0000000000..1081f20453 Binary files /dev/null and b/docs/images/prowler-app/s3/s3-integration-ui-1.png differ diff --git a/docs/images/prowler-app/s3/s3-integration-ui-2.png b/docs/images/prowler-app/s3/s3-integration-ui-2.png new file mode 100644 index 0000000000..618621ce26 Binary files /dev/null and b/docs/images/prowler-app/s3/s3-integration-ui-2.png differ diff --git a/docs/images/prowler-app/s3/s3-integration-ui-3.png b/docs/images/prowler-app/s3/s3-integration-ui-3.png new file mode 100644 index 0000000000..6fabbe2ed3 Binary files /dev/null and b/docs/images/prowler-app/s3/s3-integration-ui-3.png differ diff --git a/docs/images/prowler-app/s3/s3-integration-ui-4.png b/docs/images/prowler-app/s3/s3-integration-ui-4.png new file mode 100644 index 0000000000..1967eda8af Binary files /dev/null and b/docs/images/prowler-app/s3/s3-integration-ui-4.png differ diff --git a/docs/images/prowler-app/s3/s3-integration-ui-5.png b/docs/images/prowler-app/s3/s3-integration-ui-5.png new file mode 100644 index 0000000000..ffdea752f1 Binary files /dev/null and b/docs/images/prowler-app/s3/s3-integration-ui-5.png differ diff --git a/docs/images/prowler-app/s3/s3-integration-ui-6.png b/docs/images/prowler-app/s3/s3-integration-ui-6.png new file mode 100644 index 0000000000..330588e17b Binary files /dev/null and b/docs/images/prowler-app/s3/s3-integration-ui-6.png differ diff --git a/docs/images/prowler-app/s3/s3-integration-ui-7.png b/docs/images/prowler-app/s3/s3-integration-ui-7.png new file mode 100644 index 0000000000..3448f82a48 Binary files /dev/null and b/docs/images/prowler-app/s3/s3-integration-ui-7.png differ diff --git a/docs/images/prowler-app/s3/s3-multiple-accounts.png b/docs/images/prowler-app/s3/s3-multiple-accounts.png new file mode 100644 index 0000000000..db30e09841 Binary files /dev/null and b/docs/images/prowler-app/s3/s3-multiple-accounts.png differ diff --git a/docs/images/prowler-app/s3/s3-output-folder.png b/docs/images/prowler-app/s3/s3-output-folder.png new file mode 100644 index 0000000000..448687a8b1 Binary files /dev/null and b/docs/images/prowler-app/s3/s3-output-folder.png differ diff --git a/docs/images/prowler-app/s3/s3-same-account.png b/docs/images/prowler-app/s3/s3-same-account.png new file mode 100644 index 0000000000..39e586c762 Binary files /dev/null and b/docs/images/prowler-app/s3/s3-same-account.png differ diff --git a/docs/images/prowler-app/saml/app-catalog-browse-prowler-add.png b/docs/images/prowler-app/saml/app-catalog-browse-prowler-add.png new file mode 100644 index 0000000000..71c70bb7cd Binary files /dev/null and b/docs/images/prowler-app/saml/app-catalog-browse-prowler-add.png differ diff --git a/docs/images/prowler-app/saml/app-catalog-browse-prowler-configure.png b/docs/images/prowler-app/saml/app-catalog-browse-prowler-configure.png new file mode 100644 index 0000000000..6d5715eb92 Binary files /dev/null and b/docs/images/prowler-app/saml/app-catalog-browse-prowler-configure.png differ diff --git a/docs/images/prowler-app/saml/app-catalog-browse-prowler.png b/docs/images/prowler-app/saml/app-catalog-browse-prowler.png new file mode 100644 index 0000000000..5706997980 Binary files /dev/null and b/docs/images/prowler-app/saml/app-catalog-browse-prowler.png differ diff --git a/docs/images/prowler-app/saml/app-catalog-browse.png b/docs/images/prowler-app/saml/app-catalog-browse.png new file mode 100644 index 0000000000..d0a9b6926d Binary files /dev/null and b/docs/images/prowler-app/saml/app-catalog-browse.png differ diff --git a/docs/images/prowler-app/saml/idp_config.png b/docs/images/prowler-app/saml/idp_config.png new file mode 100644 index 0000000000..0665b34886 Binary files /dev/null and b/docs/images/prowler-app/saml/idp_config.png differ diff --git a/docs/images/prowler-app/saml/saml-signin-1.png b/docs/images/prowler-app/saml/saml-signin-1.png new file mode 100644 index 0000000000..5da2e84711 Binary files /dev/null and b/docs/images/prowler-app/saml/saml-signin-1.png differ diff --git a/docs/images/prowler-app/saml/saml-signin-2.png b/docs/images/prowler-app/saml/saml-signin-2.png new file mode 100644 index 0000000000..f0f7082fc9 Binary files /dev/null and b/docs/images/prowler-app/saml/saml-signin-2.png differ diff --git a/docs/images/prowler-app/saml/saml-sso-azure-1.png b/docs/images/prowler-app/saml/saml-sso-azure-1.png new file mode 100644 index 0000000000..c714d742e7 Binary files /dev/null and b/docs/images/prowler-app/saml/saml-sso-azure-1.png differ diff --git a/docs/images/prowler-app/saml/saml-sso-azure-2.png b/docs/images/prowler-app/saml/saml-sso-azure-2.png new file mode 100644 index 0000000000..483af4548c Binary files /dev/null and b/docs/images/prowler-app/saml/saml-sso-azure-2.png differ diff --git a/docs/images/prowler-app/saml/saml-sso-azure-3.png b/docs/images/prowler-app/saml/saml-sso-azure-3.png new file mode 100644 index 0000000000..7983694c5c Binary files /dev/null and b/docs/images/prowler-app/saml/saml-sso-azure-3.png differ diff --git a/docs/images/prowler-app/saml/saml-sso-azure-4.png b/docs/images/prowler-app/saml/saml-sso-azure-4.png new file mode 100644 index 0000000000..f6c6029343 Binary files /dev/null and b/docs/images/prowler-app/saml/saml-sso-azure-4.png differ diff --git a/docs/images/prowler-app/saml/saml-sso-azure-5.png b/docs/images/prowler-app/saml/saml-sso-azure-5.png new file mode 100644 index 0000000000..6dbeb6e65d Binary files /dev/null and b/docs/images/prowler-app/saml/saml-sso-azure-5.png differ diff --git a/docs/images/prowler-app/saml/saml-sso-azure-6.png b/docs/images/prowler-app/saml/saml-sso-azure-6.png new file mode 100644 index 0000000000..b351004a87 Binary files /dev/null and b/docs/images/prowler-app/saml/saml-sso-azure-6.png differ diff --git a/docs/images/prowler-app/saml/saml-sso-azure-7.png b/docs/images/prowler-app/saml/saml-sso-azure-7.png new file mode 100644 index 0000000000..76551bef06 Binary files /dev/null and b/docs/images/prowler-app/saml/saml-sso-azure-7.png differ diff --git a/docs/images/prowler-app/saml/saml-sso-azure-8.png b/docs/images/prowler-app/saml/saml-sso-azure-8.png new file mode 100644 index 0000000000..5dd2f1e1ef Binary files /dev/null and b/docs/images/prowler-app/saml/saml-sso-azure-8.png differ diff --git a/docs/images/prowler-app/saml/saml-sso-azure-9.png b/docs/images/prowler-app/saml/saml-sso-azure-9.png new file mode 100644 index 0000000000..130dd5e7cd Binary files /dev/null and b/docs/images/prowler-app/saml/saml-sso-azure-9.png differ diff --git a/docs/images/prowler-app/saml/saml-step-1.png b/docs/images/prowler-app/saml/saml-step-1.png new file mode 100644 index 0000000000..d505c2597c Binary files /dev/null and b/docs/images/prowler-app/saml/saml-step-1.png differ diff --git a/docs/images/prowler-app/saml/saml-step-2.png b/docs/images/prowler-app/saml/saml-step-2.png new file mode 100644 index 0000000000..ddb036c98d Binary files /dev/null and b/docs/images/prowler-app/saml/saml-step-2.png differ diff --git a/docs/images/prowler-app/saml/saml-step-3.png b/docs/images/prowler-app/saml/saml-step-3.png new file mode 100644 index 0000000000..2b8dfbd845 Binary files /dev/null and b/docs/images/prowler-app/saml/saml-step-3.png differ diff --git a/docs/images/prowler-app/saml/saml-step-4.png b/docs/images/prowler-app/saml/saml-step-4.png new file mode 100644 index 0000000000..cca0987c50 Binary files /dev/null and b/docs/images/prowler-app/saml/saml-step-4.png differ diff --git a/docs/images/prowler-app/saml/saml-step-remove.png b/docs/images/prowler-app/saml/saml-step-remove.png new file mode 100644 index 0000000000..f0868691af Binary files /dev/null and b/docs/images/prowler-app/saml/saml-step-remove.png differ diff --git a/docs/images/prowler-app/saml/saml_attribute_statements.png b/docs/images/prowler-app/saml/saml_attribute_statements.png new file mode 100644 index 0000000000..62d6366131 Binary files /dev/null and b/docs/images/prowler-app/saml/saml_attribute_statements.png differ diff --git a/docs/images/prowler-app/security-hub/create-integration.png b/docs/images/prowler-app/security-hub/create-integration.png new file mode 100644 index 0000000000..145e68d201 Binary files /dev/null and b/docs/images/prowler-app/security-hub/create-integration.png differ diff --git a/docs/images/prowler-app/security-hub/integration-settings.png b/docs/images/prowler-app/security-hub/integration-settings.png new file mode 100644 index 0000000000..3a32bf0830 Binary files /dev/null and b/docs/images/prowler-app/security-hub/integration-settings.png differ diff --git a/docs/images/prowler-app/security-hub/integrations-tab.png b/docs/images/prowler-app/security-hub/integrations-tab.png new file mode 100644 index 0000000000..9d11cae33a Binary files /dev/null and b/docs/images/prowler-app/security-hub/integrations-tab.png differ diff --git a/docs/images/prowler-app/slack-app-token.png b/docs/images/prowler-app/slack-app-token.png new file mode 100644 index 0000000000..550970503b Binary files /dev/null and b/docs/images/prowler-app/slack-app-token.png differ diff --git a/docs/images/prowler-app/slack-prowler-message.png b/docs/images/prowler-app/slack-prowler-message.png new file mode 100644 index 0000000000..1151a7be7f Binary files /dev/null and b/docs/images/prowler-app/slack-prowler-message.png differ diff --git a/docs/images/prowler-app/social-login/social_login_buttons.png b/docs/images/prowler-app/social-login/social_login_buttons.png new file mode 100644 index 0000000000..476081e0fc Binary files /dev/null and b/docs/images/prowler-app/social-login/social_login_buttons.png differ diff --git a/docs/images/prowler-app/social-login/social_login_buttons_disabled.png b/docs/images/prowler-app/social-login/social_login_buttons_disabled.png new file mode 100644 index 0000000000..7b11a7802d Binary files /dev/null and b/docs/images/prowler-app/social-login/social_login_buttons_disabled.png differ diff --git a/docs/images/prowler-cli-quick.gif b/docs/images/prowler-cli-quick.gif new file mode 100644 index 0000000000..16197a5ba7 Binary files /dev/null and b/docs/images/prowler-cli-quick.gif differ diff --git a/docs/images/prowler-logo-black.png b/docs/images/prowler-logo-black.png new file mode 100644 index 0000000000..3a6ed6490d Binary files /dev/null and b/docs/images/prowler-logo-black.png differ diff --git a/docs/images/prowler-logo-white.png b/docs/images/prowler-logo-white.png new file mode 100644 index 0000000000..b2d551f995 Binary files /dev/null and b/docs/images/prowler-logo-white.png differ diff --git a/docs/images/prowler-multi-account-environment.png b/docs/images/prowler-multi-account-environment.png new file mode 100644 index 0000000000..89adaf5f69 Binary files /dev/null and b/docs/images/prowler-multi-account-environment.png differ diff --git a/docs/images/prowler-single-account-environment.png b/docs/images/prowler-single-account-environment.png new file mode 100644 index 0000000000..585fe33a57 Binary files /dev/null and b/docs/images/prowler-single-account-environment.png differ diff --git a/docs/images/prowler_mcp_schema_dark.png b/docs/images/prowler_mcp_schema_dark.png new file mode 100644 index 0000000000..7771557601 Binary files /dev/null and b/docs/images/prowler_mcp_schema_dark.png differ diff --git a/docs/images/prowler_mcp_schema_light.png b/docs/images/prowler_mcp_schema_light.png new file mode 100644 index 0000000000..c542d84ed2 Binary files /dev/null and b/docs/images/prowler_mcp_schema_light.png differ diff --git a/docs/images/quick-inventory.jpg b/docs/images/quick-inventory.jpg new file mode 100644 index 0000000000..a7124bb7ae Binary files /dev/null and b/docs/images/quick-inventory.jpg differ diff --git a/docs/images/register-application.png b/docs/images/register-application.png new file mode 100644 index 0000000000..dd69291987 Binary files /dev/null and b/docs/images/register-application.png differ diff --git a/docs/images/scan-progress.png b/docs/images/scan-progress.png new file mode 100644 index 0000000000..3378775dcd Binary files /dev/null and b/docs/images/scan-progress.png differ diff --git a/docs/images/scan_jobs_section.png b/docs/images/scan_jobs_section.png new file mode 100644 index 0000000000..e307cf7f60 Binary files /dev/null and b/docs/images/scan_jobs_section.png differ diff --git a/docs/images/select-provider.png b/docs/images/select-provider.png new file mode 100644 index 0000000000..3151bfe5e7 Binary files /dev/null and b/docs/images/select-provider.png differ diff --git a/docs/images/sergio.png b/docs/images/sergio.png new file mode 100644 index 0000000000..d8d82846dc Binary files /dev/null and b/docs/images/sergio.png differ diff --git a/docs/images/services.png b/docs/images/services.png new file mode 100644 index 0000000000..e82a441576 Binary files /dev/null and b/docs/images/services.png differ diff --git a/docs/images/short-display.png b/docs/images/short-display.png new file mode 100644 index 0000000000..a36364acc0 Binary files /dev/null and b/docs/images/short-display.png differ diff --git a/docs/images/sign-up-button.png b/docs/images/sign-up-button.png new file mode 100644 index 0000000000..b7006e72e5 Binary files /dev/null and b/docs/images/sign-up-button.png differ diff --git a/docs/images/sign-up.png b/docs/images/sign-up.png new file mode 100644 index 0000000000..56e8901b24 Binary files /dev/null and b/docs/images/sign-up.png differ diff --git a/docs/images/sts-configuration.png b/docs/images/sts-configuration.png new file mode 100644 index 0000000000..cb78fbda74 Binary files /dev/null and b/docs/images/sts-configuration.png differ diff --git a/docs/images/test-connection-button.png b/docs/images/test-connection-button.png new file mode 100644 index 0000000000..261cb035c7 Binary files /dev/null and b/docs/images/test-connection-button.png differ diff --git a/docs/images/toni.png b/docs/images/toni.png new file mode 100644 index 0000000000..26b27d0a0f Binary files /dev/null and b/docs/images/toni.png differ diff --git a/docs/img/provider-decision-tree.png b/docs/img/provider-decision-tree.png new file mode 100644 index 0000000000..f76d375f94 Binary files /dev/null and b/docs/img/provider-decision-tree.png differ diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index c420b466c3..0000000000 --- a/docs/index.md +++ /dev/null @@ -1,27 +0,0 @@ -# What is Prowler? - -**Prowler** is the open source cloud security platform trusted by thousands to **automate security and compliance** in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size. - -The official supported providers right now are: - -| Provider | Support | Stage | Interface | -|----------|--------|-------|----------| -| **AWS** | Official | Stable | UI, API, CLI | -| **Azure** | Official | Stable | UI, API, CLI | -| **Google Cloud** | Official | Stable | UI, API, CLI | -| **Kubernetes** | Official | Stable | UI, API, CLI | -| **M365** | Official | Stable | UI, API, CLI | -| **Github** | Official | Stable | UI, API, CLI | -| **IaC** | Official | Beta | CLI | -| **MongoDB Atlas** | Official | Beta | CLI | -| **LLM** | Official | Beta | CLI | -| **NHN** | Unofficial | Beta | CLI | - -Prowler supports **auditing, incident response, continuous monitoring, hardening, forensic readiness, and remediation**. - -### Products - -- **Prowler CLI** (Command Line Interface) -- **Prowler App** (Web Application) -- [**Prowler Cloud**](https://cloud.prowler.com) – A managed service built on top of Prowler App. -- [**Prowler Hub**](https://hub.prowler.com) – A public library of versioned checks, cloud service artifacts, and compliance frameworks. diff --git a/docs/introduction.mdx b/docs/introduction.mdx new file mode 100644 index 0000000000..603bd1afbd --- /dev/null +++ b/docs/introduction.mdx @@ -0,0 +1,51 @@ +# What is Prowler? + +**Prowler** is the world’s most widely used open-source cloud security platform that **automates security and compliance** across any cloud environment. With hundreds of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler delivers AI-driven, customizable, and easy-to-use monitoring and integrations, making cloud security simple, scalable, and cost-effective for organizations of any size. + +![](/images/products/overview.png) + + + + Command Line Interface + + + Web Application + + + A managed service built on top of Prowler App. + + + A public library of versioned checks, cloud service artifacts, and compliance frameworks. + + + +## Supported Providers + +The supported providers right now are: + +| Provider | Support | Interface | +| -------------------------------------------------------------------------------- | ---------- | ------------ | +| [AWS](/user-guide/providers/aws/getting-started-aws) | Official | UI, API, CLI | +| [Azure](/user-guide/providers/azure/getting-started-azure) | Official | UI, API, CLI | +| [Google Cloud](/user-guide/providers/gcp/getting-started-gcp) | Official | UI, API, CLI | +| [Kubernetes](/user-guide/providers/kubernetes/getting-started-k8s) | Official | UI, API, CLI | +| [M365](/user-guide/providers/microsoft365/getting-started-m365) | Official | UI, API, CLI | +| [Github](/user-guide/providers/github/getting-started-github) | Official | UI, API, CLI | +| [Oracle Cloud](/user-guide/providers/oci/getting-started-oci) | Official | UI, API, CLI | +| [Infra as Code](/user-guide/providers/iac/getting-started-iac) | Official | UI, API, CLI | +| [MongoDB Atlas](/user-guide/providers/mongodbatlas/getting-started-mongodbatlas) | Official | CLI, API | +| [LLM](/user-guide/providers/llm/getting-started-llm) | Official | CLI | +| **NHN** | Unofficial | CLI | + +For more information about the checks and compliance of each provider visit [Prowler Hub](https://hub.prowler.com). + +## Where to go next? + + + + Detailed instructions on how to use Prowler. + + + Interested in contributing to Prowler? + + diff --git a/docs/security.md b/docs/security.mdx similarity index 99% rename from docs/security.md rename to docs/security.mdx index 9e7e4bd0ea..d8a3f63c73 100644 --- a/docs/security.md +++ b/docs/security.mdx @@ -1,4 +1,6 @@ -# Security +--- +title: 'Security' +--- ## Compliance and Trust We publish our live SOC 2 Type 2 Compliance data at [https://trust.prowler.com](https://trust.prowler.com) @@ -14,7 +16,7 @@ We use encryption everywhere possible. The data and communications used by **Pro Prowler Cloud is GDPR compliant in regards to personal data and the ["right to be forgotten"](https://gdpr.eu/right-to-be-forgotten/). When a user deletes their account their user information will be deleted from Prowler Cloud online and backup systems within 10 calendar days. -## Software Security +## Software Security We follow a **security-by-design approach** throughout our software development lifecycle. All changes go through automated checks at every stage, from local development to production deployment. diff --git a/docs/snippets/version-badge.mdx b/docs/snippets/version-badge.mdx new file mode 100644 index 0000000000..7541ff823a --- /dev/null +++ b/docs/snippets/version-badge.mdx @@ -0,0 +1,12 @@ +export const VersionBadge = ({ version }) => { + return ( + +

+ Added in:  + {version} +

+ + + + ); +}; diff --git a/docs/style.css b/docs/style.css new file mode 100644 index 0000000000..3e9bedbc80 --- /dev/null +++ b/docs/style.css @@ -0,0 +1,51 @@ +/* Version Badge Styling */ +.version-badge-container { + display: inline-block; + margin: 0 0 1rem 0; + padding: 0; +} + +.version-badge { + display: inline-flex; + align-items: center; + margin: 0; + padding: 0.375rem 0.75rem; + background: linear-gradient(135deg, #1a1a1a 0%, #000000 100%); + color: #ffffff; + border-radius: 1.25rem; + font-weight: 400; + font-size: 0.875rem; + line-height: 1.25rem; + border: 1px solid rgba(0, 0, 0, 0.15); + box-shadow: none; +} + +.version-badge-label { + font-weight: 400; + opacity: 1; +} + +.version-badge-version { + background: rgba(255, 255, 255, 0.12); + padding: 0.125rem 0.5rem; + border-radius: 0.875rem; + font-family: ui-monospace, SFMono-Regular, 'SF Mono', Menlo, Monaco, 'Cascadia Code', 'Roboto Mono', Consolas, 'Courier New', monospace; + font-weight: 600; + font-size: 0.875rem; + color: #ffffff; + border: none; +} + + +.dark .version-badge { + background: #55B685; + color: #000000; + border: 2px solid rgba(85, 182, 133, 0.3); + box-shadow: none; + } + + .dark .version-badge-version { + background: rgba(0, 0, 0, 0.1); + color: #000000; + border: none; +} diff --git a/docs/troubleshooting.md b/docs/troubleshooting.mdx similarity index 94% rename from docs/troubleshooting.md rename to docs/troubleshooting.mdx index e5650723db..80b6590669 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.mdx @@ -1,4 +1,6 @@ -# Troubleshooting +--- +title: 'Troubleshooting' +--- - **Running `prowler` I get `[File: utils.py:15] [Module: utils] CRITICAL: path/redacted: OSError[13]`**: @@ -11,7 +13,7 @@ This error is also related with a lack of system requirements. To improve performance, Prowler stores information in memory so it may need to be run in a system with more than 1GB of memory. -See section [Logging](./tutorials/logging.md) for further information or [contact us](./contact.md). +See section [Logging](/user-guide/cli/tutorials/logging) for further information or [contact us](/contact). ## Common Issues with Docker Compose Installation diff --git a/docs/tutorials/gcp/getting-started-gcp.md b/docs/tutorials/gcp/getting-started-gcp.md deleted file mode 100644 index 81d640cf59..0000000000 --- a/docs/tutorials/gcp/getting-started-gcp.md +++ /dev/null @@ -1,121 +0,0 @@ -# Getting Started With GCP on Prowler - -## Prowler App - -### Step 1: Get the GCP Project ID - -1. Go to the [GCP Console](https://console.cloud.google.com/) -2. Locate the Project ID on the welcome screen - -![Get the Project ID](./img/project-id-console.png) - -### Step 2: Access Prowler Cloud or Prowler App - -1. Navigate to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](../prowler-app.md) -2. Go to "Configuration" > "Cloud Providers" - - ![Cloud Providers Page](../img/cloud-providers-page.png) - -3. Click "Add Cloud Provider" - - ![Add a Cloud Provider](../img/add-cloud-provider.png) - -4. Select "Google Cloud Platform" - - ![Select GCP](./img/select-gcp.png) - -5. Add the Project ID and optionally provide a provider alias, then click "Next" - - ![Add Project ID](./img/add-project-id.png) - -### Step 3: Set Up GCP Authentication - -Choose the preferred authentication mode before proceeding: - -**User Credentials (Application Default Credentials)** - -* Quick scan as current user -* Uses Google Cloud CLI authentication -* Credentials may time out - -**Service Account Key File** - -* Authenticates as a service identity -* Stable and auditable -* Recommended for production - -For detailed instructions on how to set up authentication, see [Authentication](./authentication.md). - -6. Once credentials are configured, return to Prowler App and enter the required values: - - For "Service Account Key": - - - `Service Account Key JSON` - - For "Application Default Credentials": - - - `client_id` - - `client_secret` - - `refresh_token` - - ![Enter the Credentials](./img/enter-credentials-prowler-cloud.png) - -7. Click "Next", then "Launch Scan" - - ![Launch Scan GCP](./img/launch-scan.png) - ---- - -## Prowler CLI - -### Credentials Lookup Order - -Prowler follows the same credential search process as [Google authentication libraries](https://cloud.google.com/docs/authentication/application-default-credentials#search_order), checking credentials in this order: - -1. [`GOOGLE_APPLICATION_CREDENTIALS` environment variable](https://cloud.google.com/docs/authentication/application-default-credentials#GAC) -2. [`CLOUDSDK_AUTH_ACCESS_TOKEN` + optional `GOOGLE_CLOUD_PROJECT`](https://cloud.google.com/sdk/gcloud/reference/auth/print-access-token) -3. [User credentials set up by using the Google Cloud CLI](https://cloud.google.com/docs/authentication/application-default-credentials#personal) -4. [Attached service account (e.g., Cloud Run, GCE, Cloud Functions)](https://cloud.google.com/docs/authentication/application-default-credentials#attached-sa) - -???+ note - The credentials must belong to a user or service account with the necessary permissions. - For detailed instructions on how to set the permissions, see [Authentication > Required Permissions](./authentication.md#required-permissions). - -???+ note - Prowler will use the enabled Google Cloud APIs to get the information needed to perform the checks. - -### Configure GCP Credentials - -To authenticate with GCP, use one of the following methods: - -```console -gcloud auth application-default login -``` - -or set the credentials file path: - -```console -export GOOGLE_APPLICATION_CREDENTIALS="/path/to/credentials.json" -``` - -These credentials must belong to a user or service account with the necessary permissions to perform security checks. - -For more authentication details, see the [Authentication](./authentication.md) page. - -### Project Specification - -To scan specific projects, specify them with the following command: - -```console -prowler gcp --project-ids -``` - -### Service Account Impersonation - -For service account impersonation, use the `--impersonate-service-account` flag: - -```console -prowler gcp --impersonate-service-account -``` - -More details on authentication methods in the [Authentication](./authentication.md) page. diff --git a/docs/tutorials/gcp/organization.md b/docs/tutorials/gcp/organization.md deleted file mode 100644 index 96c7406711..0000000000 --- a/docs/tutorials/gcp/organization.md +++ /dev/null @@ -1,23 +0,0 @@ -# Scanning a Specific GCP Organization - -By default, Prowler scans all Google Cloud projects accessible to the authenticated user. - -To limit the scan to projects within a specific Google Cloud organization, use the `--organization-id` option with the GCP organization’s ID: - -```console -prowler gcp --organization-id organization-id -``` - -???+ warning - Ensure the credentials used have one of the following roles at the organization level: - Cloud Asset Viewer (`roles/cloudasset.viewer`), or Cloud Asset Owner (`roles/cloudasset.owner`). - -???+ note - With this option, Prowler retrieves all projects under the specified Google Cloud organization, including those organized within folders and nested subfolders. This ensures full visibility across the entire organization’s hierarchy. - -???+ note - To obtain the Google Cloud organization ID, use: - - ```console - gcloud organizations list - ``` diff --git a/docs/tutorials/kubernetes/in-cluster.md b/docs/tutorials/kubernetes/in-cluster.md deleted file mode 100644 index ac645c0010..0000000000 --- a/docs/tutorials/kubernetes/in-cluster.md +++ /dev/null @@ -1,38 +0,0 @@ -# In-Cluster Execution - -For in-cluster execution, use the supplied yaml files inside `/kubernetes`: - -* [prowler-sa.yaml](https://github.com/prowler-cloud/prowler/blob/master/kubernetes/prowler-sa.yaml) -* [job.yaml](https://github.com/prowler-cloud/prowler/blob/master/kubernetes/job.yaml) -* [prowler-role.yaml](https://github.com/prowler-cloud/prowler/blob/master/kubernetes/prowler-role.yaml) -* [prowler-rolebinding.yaml](https://github.com/prowler-cloud/prowler/blob/master/kubernetes/prowler-rolebinding.yaml) - -They can be used to run Prowler as a job within a new Prowler namespace: - -```console -kubectl apply -f kubernetes/prowler-sa.yaml -kubectl apply -f kubernetes/job.yaml -kubectl apply -f kubernetes/prowler-role.yaml -kubectl apply -f kubernetes/prowler-rolebinding.yaml -kubectl get pods --namespace prowler-ns --> prowler-XXXXX -kubectl logs prowler-XXXXX --namespace prowler-ns -``` - -???+ note - By default, `prowler` will scan all namespaces in your active Kubernetes context. Use the [`--namespace`](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/kubernetes/namespace/) flag to specify the namespace(s) to be scanned. - -???+ tip "Identifying the cluster in reports" - When running in in-cluster mode, the Kubernetes API does not expose the actual cluster name by default. - - To uniquely identify the cluster in logs and reports, you can: - - - Use the `--cluster-name` flag to manually set the cluster name: - ```bash - prowler -p kubernetes --cluster-name production-cluster - ``` - - Or set the `CLUSTER_NAME` environment variable: - ```yaml - env: - - name: CLUSTER_NAME - value: production-cluster - ``` diff --git a/docs/tutorials/kubernetes/outside-cluster.md b/docs/tutorials/kubernetes/outside-cluster.md deleted file mode 100644 index 5a8f15ac92..0000000000 --- a/docs/tutorials/kubernetes/outside-cluster.md +++ /dev/null @@ -1,16 +0,0 @@ -# Non In-Cluster Execution - -For execution outside the cluster environment, specify the location of the [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) file using the following argument: - -```console -prowler kubernetes --kubeconfig-file /path/to/kubeconfig -``` - -???+ note - If no `--kubeconfig-file` is provided, Prowler will use the default KubeConfig file location (`~/.kube/config`). - -???+ note - `prowler` will scan the active Kubernetes context by default. Use the [`--context`](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/kubernetes/context/) flag to specify the context to be scanned. - -???+ note - By default, `prowler` will scan all namespaces in your active Kubernetes context. Use the [`--namespace`](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/kubernetes/namespace/) flag to specify the namespace(s) to be scanned. diff --git a/docs/tutorials/microsoft365/getting-started-m365.md b/docs/tutorials/microsoft365/getting-started-m365.md deleted file mode 100644 index 8be16e755e..0000000000 --- a/docs/tutorials/microsoft365/getting-started-m365.md +++ /dev/null @@ -1,99 +0,0 @@ -# Getting Started With Microsoft 365 on Prowler - -???+ note "Government Cloud Support" - Government cloud accounts or tenants (Microsoft 365 Government) are currently unsupported, but we expect to add support for them in the near future. - -## Prerequisites - -Configure authentication for Microsoft 365 by following the [Microsoft 365 Authentication](authentication.md) guide. This includes: - -- Creating a Service Principal Application -- Granting required Microsoft Graph API permissions -- Setting up PowerShell module permissions (for full security coverage) -- Assigning appropriate roles to users (if using user authentication) - -## Prowler App - -### Step 1: Obtain Domain ID - -1. Go to the Entra ID portal, then search for "Domain" or go to Identity > Settings > Domain Names - - ![Search Domain Names](./img/search-domain-names.png) - - ![Custom Domain Names](./img/custom-domain-names.png) - -2. Select the domain to use as unique identifier for the Microsoft 365 account in Prowler App - -### Step 2: Access Prowler App - -1. Go to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](../prowler-app.md) -2. Navigate to "Configuration" > "Cloud Providers" - - ![Cloud Providers Page](../img/cloud-providers-page.png) - -3. Click on "Add Cloud Provider" - - ![Add a Cloud Provider](../img/add-cloud-provider.png) - -4. Select "Microsoft 365" - - ![Select Microsoft 365](./img/select-m365-prowler-cloud.png) - -5. Add the Domain ID and an optional alias, then click "Next" - - ![Add Domain ID](./img/add-domain-id.png) - -### Step 3: Add Credentials to Prowler App - -1. Go to App Registration overview and copy the Client ID and Tenant ID - - ![App Overview](./img/app-overview.png) - -2. Go to Prowler App and paste: - - - Client ID - - Tenant ID - - `AZURE_CLIENT_SECRET` from the Service Principal setup - - ![Prowler Cloud M365 Credentials](./img/m365-credentials.png) - -3. Click "Next" - - ![Next Detail](./img/click-next-m365.png) - -4. Click "Launch Scan" - - ![Launch Scan M365](./img/launch-scan.png) - ---- - -## Prowler CLI - -Use Prowler CLI to scan Microsoft 365 environments. - -### PowerShell Requirements - -PowerShell 7.4+ is required for comprehensive Microsoft 365 security coverage. Installation instructions are available in the [Authentication guide](authentication.md#supported-powershell-versions). - -### Authentication Options - -Select an authentication method from the [Microsoft 365 Authentication](authentication.md) guide: - -- **Service Principal Application** (recommended): `--sp-env-auth` -- **Interactive Browser Authentication**: `--browser-auth` - -### Basic Usage - -After configuring authentication, run a basic scan: - -```console -prowler m365 --sp-env-auth -``` - -For comprehensive scans including PowerShell checks: - -```console -prowler m365 --sp-env-auth --init-modules -``` - ---- diff --git a/docs/tutorials/prowler-app.md b/docs/tutorials/prowler-app.md deleted file mode 100644 index 0b5e2373a3..0000000000 --- a/docs/tutorials/prowler-app.md +++ /dev/null @@ -1,329 +0,0 @@ -# Prowler App - -**Prowler App** is a web application that simplifies running Prowler. This tutorial will guide you through setting up and using it. - -## Accessing Prowler App and API Documentation - -After [installing](../installation/prowler-app.md) **Prowler App**, access it at [http://localhost:3000](http://localhost:3000). To view the auto-generated **Prowler API** documentation, navigate to [http://localhost:8080/api/v1/docs](http://localhost:8080/api/v1/docs). This documentation provides details on available endpoints, parameters, and responses. - -???+ note - If you are a [Prowler Cloud](https://cloud.prowler.com/sign-in) user, you can access API docs at [https://api.prowler.com/api/v1/docs](https://api.prowler.com/api/v1/docs) - -## **Step 1: Sign Up** - -### **Sign Up with Email** - -To get started, sign up using your email and password: - -Sign Up Button -Sign Up - -### **Sign Up with Social Login** - -If Social Login is enabled, you can sign up using your preferred provider (e.g., Google, GitHub). - -???+ note "How Social Login Works" - If your email is already registered, you will be logged in, and your social account will be linked. - If your email is not registered, a new account will be created using your social account email. - -???+ note "Enable Social Login" - See [how to configure Social Login for Prowler](prowler-app-social-login.md) to enable this feature in your own deployments. - -## **Step 2: Log In** - -Once registered, log in with your email and password to access Prowler App. - -Log In - -Upon logging in, the Overview page will display. At this stage, no data is present: add a provider to begin scanning your cloud environment. - -## **Step 3: Add a Provider** - -To perform security scans, link a cloud provider account. Prowler supports the following providers and more: - -- **AWS** - -- **Azure** - -- **Google Cloud Platform (GCP)** - -- **Kubernetes** - -- **M365** - -- **GitHub** - -Steps to add a provider: - -1. Navigate to `Settings > Cloud Providers`. -2. Click `Add Account` to set up a new provider and provide your credentials. - -Add Provider - -## **Step 4: Configure the Provider** - -Select the cloud provider you want to scan. - -Select a Provider - -Once chosen, enter the Provider UID for authentication: - -- **AWS**: Enter your AWS Account ID. -- **GCP**: Enter your GCP Project ID. -- **Azure**: Enter your Azure Subscription ID. -- **Kubernetes**: Enter your Kubernetes Cluster context of your kubeconfig file. -- **M365**: Enter your M365 Domain ID. - -Optionally, provide a **Provider Alias** for easier identification. Follow the instructions provided to add your credentials: - -### **Step 4.1: AWS Credentials** - -For AWS, enter your `AWS Account ID` and choose one of the following methods to connect: - -#### **Step 4.1.1: IAM Access Keys** - -1. Select `Connect via Credentials`. - - AWS Credentials - -2. Enter your `Access Key ID`, `Secret Access Key` and optionally a `Session Token`: - - AWS Credentials - -#### **Step 4.1.2: IAM Role** - -1. Select `Connect assuming IAM Role`. - - AWS Role - -2. Enter the `Role ARN` and any optional field like the AWS Access Keys to assume the role, the `External ID`, the `Role Session Name` or the `Session Duration`: - - AWS Role - -???+ note - Check if your AWS Security Token Service (STS) has the EU (Ireland) endpoint active. If not, we will not be able to connect to your AWS account. - - If that is the case your STS configuration may look like this: - - AWS Role - - To solve this issue, please activate the EU (Ireland) STS endpoint. - -### **Step 4.2: Azure Credentials**: - -For Azure, Prowler App uses a service principal application to authenticate. For more information about the process of creating and adding permissions to a service principal refer to this [section](../tutorials/azure/authentication.md). When you finish creating and adding the [Entra](./azure/create-prowler-service-principal.md#assigning-proper-permissions) and [Subscription](./azure/subscriptions.md) scope permissions to the service principal, enter the `Tenant ID`, `Client ID` and `Client Secret` of the service principal application. - -Azure Credentials - ---- -### **Step 4.3: GCP Credentials** - -For Google Cloud, first enter your `GCP Project ID` and then select the authentication method you want to use: - -- **Service Account Authentication** (**Recommended**) -- **Application Default Credentials** - -**Service Account Authentication** is the recommended authentication method for automated systems and machine-to-machine interactions, like Prowler. For detailed information about this, refer to the [Google Cloud documentation](https://cloud.google.com/iam/docs/service-account-overview). - -GCP Authentication Methods - -#### **Step 4.3.1: Service Account Authentication** - -First of all, in the same project that you selected in the previous step, you need to create a service account and then generate a key in JSON format for it. For more information about this, you can follow the next Google Cloud documentation tutorials: - -- [Create a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) -- [Generate a key for a service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) - -GCP Service Account Credentials - -#### **Step 4.3.2: Application Default Credentials** - -1. Run the following command in your terminal to authenticate with GCP: - - ```bash - gcloud auth application-default login - ``` - -2. Once authenticated, get the `Client ID`, `Client Secret` and `Refresh Token` from `~/.config/gcloud/application_default_credentials`. - -3. Paste the `Client ID`, `Client Secret` and `Refresh Token` into Prowler App. - -GCP Credentials - -### **Step 4.4: Kubernetes Credentials**: - -For Kubernetes, Prowler App uses a `kubeconfig` file to authenticate, paste the contents of your `kubeconfig` file into the `Kubeconfig content` field. - -By default, the `kubeconfig` file is located at `~/.kube/config`. - -Kubernetes Credentials - -If you are adding an **EKS**, **GKE**, **AKS** or external cluster, follow these additional steps to ensure proper authentication: - -**Make sure your cluster allow traffic from the Prowler Cloud IP address `52.48.254.174/32`** - -1. Apply the necessary Kubernetes resources to your EKS, GKE, AKS or external cluster (you can find the files in the [`kubernetes` directory of the Prowler repository](https://github.com/prowler-cloud/prowler/tree/master/kubernetes)): - - ```console - kubectl apply -f kubernetes/prowler-sa.yaml - kubectl apply -f kubernetes/prowler-role.yaml - kubectl apply -f kubernetes/prowler-rolebinding.yaml - ``` - -2. Generate a long-lived token for authentication: - - ```console - kubectl create token prowler-sa -n prowler-ns --duration=0 - ``` - - - **Security Note:** The `--duration=0` option generates a non-expiring token, which may pose a security risk if not managed properly. Users should decide on an appropriate expiration time based on their security policies. If a limited-time token is preferred, set `--duration=