diff --git a/.env b/.env index 7f739e01e5..00d48ed6a2 100644 --- a/.env +++ b/.env @@ -10,13 +10,16 @@ NEXT_PUBLIC_API_BASE_URL=${API_BASE_URL} NEXT_PUBLIC_API_DOCS_URL=http://prowler-api:8080/api/v1/docs AUTH_TRUST_HOST=true UI_PORT=3000 -# Temp URL for feeds need to use actual -RSS_FEED_URL=https://prowler.com/blog/rss # openssl rand -base64 32 AUTH_SECRET="N/c6mnaS5+SWq81+819OrzQZlmx1Vxtp/orjttJSmw8=" # Google Tag Manager ID NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID="" +#### Code Review Configuration #### +# Enable Claude Code standards validation on pre-push hook +# Set to 'true' to validate changes against AGENTS.md standards via Claude Code +# Set to 'false' to skip validation +CODE_REVIEW_ENABLED=true #### Prowler API Configuration #### PROWLER_API_VERSION="stable" @@ -35,6 +38,8 @@ POSTGRES_DB=prowler_db # POSTGRES_REPLICA_USER=prowler # POSTGRES_REPLICA_PASSWORD=postgres # POSTGRES_REPLICA_DB=prowler_db +# POSTGRES_REPLICA_MAX_ATTEMPTS=3 +# POSTGRES_REPLICA_RETRY_BASE_DELAY=0.5 # Celery-Prowler task settings TASK_RETRY_DELAY_SECONDS=0.1 @@ -103,6 +108,8 @@ DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute # Sentry settings SENTRY_ENVIRONMENT=local SENTRY_RELEASE=local +NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT} + #### Prowler release version #### NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.12.2 @@ -124,3 +131,12 @@ LANGSMITH_TRACING=false LANGSMITH_ENDPOINT="https://api.smith.langchain.com" LANGSMITH_API_KEY="" LANGCHAIN_PROJECT="" + +# RSS Feed Configuration +# Multiple feed sources can be configured as a JSON array (must be valid JSON, no trailing commas) +# Each source requires: id, name, type (github_releases|blog|custom), url, and enabled flag +# IMPORTANT: Must be a single line with valid JSON (no newlines, no trailing commas) +# Example with one source: +RSS_FEED_SOURCES='[{"id":"prowler-releases","name":"Prowler Releases","type":"github_releases","url":"https://github.com/prowler-cloud/prowler/releases.atom","enabled":true}]' +# Example with multiple sources (no trailing comma after last item): +# RSS_FEED_SOURCES='[{"id":"prowler-releases","name":"Prowler Releases","type":"github_releases","url":"https://github.com/prowler-cloud/prowler/releases.atom","enabled":true},{"id":"prowler-blog","name":"Prowler Blog","type":"blog","url":"https://prowler.com/blog/rss","enabled":false}]' diff --git a/.github/actions/setup-python-poetry/action.yml b/.github/actions/setup-python-poetry/action.yml index 3b774262f9..790f3ef0e6 100644 --- a/.github/actions/setup-python-poetry/action.yml +++ b/.github/actions/setup-python-poetry/action.yml @@ -22,8 +22,8 @@ inputs: runs: using: 'composite' steps: - - name: Replace @master with current branch in pyproject.toml - if: github.event_name == 'pull_request' && github.base_ref == 'master' + - name: Replace @master with current branch in pyproject.toml (prowler repo only) + if: github.event_name == 'pull_request' && github.base_ref == 'master' && github.repository == 'prowler-cloud/prowler' shell: bash working-directory: ${{ inputs.working-directory }} run: | @@ -37,8 +37,8 @@ runs: python -m pip install --upgrade pip pipx install poetry==${{ inputs.poetry-version }} - - name: Update SDK resolved_reference to latest commit - if: github.event_name == 'push' && github.ref == 'refs/heads/master' + - name: Update poetry.lock with latest Prowler commit + if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler' shell: bash working-directory: ${{ inputs.working-directory }} run: | @@ -50,7 +50,21 @@ runs: echo "Updated resolved_reference:" grep -A2 -B2 "resolved_reference" poetry.lock - - name: Update poetry.lock + - name: Update SDK resolved_reference to latest commit (prowler repo on push) + if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha') + echo "Latest commit hash: $LATEST_COMMIT" + sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / { + s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/ + }' poetry.lock + echo "Updated resolved_reference:" + grep -A2 -B2 "resolved_reference" poetry.lock + + - name: Update poetry.lock (prowler repo only) + if: github.repository == 'prowler-cloud/prowler' shell: bash working-directory: ${{ inputs.working-directory }} run: poetry lock @@ -69,3 +83,11 @@ runs: run: | poetry install --no-root poetry run pip list + + - name: Update Prowler Cloud API Client + if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + poetry remove prowler-cloud-api-client + poetry add ./prowler-cloud-api-client diff --git a/.github/actions/slack-notification/README.md b/.github/actions/slack-notification/README.md new file mode 100644 index 0000000000..ce8843df2a --- /dev/null +++ b/.github/actions/slack-notification/README.md @@ -0,0 +1,198 @@ +# Slack Notification Action + +A generic and flexible GitHub composite action for sending Slack notifications using JSON template files. Supports both standalone messages and message updates, with automatic status detection. + +## Features + +- **Template-based**: All messages use JSON template files for consistency +- **Automatic status detection**: Pass `step-outcome` to auto-calculate success/failure +- **Message updates**: Supports updating existing messages (using `chat.update`) +- **Simple API**: Clean and minimal interface +- **Reusable**: Use across all workflows and scenarios +- **Maintainable**: Centralized message templates + +## Use Cases + +1. **Container releases**: Track push start and completion with automatic status +2. **Deployments**: Track deployment progress with rich Block Kit formatting +3. **Custom notifications**: Any scenario where you need to notify Slack + +## Inputs + +| Input | Description | Required | Default | +|-------|-------------|----------|---------| +| `slack-bot-token` | Slack bot token for authentication | Yes | - | +| `payload-file-path` | Path to JSON file with the Slack message payload | Yes | - | +| `update-ts` | Message timestamp to update (leave empty for new messages) | No | `''` | +| `step-outcome` | Step outcome for automatic status detection (sets STATUS_EMOJI and STATUS_TEXT env vars) | No | `''` | + +## Outputs + +| Output | Description | +|--------|-------------| +| `ts` | Timestamp of the Slack message (use for updates) | + +## Usage Examples + +### Example 1: Container Release with Automatic Status Detection + +Using JSON template files with automatic status detection: + +```yaml +# Send start notification +- name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + +# Do the work +- name: Build and push container + if: github.event_name == 'release' + id: container-push + uses: docker/build-push-action@... + with: + push: true + tags: ... + +# Send completion notification with automatic status detection +- name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} +``` + +**Benefits:** +- No status calculation needed in workflow +- Reusable template files +- Clean and concise +- Automatic `STATUS_EMOJI` and `STATUS_TEXT` env vars set by action +- Consistent message format across all workflows + +### Example 2: Deployment with Message Update Pattern + +```yaml +# Send initial deployment message +- name: Notify deployment started + id: slack-start + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + ENVIRONMENT: PRODUCTION + COMMIT_HASH: ${{ github.sha }} + VERSION_DEPLOYED: latest + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/deployment-started.json" + +# Run deployment +- name: Deploy + id: deploy + run: terraform apply -auto-approve + +# Determine additional status variables +- name: Determine deployment status + if: always() + id: deploy-status + run: | + if [[ "${{ steps.deploy.outcome }}" == "success" ]]; then + echo "STATUS_COLOR=28a745" >> $GITHUB_ENV + echo "STATUS=Completed" >> $GITHUB_ENV + else + echo "STATUS_COLOR=fc3434" >> $GITHUB_ENV + echo "STATUS=Failed" >> $GITHUB_ENV + fi + +# Update the same message with final status +- name: Update deployment notification + if: always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + MESSAGE_TS: ${{ steps.slack-start.outputs.ts }} + COMPONENT: API + ENVIRONMENT: PRODUCTION + COMMIT_HASH: ${{ github.sha }} + VERSION_DEPLOYED: latest + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + STATUS: ${{ env.STATUS }} + STATUS_COLOR: ${{ env.STATUS_COLOR }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + update-ts: ${{ steps.slack-start.outputs.ts }} + payload-file-path: "./.github/scripts/slack-messages/deployment-completed.json" + step-outcome: ${{ steps.deploy.outcome }} +``` + +## Automatic Status Detection + +When you provide `step-outcome` input, the action automatically sets these environment variables: + +| Outcome | STATUS_EMOJI | STATUS_TEXT | +|---------|--------------|-------------| +| success | `[✓]` | `completed successfully!` | +| failure | `[✗]` | `failed` | + +These variables are then available in your payload template files. + +## Template File Format + +All template files must be valid JSON and support environment variable substitution. Example: + +```json +{ + "channel": "$SLACK_CHANNEL_ID", + "text": "$STATUS_EMOJI $COMPONENT container release $RELEASE_TAG push $STATUS_TEXT <$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID|View run>" +} +``` + +See available templates in [`.github/scripts/slack-messages/`](../../scripts/slack-messages/). + +## Requirements + +- Slack Bot Token with scopes: `chat:write`, `chat:write.public` +- Slack Channel ID where messages will be posted +- JSON template files for your messages + +## Benefits + +- **Consistency**: All notifications use standardized templates +- **Automatic status handling**: No need to calculate success/failure in workflows +- **Clean workflows**: Minimal boilerplate code +- **Reusable templates**: One template for all components +- **Easy to maintain**: Change template once, applies everywhere +- **Version controlled**: All message formats in git + +## Related Resources + +- [Slack Block Kit Builder](https://app.slack.com/block-kit-builder) +- [Slack API Method Documentation](https://docs.slack.dev/tools/slack-github-action/sending-techniques/sending-data-slack-api-method/) +- [Message templates documentation](../../scripts/slack-messages/README.md) diff --git a/.github/actions/slack-notification/action.yml b/.github/actions/slack-notification/action.yml new file mode 100644 index 0000000000..1427fed3e0 --- /dev/null +++ b/.github/actions/slack-notification/action.yml @@ -0,0 +1,74 @@ +name: 'Slack Notification' +description: 'Generic action to send Slack notifications with optional message updates and automatic status detection' +inputs: + slack-bot-token: + description: 'Slack bot token for authentication' + required: true + payload-file-path: + description: 'Path to JSON file with the Slack message payload' + required: true + update-ts: + description: 'Message timestamp to update (only for updates, leave empty for new messages)' + required: false + default: '' + step-outcome: + description: 'Outcome of a step to determine status (success/failure) - automatically sets STATUS_TEXT and STATUS_COLOR env vars' + required: false + default: '' +outputs: + ts: + description: 'Timestamp of the Slack message' + value: ${{ steps.slack-notification.outputs.ts }} +runs: + using: 'composite' + steps: + - name: Determine status + id: status + shell: bash + run: | + if [[ "${{ inputs.step-outcome }}" == "success" ]]; then + echo "STATUS_TEXT=Completed" >> $GITHUB_ENV + echo "STATUS_COLOR=#6aa84f" >> $GITHUB_ENV + elif [[ "${{ inputs.step-outcome }}" == "failure" ]]; then + echo "STATUS_TEXT=Failed" >> $GITHUB_ENV + echo "STATUS_COLOR=#fc3434" >> $GITHUB_ENV + else + # No outcome provided - pending/in progress state + echo "STATUS_COLOR=#dbab09" >> $GITHUB_ENV + fi + + - name: Send Slack notification (new message) + if: inputs.update-ts == '' + id: slack-notification-post + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_PAYLOAD_FILE_PATH: ${{ inputs.payload-file-path }} + with: + method: chat.postMessage + token: ${{ inputs.slack-bot-token }} + payload-file-path: ${{ inputs.payload-file-path }} + payload-templated: true + errors: true + + - name: Update Slack notification + if: inputs.update-ts != '' + id: slack-notification-update + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_PAYLOAD_FILE_PATH: ${{ inputs.payload-file-path }} + with: + method: chat.update + token: ${{ inputs.slack-bot-token }} + payload-file-path: ${{ inputs.payload-file-path }} + payload-templated: true + errors: true + + - name: Set output + id: slack-notification + shell: bash + run: | + if [[ "${{ inputs.update-ts }}" == "" ]]; then + echo "ts=${{ steps.slack-notification-post.outputs.ts }}" >> $GITHUB_OUTPUT + else + echo "ts=${{ inputs.update-ts }}" >> $GITHUB_OUTPUT + fi diff --git a/.github/actions/trivy-scan/action.yml b/.github/actions/trivy-scan/action.yml index 91c4c4d332..e3c2404748 100644 --- a/.github/actions/trivy-scan/action.yml +++ b/.github/actions/trivy-scan/action.yml @@ -45,22 +45,13 @@ outputs: runs: using: 'composite' steps: - - name: Run Trivy vulnerability scan (SARIF) - if: inputs.upload-sarif == 'true' - uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 + - name: Cache Trivy vulnerability database + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - image-ref: ${{ inputs.image-name }}:${{ inputs.image-tag }} - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH' - exit-code: '0' - - - name: Upload Trivy results to GitHub Security tab - if: inputs.upload-sarif == 'true' - uses: github/codeql-action/upload-sarif@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 - with: - sarif_file: 'trivy-results.sarif' - category: 'trivy-container' + path: ~/.cache/trivy + key: trivy-db-${{ runner.os }}-${{ github.run_id }} + restore-keys: | + trivy-db-${{ runner.os }}- - name: Run Trivy vulnerability scan (JSON) uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 @@ -70,6 +61,27 @@ runs: output: 'trivy-report.json' severity: ${{ inputs.severity }} exit-code: '0' + scanners: 'vuln' + timeout: '5m' + + - name: Run Trivy vulnerability scan (SARIF) + if: inputs.upload-sarif == 'true' && github.event_name == 'push' + uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 + with: + image-ref: ${{ inputs.image-name }}:${{ inputs.image-tag }} + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + exit-code: '0' + scanners: 'vuln' + timeout: '5m' + + - name: Upload Trivy results to GitHub Security tab + if: inputs.upload-sarif == 'true' && github.event_name == 'push' + uses: github/codeql-action/upload-sarif@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + with: + sarif_file: 'trivy-results.sarif' + category: 'trivy-container' - name: Upload Trivy report artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -109,20 +121,20 @@ runs: with: script: | const comment = require('./.github/scripts/trivy-pr-comment.js'); - + // Unique identifier to find our comment const marker = ''; const body = marker + '\n' + comment; - + // Find existing comment const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, }); - + const existingComment = comments.find(c => c.body?.includes(marker)); - + if (existingComment) { // Update existing comment await github.rest.issues.updateComment({ diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 94122d8349..497c4d6214 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -22,6 +22,13 @@ Please add a detailed description of how to review this PR. - [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md) - [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable. +#### UI +- [ ] All issue/task requirements work as expected on the UI +- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px) +- [ ] Screenshots/Video of the functionality flow (if applicable) - Table (640px > X < 1024px) +- [ ] Screenshots/Video of the functionality flow (if applicable) - Desktop (X > 1024px) +- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable. + #### API - [ ] Verify if API specs need to be regenerated. - [ ] Check if version updates are required (e.g., specs, Poetry, etc.). diff --git a/.github/scripts/slack-messages/README.md b/.github/scripts/slack-messages/README.md new file mode 100644 index 0000000000..e7fcf00fdd --- /dev/null +++ b/.github/scripts/slack-messages/README.md @@ -0,0 +1,462 @@ +# Slack Message Templates + +This directory contains reusable message templates for Slack notifications sent from GitHub Actions workflows. + +## Usage + +These JSON templates are used with the `slackapi/slack-github-action` using the Slack API method (`chat.postMessage` and `chat.update`). All templates support rich Block Kit formatting and message updates. + +### Available Templates + +**Container Releases** +- `container-release-started.json`: Simple one-line notification when container push starts +- `container-release-completed.json`: Simple one-line notification when container release completes + +**Deployments** +- `deployment-started.json`: Deployment start notification with Block Kit formatting +- `deployment-completed.json`: Deployment completion notification (updates the start message) + +All templates use the Slack API method and require a Slack Bot Token. + +## Setup Requirements + +1. Create a Slack App (or use existing) +2. Add Bot Token Scopes: `chat:write`, `chat:write.public` +3. Install the app to your workspace +4. Get the Bot Token from OAuth & Permissions page +5. Add secrets: + - `SLACK_BOT_TOKEN`: Your bot token + - `SLACK_CHANNEL_ID`: The channel ID where messages will be posted + +Reference: [Sending data using a Slack API method](https://docs.slack.dev/tools/slack-github-action/sending-techniques/sending-data-slack-api-method/) + +## Environment Variables + +### Required Secrets (GitHub Secrets) +- `SLACK_BOT_TOKEN`: Passed as `token` parameter to the action (not as env variable) +- `SLACK_CHANNEL_ID`: Used in payload as env variable + +### Container Release Variables (configured as env) +- `COMPONENT`: Component name (e.g., "API", "SDK", "UI", "MCP") +- `RELEASE_TAG` / `PROWLER_VERSION`: The release tag or version being deployed +- `GITHUB_SERVER_URL`: Provided by GitHub context +- `GITHUB_REPOSITORY`: Provided by GitHub context +- `GITHUB_RUN_ID`: Provided by GitHub context +- `STATUS_EMOJI`: Status symbol (calculated: `[✓]` for success, `[✗]` for failure) +- `STATUS_TEXT`: Status text (calculated: "completed successfully!" or "failed") + +### Deployment Variables (configured as env) +- `COMPONENT`: Component name (e.g., "API", "SDK", "UI", "MCP") +- `ENVIRONMENT`: Environment name (e.g., "DEVELOPMENT", "PRODUCTION") +- `COMMIT_HASH`: Commit hash being deployed +- `VERSION_DEPLOYED`: Version being deployed +- `GITHUB_ACTOR`: User who triggered the workflow +- `GITHUB_WORKFLOW`: Workflow name +- `GITHUB_SERVER_URL`: Provided by GitHub context +- `GITHUB_REPOSITORY`: Provided by GitHub context +- `GITHUB_RUN_ID`: Provided by GitHub context + +All other variables (MESSAGE_TS, STATUS, STATUS_COLOR, STATUS_EMOJI, etc.) are calculated internally within the workflow and should NOT be configured as environment variables. + +## Example Workflow Usage + +### Using the Generic Slack Notification Action (Recommended) + +**Recommended approach**: Use the generic reusable action `.github/actions/slack-notification` which provides maximum flexibility: + +#### Example 1: Container Release (Start + Completion) + +```yaml +# Send start notification +- name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": "API container release ${{ env.RELEASE_TAG }} push started... <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + } + +# Build and push container +- name: Build and push container + if: github.event_name == 'release' + id: container-push + uses: docker/build-push-action@... + with: + push: true + tags: ... + +# Calculate status +- name: Determine push status + if: github.event_name == 'release' && always() + id: push-status + run: | + if [[ "${{ steps.container-push.outcome }}" == "success" ]]; then + echo "emoji=[✓]" >> $GITHUB_OUTPUT + echo "text=completed successfully!" >> $GITHUB_OUTPUT + else + echo "emoji=[✗]" >> $GITHUB_OUTPUT + echo "text=failed" >> $GITHUB_OUTPUT + fi + +# Send completion notification +- name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": "${{ steps.push-status.outputs.emoji }} API container release ${{ env.RELEASE_TAG }} push ${{ steps.push-status.outputs.text }} <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + } +``` + +#### Example 2: Simple One-Time Message + +```yaml +- name: Send notification + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": "Deployment completed successfully!" + } +``` + +#### Example 3: Deployment with Message Update Pattern + +```yaml +# Send initial deployment message +- name: Notify deployment started + id: slack-start + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "text": "API deployment to PRODUCTION started", + "attachments": [ + { + "color": "dbab09", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "API | Deployment to PRODUCTION" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Status:*\nIn Progress" + } + ] + } + ] + } + ] + } + +# Run deployment +- name: Deploy + id: deploy + run: terraform apply -auto-approve + +# Calculate status +- name: Determine status + if: always() + id: status + run: | + if [[ "${{ steps.deploy.outcome }}" == "success" ]]; then + echo "color=28a745" >> $GITHUB_OUTPUT + echo "emoji=[✓]" >> $GITHUB_OUTPUT + echo "status=Completed" >> $GITHUB_OUTPUT + else + echo "color=fc3434" >> $GITHUB_OUTPUT + echo "emoji=[✗]" >> $GITHUB_OUTPUT + echo "status=Failed" >> $GITHUB_OUTPUT + fi + +# Update the same message with final status +- name: Update deployment notification + if: always() + uses: ./.github/actions/slack-notification + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + update-ts: ${{ steps.slack-start.outputs.ts }} + payload: | + { + "channel": "${{ secrets.SLACK_CHANNEL_ID }}", + "ts": "${{ steps.slack-start.outputs.ts }}", + "text": "${{ steps.status.outputs.emoji }} API deployment to PRODUCTION ${{ steps.status.outputs.status }}", + "attachments": [ + { + "color": "${{ steps.status.outputs.color }}", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "API | Deployment to PRODUCTION" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Status:*\n${{ steps.status.outputs.emoji }} ${{ steps.status.outputs.status }}" + } + ] + } + ] + } + ] + } +``` + +**Benefits of using the generic action:** +- Maximum flexibility: Build any payload you need directly in the workflow +- No template files needed: Everything inline +- Supports all scenarios: one-time messages, start/update patterns, rich Block Kit +- Easy to customize per use case +- Generic: Works for containers, deployments, or any notification type + +For more details, see [Slack Notification Action](../../actions/slack-notification/README.md). + +### Using Message Templates (Alternative Approach) + +Simple one-line notifications for container releases: + +```yaml +# Step 1: Notify when push starts +- name: Notify container push started + if: github.event_name == 'release' + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + +# Step 2: Build and push container +- name: Build and push container + id: container-push + uses: docker/build-push-action@... + with: + push: true + tags: ... + +# Step 3: Determine push status +- name: Determine push status + if: github.event_name == 'release' && always() + id: push-status + run: | + if [[ "${{ steps.container-push.outcome }}" == "success" ]]; then + echo "status-emoji=[✓]" >> $GITHUB_OUTPUT + echo "status-text=completed successfully!" >> $GITHUB_OUTPUT + else + echo "status-emoji=[✗]" >> $GITHUB_OUTPUT + echo "status-text=failed" >> $GITHUB_OUTPUT + fi + +# Step 4: Notify when push completes (success or failure) +- name: Notify container push completed + if: github.event_name == 'release' && always() + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + STATUS_EMOJI: ${{ steps.push-status.outputs.status-emoji }} + STATUS_TEXT: ${{ steps.push-status.outputs.status-text }} + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" +``` + +### Deployment with Update Pattern + +For deployments that start with one message and update it with the final status: + +```yaml +# Step 1: Send deployment start notification +- name: Notify Deployment Start + id: slack-notification-start + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + COMPONENT: API + ENVIRONMENT: PRODUCTION + COMMIT_HASH: ${{ github.sha }} + VERSION_DEPLOYED: latest + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/deployment-started.json" + +# Step 2: Run your deployment steps +- name: Terraform Plan + id: terraform-plan + run: terraform plan + +- name: Terraform Apply + id: terraform-apply + run: terraform apply -auto-approve + +# Step 3: Determine status (calculated internally, not configured) +- name: Determine Status + if: always() + id: determine-status + run: | + if [[ "${{ steps.terraform-apply.outcome }}" == "success" ]]; then + echo "status=Completed" >> $GITHUB_OUTPUT + echo "status-color=28a745" >> $GITHUB_OUTPUT + echo "status-emoji=[✓]" >> $GITHUB_OUTPUT + echo "plan-emoji=[✓]" >> $GITHUB_OUTPUT + echo "apply-emoji=[✓]" >> $GITHUB_OUTPUT + elif [[ "${{ steps.terraform-plan.outcome }}" == "failure" || "${{ steps.terraform-apply.outcome }}" == "failure" ]]; then + echo "status=Failed" >> $GITHUB_OUTPUT + echo "status-color=fc3434" >> $GITHUB_OUTPUT + echo "status-emoji=[✗]" >> $GITHUB_OUTPUT + if [[ "${{ steps.terraform-plan.outcome }}" == "failure" ]]; then + echo "plan-emoji=[✗]" >> $GITHUB_OUTPUT + else + echo "plan-emoji=[✓]" >> $GITHUB_OUTPUT + fi + if [[ "${{ steps.terraform-apply.outcome }}" == "failure" ]]; then + echo "apply-emoji=[✗]" >> $GITHUB_OUTPUT + else + echo "apply-emoji=[✓]" >> $GITHUB_OUTPUT + fi + else + echo "status=Failed" >> $GITHUB_OUTPUT + echo "status-color=fc3434" >> $GITHUB_OUTPUT + echo "status-emoji=[✗]" >> $GITHUB_OUTPUT + echo "plan-emoji=[?]" >> $GITHUB_OUTPUT + echo "apply-emoji=[?]" >> $GITHUB_OUTPUT + fi + +# Step 4: Update the same Slack message (using calculated values) +- name: Notify Deployment Result + if: always() + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + MESSAGE_TS: ${{ steps.slack-notification-start.outputs.ts }} + COMPONENT: API + ENVIRONMENT: PRODUCTION + COMMIT_HASH: ${{ github.sha }} + VERSION_DEPLOYED: latest + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + STATUS: ${{ steps.determine-status.outputs.status }} + STATUS_COLOR: ${{ steps.determine-status.outputs.status-color }} + STATUS_EMOJI: ${{ steps.determine-status.outputs.status-emoji }} + PLAN_EMOJI: ${{ steps.determine-status.outputs.plan-emoji }} + APPLY_EMOJI: ${{ steps.determine-status.outputs.apply-emoji }} + TERRAFORM_PLAN_OUTCOME: ${{ steps.terraform-plan.outcome }} + TERRAFORM_APPLY_OUTCOME: ${{ steps.terraform-apply.outcome }} + with: + method: chat.update + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/deployment-completed.json" +``` + +**Note**: Variables like `STATUS`, `STATUS_COLOR`, `STATUS_EMOJI`, `PLAN_EMOJI`, `APPLY_EMOJI` are calculated by the `determine-status` step based on the outcomes of previous steps. They should NOT be manually configured. + +## Key Features + +### Benefits of Using Slack API Method + +- **Rich Block Kit Formatting**: Full support for Slack's Block Kit including headers, sections, fields, colors, and attachments +- **Message Updates**: Update the same message instead of posting multiple messages (using `chat.update` with `ts`) +- **Consistent Experience**: Same look and feel as Prowler Cloud notifications +- **Flexible**: Easy to customize message appearance by editing JSON templates + +### Differences from Webhook Method + +| Feature | webhook-trigger | Slack API (chat.postMessage) | +|---------|-----------------|------------------------------| +| Setup | Workflow Builder webhook | Slack Bot Token + Channel ID | +| Formatting | Plain text/simple | Full Block Kit support | +| Message Update | No | Yes (with chat.update) | +| Authentication | Webhook URL | Bot Token | +| Scopes Required | None | chat:write, chat:write.public | + +## Message Appearance + +### Container Release (Simple One-Line) + +**Start message:** +``` +API container release 4.5.0 push started... View run +``` + +**Completion message (success):** +``` +[✓] API container release 4.5.0 push completed successfully! View run +``` + +**Completion message (failure):** +``` +[✗] API container release 4.5.0 push failed View run +``` + +All messages are simple one-liners with a clickable "View run" link. The completion message adapts to show success `[✓]` or failure `[✗]` based on the outcome of the container push. + +### Deployment Start +- Header: Component and environment +- Yellow bar (color: `dbab09`) +- Status: In Progress +- Details: Commit, version, actor, workflow +- Link: Direct link to deployment run + +### Deployment Completion +- Header: Component and environment +- Green bar for success (color: `28a745`) / Red bar for failure (color: `fc3434`) +- Status: [✓] Completed or [✗] Failed +- Details: All deployment info plus terraform outcomes +- Link: Direct link to deployment run + +## Adding New Templates + +1. Create a new JSON file with Block Kit structure +2. Use environment variable placeholders (e.g., `$VAR_NAME`) +3. Include `channel` and `text` fields (required) +4. Add `blocks` or `attachments` for rich formatting +5. For update templates, include `ts` field as `$MESSAGE_TS` +6. Document the template in this README +7. Reference it in your workflow using `payload-file-path` + +## Reference + +- [Slack Block Kit Builder](https://app.slack.com/block-kit-builder) +- [Slack API Method Documentation](https://docs.slack.dev/tools/slack-github-action/sending-techniques/sending-data-slack-api-method/) diff --git a/.github/scripts/slack-messages/container-release-completed.json b/.github/scripts/slack-messages/container-release-completed.json new file mode 100644 index 0000000000..c1fc147191 --- /dev/null +++ b/.github/scripts/slack-messages/container-release-completed.json @@ -0,0 +1,17 @@ +{ + "channel": "${{ env.SLACK_CHANNEL_ID }}", + "attachments": [ + { + "color": "${{ env.STATUS_COLOR }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*Status:*\n${{ env.STATUS_TEXT }}\n\n${{ env.COMPONENT }} container release ${{ env.RELEASE_TAG }} push ${{ env.STATUS_TEXT }}\n\n<${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/actions/runs/${{ env.GITHUB_RUN_ID }}|View run>" + } + } + ] + } + ] +} diff --git a/.github/scripts/slack-messages/container-release-started.json b/.github/scripts/slack-messages/container-release-started.json new file mode 100644 index 0000000000..f67129e3ba --- /dev/null +++ b/.github/scripts/slack-messages/container-release-started.json @@ -0,0 +1,17 @@ +{ + "channel": "${{ env.SLACK_CHANNEL_ID }}", + "attachments": [ + { + "color": "${{ env.STATUS_COLOR }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*Status:*\nStarted\n\n${{ env.COMPONENT }} container release ${{ env.RELEASE_TAG }} push started...\n\n<${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/actions/runs/${{ env.GITHUB_RUN_ID }}|View run>" + } + } + ] + } + ] +} diff --git a/.github/workflows/api-build-lint-push-containers.yml b/.github/workflows/api-build-lint-push-containers.yml deleted file mode 100644 index e7010cef24..0000000000 --- a/.github/workflows/api-build-lint-push-containers.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: API - Build and Push containers - -on: - push: - branches: - - "master" - paths: - - "api/**" - - "prowler/**" - - ".github/workflows/api-build-lint-push-containers.yml" - - # Uncomment the code below to test this action on PRs - # pull_request: - # branches: - # - "master" - # paths: - # - "api/**" - # - ".github/workflows/api-build-lint-push-containers.yml" - - release: - types: [published] - -env: - # Tags - LATEST_TAG: latest - RELEASE_TAG: ${{ github.event.release.tag_name }} - STABLE_TAG: stable - - WORKING_DIRECTORY: ./api - - # Container Registries - PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud - PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-api - -jobs: - repository-check: - name: Repository check - runs-on: ubuntu-latest - outputs: - is_repo: ${{ steps.repository_check.outputs.is_repo }} - steps: - - name: Repository check - id: repository_check - working-directory: /tmp - run: | - if [[ ${{ github.repository }} == "prowler-cloud/prowler" ]] - then - echo "is_repo=true" >> "${GITHUB_OUTPUT}" - else - echo "This action only runs for prowler-cloud/prowler" - echo "is_repo=false" >> "${GITHUB_OUTPUT}" - fi - - # Build Prowler OSS container - container-build-push: - needs: repository-check - if: needs.repository-check.outputs.is_repo == 'true' - runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ env.WORKING_DIRECTORY }} - - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Set short git commit SHA - id: vars - run: | - shortSha=$(git rev-parse --short ${{ github.sha }}) - echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV - - - name: Login to DockerHub - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Build and push container image (latest) - # Comment the following line for testing - if: github.event_name == 'push' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.WORKING_DIRECTORY }} - # Set push: false for testing - push: true - tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.SHORT_SHA }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push container image (release) - if: github.event_name == 'release' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.WORKING_DIRECTORY }} - push: true - tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Trigger deployment - if: github.event_name == 'push' - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 - with: - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - repository: ${{ secrets.CLOUD_DISPATCH }} - event-type: prowler-api-deploy - client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ env.SHORT_SHA }}"}' diff --git a/.github/workflows/api-code-quality.yml b/.github/workflows/api-code-quality.yml new file mode 100644 index 0000000000..a9dddaa3aa --- /dev/null +++ b/.github/workflows/api-code-quality.yml @@ -0,0 +1,71 @@ +name: 'API: Code Quality' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + API_WORKING_DIR: ./api + +jobs: + api-code-quality: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.12' + defaults: + run: + working-directory: ./api + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + api/** + .github/workflows/api-code-quality.yml + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Setup Python with Poetry + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/setup-python-poetry + with: + python-version: ${{ matrix.python-version }} + working-directory: ./api + + - name: Poetry check + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry check --lock + + - name: Ruff lint + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run ruff check . --exclude contrib + + - name: Ruff format + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run ruff format --check . --exclude contrib + + - name: Pylint + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn src/ diff --git a/.github/workflows/api-codeql.yml b/.github/workflows/api-codeql.yml index e9af830156..16ea82538e 100644 --- a/.github/workflows/api-codeql.yml +++ b/.github/workflows/api-codeql.yml @@ -25,7 +25,7 @@ concurrency: cancel-in-progress: true jobs: - analyze: + api-analyze: name: CodeQL Security Analysis runs-on: ubuntu-latest timeout-minutes: 30 @@ -45,12 +45,12 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Initialize CodeQL - uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/api-codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/api-container-build-push.yml b/.github/workflows/api-container-build-push.yml new file mode 100644 index 0000000000..3b3b4a4222 --- /dev/null +++ b/.github/workflows/api-container-build-push.yml @@ -0,0 +1,177 @@ +name: 'API: Container Build and Push' + +on: + push: + branches: + - 'master' + paths: + - 'api/**' + - 'prowler/**' + - '.github/workflows/api-build-lint-push-containers.yml' + release: + types: + - 'published' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + # Tags + LATEST_TAG: latest + RELEASE_TAG: ${{ github.event.release.tag_name }} + STABLE_TAG: stable + WORKING_DIRECTORY: ./api + + # Container registries + PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud + PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-api + +jobs: + setup: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + short-sha: ${{ steps.set-short-sha.outputs.short-sha }} + steps: + - name: Calculate short SHA + id: set-short-sha + run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + container-build-push: + needs: setup + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 + timeout-minutes: 30 + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Login to DockerHub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build and push API container for ${{ matrix.arch }} + if: github.event_name == 'push' || github.event_name == 'release' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.WORKING_DIRECTORY }} + push: true + platforms: ${{ matrix.platform }} + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + + - name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + - name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: API + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} + + # Create and push multi-architecture manifest + create-manifest: + needs: [setup, container-build-push] + if: github.event_name == 'push' || github.event_name == 'release' + runs-on: ubuntu-latest + + steps: + - name: Login to DockerHub + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Create and push manifests for push event + if: github.event_name == 'push' + run: | + docker buildx imagetools create \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64 + + - name: Create and push manifests for release event + if: github.event_name == 'release' + run: | + docker buildx imagetools create \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64 + + - name: Install regctl + if: always() + uses: regclient/actions/regctl-installer@f61d18f46c86af724a9c804cb9ff2a6fec741c7c # main + + - name: Cleanup intermediate architecture tags + if: always() + run: | + echo "Cleaning up intermediate tags..." + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64" || true + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true + echo "Cleanup completed" + + trigger-deployment: + if: github.event_name == 'push' + needs: [setup, container-build-push] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Trigger API deployment + uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + with: + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + repository: ${{ secrets.CLOUD_DISPATCH }} + event-type: api-prowler-deployment + client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}' diff --git a/.github/workflows/api-container-checks.yml b/.github/workflows/api-container-checks.yml new file mode 100644 index 0000000000..d069b0d5a6 --- /dev/null +++ b/.github/workflows/api-container-checks.yml @@ -0,0 +1,89 @@ +name: 'API: Container Checks' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + API_WORKING_DIR: ./api + IMAGE_NAME: prowler-api + +jobs: + api-dockerfile-lint: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check if Dockerfile changed + id: dockerfile-changed + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: api/Dockerfile + + - name: Lint Dockerfile with Hadolint + if: steps.dockerfile-changed.outputs.any_changed == 'true' + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: api/Dockerfile + ignore: DL3013 + + api-container-build-and-scan: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: api/** + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Set up Docker Buildx + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build container + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.API_WORKING_DIR }} + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Scan container with Trivy + if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/trivy-scan + with: + image-name: ${{ env.IMAGE_NAME }} + image-tag: ${{ github.sha }} + fail-on-critical: 'false' + severity: 'CRITICAL' diff --git a/.github/workflows/api-pull-request.yml b/.github/workflows/api-pull-request.yml deleted file mode 100644 index 1266ce78c9..0000000000 --- a/.github/workflows/api-pull-request.yml +++ /dev/null @@ -1,228 +0,0 @@ -name: 'API: Pull Request' - -on: - push: - branches: - - 'master' - - 'v5.*' - paths: - - '.github/workflows/api-pull-request.yml' - - 'api/**' - - '!api/docs/**' - - '!api/README.md' - - '!api/CHANGELOG.md' - pull_request: - branches: - - 'master' - - 'v5.*' - paths: - - '.github/workflows/api-pull-request.yml' - - 'api/**' - - '!api/docs/**' - - '!api/README.md' - - '!api/CHANGELOG.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 - POSTGRES_ADMIN_USER: prowler - POSTGRES_ADMIN_PASSWORD: S3cret - POSTGRES_USER: prowler_user - POSTGRES_PASSWORD: prowler - POSTGRES_DB: postgres-db - VALKEY_HOST: localhost - VALKEY_PORT: 6379 - VALKEY_DB: 0 - API_WORKING_DIR: ./api - IMAGE_NAME: prowler-api - -jobs: - code-quality: - if: github.repository == 'prowler-cloud/prowler' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - strategy: - matrix: - python-version: - - '3.12' - defaults: - run: - working-directory: ./api - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup Python with Poetry - uses: ./.github/actions/setup-python-poetry - with: - python-version: ${{ matrix.python-version }} - working-directory: ./api - - - name: Poetry check - run: poetry check --lock - - - name: Ruff lint - run: poetry run ruff check . --exclude contrib - - - name: Ruff format - run: poetry run ruff format --check . --exclude contrib - - - name: Pylint - run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn src/ - - security-scans: - if: github.repository == 'prowler-cloud/prowler' - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - strategy: - matrix: - python-version: - - '3.12' - defaults: - run: - working-directory: ./api - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup Python with Poetry - uses: ./.github/actions/setup-python-poetry - with: - python-version: ${{ matrix.python-version }} - working-directory: ./api - - - name: Bandit - run: poetry run bandit -q -lll -x '*_test.py,./contrib/' -r . - - - name: Safety - # 76352, 76353, 77323 come from SDK, but they cannot upgrade it yet. It does not affect API - # TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X - run: poetry run safety check --ignore 70612,66963,74429,76352,76353,77323,77744,77745 - - - name: Vulture - run: poetry run vulture --exclude "contrib,tests,conftest.py" --min-confidence 100 . - - tests: - if: github.repository == 'prowler-cloud/prowler' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - strategy: - matrix: - python-version: - - '3.12' - defaults: - run: - working-directory: ./api - - services: - postgres: - image: postgres - env: - POSTGRES_HOST: ${{ env.POSTGRES_HOST }} - POSTGRES_PORT: ${{ env.POSTGRES_PORT }} - POSTGRES_USER: ${{ env.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} - POSTGRES_DB: ${{ env.POSTGRES_DB }} - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - valkey: - image: valkey/valkey:7-alpine3.19 - env: - VALKEY_HOST: ${{ env.VALKEY_HOST }} - VALKEY_PORT: ${{ env.VALKEY_PORT }} - VALKEY_DB: ${{ env.VALKEY_DB }} - ports: - - 6379:6379 - options: >- - --health-cmd "valkey-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup Python with Poetry - uses: ./.github/actions/setup-python-poetry - with: - python-version: ${{ matrix.python-version }} - working-directory: ./api - - - name: Run tests with pytest - run: poetry run pytest --cov=./src/backend --cov-report=xml src/backend - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - flags: api - - dockerfile-lint: - if: github.repository == 'prowler-cloud/prowler' - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Lint Dockerfile with Hadolint - uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 - with: - dockerfile: api/Dockerfile - ignore: DL3013 - - container-build-and-scan: - if: github.repository == 'prowler-cloud/prowler' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - security-events: write - pull-requests: write - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Build container - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.API_WORKING_DIR }} - push: false - load: true - tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Scan container with Trivy - uses: ./.github/actions/trivy-scan - with: - image-name: ${{ env.IMAGE_NAME }} - image-tag: ${{ github.sha }} - fail-on-critical: 'false' - severity: 'CRITICAL' diff --git a/.github/workflows/api-security.yml b/.github/workflows/api-security.yml new file mode 100644 index 0000000000..33134ec9c9 --- /dev/null +++ b/.github/workflows/api-security.yml @@ -0,0 +1,69 @@ +name: 'API: Security' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + API_WORKING_DIR: ./api + +jobs: + api-security-scans: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.12' + defaults: + run: + working-directory: ./api + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + api/** + .github/workflows/api-security.yml + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Setup Python with Poetry + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/setup-python-poetry + with: + python-version: ${{ matrix.python-version }} + working-directory: ./api + + - name: Bandit + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run bandit -q -lll -x '*_test.py,./contrib/' -r . + + - name: Safety + if: steps.check-changes.outputs.any_changed == 'true' + # 76352, 76353, 77323 come from SDK, but they cannot upgrade it yet. It does not affect API + # TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X + run: poetry run safety check --ignore 70612,66963,74429,76352,76353,77323,77744,77745 + + - name: Vulture + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run vulture --exclude "contrib,tests,conftest.py" --min-confidence 100 . diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml new file mode 100644 index 0000000000..daeb79abac --- /dev/null +++ b/.github/workflows/api-tests.yml @@ -0,0 +1,107 @@ +name: 'API: Tests' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + POSTGRES_ADMIN_USER: prowler + POSTGRES_ADMIN_PASSWORD: S3cret + POSTGRES_USER: prowler_user + POSTGRES_PASSWORD: prowler + POSTGRES_DB: postgres-db + VALKEY_HOST: localhost + VALKEY_PORT: 6379 + VALKEY_DB: 0 + API_WORKING_DIR: ./api + +jobs: + api-tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.12' + defaults: + run: + working-directory: ./api + + services: + postgres: + image: postgres + env: + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} + POSTGRES_PORT: ${{ env.POSTGRES_PORT }} + POSTGRES_USER: ${{ env.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} + POSTGRES_DB: ${{ env.POSTGRES_DB }} + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + valkey: + image: valkey/valkey:7-alpine3.19 + env: + VALKEY_HOST: ${{ env.VALKEY_HOST }} + VALKEY_PORT: ${{ env.VALKEY_PORT }} + VALKEY_DB: ${{ env.VALKEY_DB }} + ports: + - 6379:6379 + options: >- + --health-cmd "valkey-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + api/** + .github/workflows/api-tests.yml + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Setup Python with Poetry + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/setup-python-poetry + with: + python-version: ${{ matrix.python-version }} + working-directory: ./api + + - name: Run tests with pytest + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run pytest --cov=./src/backend --cov-report=xml src/backend + + - name: Upload coverage reports to Codecov + if: steps.check-changes.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: api diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 4fd1347f44..974d919fc6 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -7,8 +7,6 @@ on: types: - 'labeled' - 'closed' - paths: - - '.github/workflows/backport.yml' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -40,7 +38,7 @@ jobs: - name: Backport PR if: steps.label_check.outputs.label_check == 'success' - uses: sorenlouv/backport-github-action@ad888e978060bc1b2798690dd9d03c4036560947 # v9.5.1 + uses: sorenlouv/backport-github-action@516854e7c9f962b9939085c9a92ea28411d1ae90 # v10.2.0 with: github_token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} auto_backport_label_prefix: ${{ env.BACKPORT_LABEL_PREFIX }} diff --git a/.github/workflows/comment-label-update.yml b/.github/workflows/comment-label-update.yml new file mode 100644 index 0000000000..9bddd6a9dd --- /dev/null +++ b/.github/workflows/comment-label-update.yml @@ -0,0 +1,39 @@ +name: 'Tools: Comment Label Update' + +on: + issue_comment: + types: + - 'created' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.number }} + cancel-in-progress: false + +jobs: + update-labels: + if: contains(github.event.issue.labels.*.name, 'status/awaiting-response') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + pull-requests: write + + steps: + - name: Remove 'status/awaiting-response' label + env: + GH_TOKEN: ${{ github.token }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + run: | + echo "Removing 'status/awaiting-response' label from #$ISSUE_NUMBER" + gh api /repos/${{ github.repository }}/issues/$ISSUE_NUMBER/labels/status%2Fawaiting-response \ + -X DELETE + + - name: Add 'status/waiting-for-revision' label + env: + GH_TOKEN: ${{ github.token }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + run: | + echo "Adding 'status/waiting-for-revision' label to #$ISSUE_NUMBER" + gh api /repos/${{ github.repository }}/issues/$ISSUE_NUMBER/labels \ + -X POST \ + -f labels[]='status/waiting-for-revision' diff --git a/.github/workflows/conventional-commit.yml b/.github/workflows/conventional-commit.yml index c5ba446edf..58e1653b74 100644 --- a/.github/workflows/conventional-commit.yml +++ b/.github/workflows/conventional-commit.yml @@ -26,6 +26,6 @@ jobs: steps: - name: Check PR title format - uses: agenthunt/conventional-commit-checker-action@9e552d650d0e205553ec7792d447929fc78e012b # v2.0.0 + uses: agenthunt/conventional-commit-checker-action@f1823f632e95a64547566dcd2c7da920e67117ad # v2.0.1 with: pr-title-regex: '^(feat|fix|docs|style|refactor|perf|test|chore|build|ci|revert)(\([^)]+\))?!?: .+' diff --git a/.github/workflows/find-secrets.yml b/.github/workflows/find-secrets.yml index 6428cf8f08..80b49e3279 100644 --- a/.github/workflows/find-secrets.yml +++ b/.github/workflows/find-secrets.yml @@ -28,6 +28,6 @@ jobs: fetch-depth: 0 - name: Scan for secrets with TruffleHog - uses: trufflesecurity/trufflehog@ad6fc8fb446b8fafbf7ea8193d2d6bfd42f45690 # v3.90.11 + uses: trufflesecurity/trufflehog@b84c3d14d189e16da175e2c27fa8136603783ffc # v3.90.12 with: extra_args: '--results=verified,unknown' diff --git a/.github/workflows/labeler-community.yml b/.github/workflows/labeler-community.yml deleted file mode 100644 index be64a0c397..0000000000 --- a/.github/workflows/labeler-community.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Label Community Contributors PRs - -on: - pull_request: - types: [opened] - -jobs: - add-community-label: - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - name: Label community contributors - env: - GH_TOKEN: ${{ github.token }} - run: | - # Fetch fresh PR data to get current author_association - ASSOCIATION=$(gh api /repos/${{ github.repository }}/pulls/${{ github.event.number }} --jq '.author_association') - AUTHOR=$(gh api /repos/${{ github.repository }}/pulls/${{ github.event.number }} --jq '.user.login') - - echo "Author: $AUTHOR, Association: $ASSOCIATION" - - # Members have associations like: OWNER, MEMBER, COLLABORATOR - # Non-members have: CONTRIBUTOR, FIRST_TIME_CONTRIBUTOR, FIRST_TIMER, NONE - if [[ "$ASSOCIATION" != "OWNER" && "$ASSOCIATION" != "MEMBER" && "$ASSOCIATION" != "COLLABORATOR" ]]; then - gh api /repos/${{ github.repository }}/issues/${{ github.event.number }}/labels \ - -X POST \ - -f labels[]='community' - echo "Added 'community' label for $ASSOCIATION contributor" - else - echo "Skipped labeling for $ASSOCIATION" - fi diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index fad94177f7..85ccd34cc2 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -27,3 +27,66 @@ jobs: uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 with: sync-labels: true + + label-community: + name: Add 'community' label if the PR is from a community contributor + needs: labeler + if: github.repository == 'prowler-cloud/prowler' && github.event.action == 'opened' + runs-on: ubuntu-latest + permissions: + pull-requests: write + + steps: + - name: Check if author is org member + id: check_membership + env: + AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + # Hardcoded list of prowler-cloud organization members + # This list includes members who have set their organization membership as private + ORG_MEMBERS=( + "AdriiiPRodri" + "Alan-TheGentleman" + "alejandrobailo" + "amitsharm" + "andoniaf" + "cesararroba" + "Chan9390" + "danibarranqueroo" + "HugoPBrito" + "jfagoagas" + "josemazo" + "lydiavilchez" + "mmuller88" + "MrCloudSec" + "pedrooot" + "prowler-bot" + "puchy22" + "rakan-pro" + "RosaRivasProwler" + "StylusFrost" + "toniblyx" + "vicferpoy" + ) + + echo "Checking if $AUTHOR is a member of prowler-cloud organization" + + # Check if author is in the org members list + if printf '%s\n' "${ORG_MEMBERS[@]}" | grep -q "^${AUTHOR}$"; then + echo "is_member=true" >> $GITHUB_OUTPUT + echo "$AUTHOR is an organization member" + else + echo "is_member=false" >> $GITHUB_OUTPUT + echo "$AUTHOR is not an organization member" + fi + + - name: Add community label + if: steps.check_membership.outputs.is_member == 'false' + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + GH_TOKEN: ${{ github.token }} + run: | + echo "Adding 'community' label to PR #$PR_NUMBER" + gh api /repos/${{ github.repository }}/issues/${{ github.event.number }}/labels \ + -X POST \ + -f labels[]='community' diff --git a/.github/workflows/mcp-container-build-push.yml b/.github/workflows/mcp-container-build-push.yml index aecec30592..27bc3e84e0 100644 --- a/.github/workflows/mcp-container-build-push.yml +++ b/.github/workflows/mcp-container-build-push.yml @@ -16,7 +16,7 @@ permissions: concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false env: # Tags @@ -43,7 +43,16 @@ jobs: container-build-push: needs: setup - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 timeout-minutes: 30 permissions: contents: read @@ -61,47 +70,111 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - name: Build and push MCP container (latest) - if: github.event_name == 'push' + - name: Build and push MCP container for ${{ matrix.arch }} + if: github.event_name == 'push' || github.event_name == 'release' uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: ${{ env.WORKING_DIRECTORY }} push: true + platforms: ${{ matrix.platform }} tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }} labels: | org.opencontainers.image.title=Prowler MCP Server org.opencontainers.image.description=Model Context Protocol server for Prowler org.opencontainers.image.vendor=ProwlerPro, Inc. org.opencontainers.image.source=https://github.com/${{ github.repository }} org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.created=${{ github.event.head_commit.timestamp }} - cache-from: type=gha - cache-to: type=gha,mode=max + org.opencontainers.image.created=${{ github.event_name == 'release' && github.event.release.published_at || github.event.head_commit.timestamp }} + ${{ github.event_name == 'release' && format('org.opencontainers.image.version={0}', env.RELEASE_TAG) || '' }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} - - name: Build and push MCP container (release) + - name: Notify container push started if: github.event_name == 'release' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: MCP + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} with: - context: ${{ env.WORKING_DIRECTORY }} - push: true - tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} - labels: | - org.opencontainers.image.title=Prowler MCP Server - org.opencontainers.image.description=Model Context Protocol server for Prowler - org.opencontainers.image.vendor=ProwlerPro, Inc. - org.opencontainers.image.version=${{ env.RELEASE_TAG }} - org.opencontainers.image.source=https://github.com/${{ github.repository }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.created=${{ github.event.release.published_at }} - cache-from: type=gha - cache-to: type=gha,mode=max + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" - - name: Trigger MCP deployment + - name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: MCP + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} + + # Create and push multi-architecture manifest + create-manifest: + needs: [setup, container-build-push] + if: github.event_name == 'push' || github.event_name == 'release' + runs-on: ubuntu-latest + + steps: + - name: Login to DockerHub + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Create and push manifests for push event if: github.event_name == 'push' + run: | + docker buildx imagetools create \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64 + + - name: Create and push manifests for release event + if: github.event_name == 'release' + run: | + docker buildx imagetools create \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64 + + - name: Install regctl + if: always() + uses: regclient/actions/regctl-installer@main + + - name: Cleanup intermediate architecture tags + if: always() + run: | + echo "Cleaning up intermediate tags..." + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64" || true + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true + echo "Cleanup completed" + + trigger-deployment: + if: github.event_name == 'push' + needs: [setup, container-build-push] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Trigger MCP deployment uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 with: token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} diff --git a/.github/workflows/mcp-pull-request.yml b/.github/workflows/mcp-container-checks.yml similarity index 65% rename from .github/workflows/mcp-pull-request.yml rename to .github/workflows/mcp-container-checks.yml index 31b4934c35..3310cd8c5a 100644 --- a/.github/workflows/mcp-pull-request.yml +++ b/.github/workflows/mcp-container-checks.yml @@ -1,24 +1,14 @@ -name: 'MCP: Pull Request' +name: 'MCP: Container Checks' on: push: branches: - 'master' - 'v5.*' - paths: - - '.github/workflows/mcp-pull-request.yml' - - 'mcp_server/**' - - '!mcp_server/README.md' - - '!mcp_server/CHANGELOG.md' pull_request: branches: - 'master' - 'v5.*' - paths: - - '.github/workflows/mcp-pull-request.yml' - - 'mcp_server/**' - - '!mcp_server/README.md' - - '!mcp_server/CHANGELOG.md' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -29,8 +19,7 @@ env: IMAGE_NAME: prowler-mcp jobs: - dockerfile-lint: - if: github.repository == 'prowler-cloud/prowler' + mcp-dockerfile-lint: runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -40,13 +29,19 @@ jobs: - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Check if Dockerfile changed + id: dockerfile-changed + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: mcp_server/Dockerfile + - name: Lint Dockerfile with Hadolint + if: steps.dockerfile-changed.outputs.any_changed == 'true' uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 with: dockerfile: mcp_server/Dockerfile - container-build-and-scan: - if: github.repository == 'prowler-cloud/prowler' + mcp-container-build-and-scan: runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -58,10 +53,21 @@ jobs: - name: Checkout repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Check for MCP changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: mcp_server/** + files_ignore: | + mcp_server/README.md + mcp_server/CHANGELOG.md + - name: Set up Docker Buildx + if: steps.check-changes.outputs.any_changed == 'true' uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Build MCP container + if: steps.check-changes.outputs.any_changed == 'true' uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: ${{ env.MCP_WORKING_DIR }} @@ -72,6 +78,7 @@ jobs: cache-to: type=gha,mode=max - name: Scan MCP container with Trivy + if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' uses: ./.github/actions/trivy-scan with: image-name: ${{ env.IMAGE_NAME }} diff --git a/.github/workflows/pr-check-changelog.yml b/.github/workflows/pr-check-changelog.yml index f8de212e2a..d1a3220402 100644 --- a/.github/workflows/pr-check-changelog.yml +++ b/.github/workflows/pr-check-changelog.yml @@ -83,7 +83,7 @@ jobs: - name: Update PR comment with changelog status if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 with: issue-number: ${{ github.event.pull_request.number }} comment-id: ${{ steps.find-comment.outputs.comment-id }} diff --git a/.github/workflows/pr-conflict-checker.yml b/.github/workflows/pr-conflict-checker.yml index 3761d252a3..c46fafc2d0 100644 --- a/.github/workflows/pr-conflict-checker.yml +++ b/.github/workflows/pr-conflict-checker.yml @@ -97,7 +97,7 @@ jobs: body-includes: '' - name: Create or update comment - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 with: comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 55b7d2e441..ae039bd5a6 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -47,7 +47,7 @@ jobs: git config --global user.name 'prowler-bot' git config --global user.email '179230569+prowler-bot@users.noreply.github.com' - - name: Parse version and read changelogs + - name: Parse version and determine branch run: | # Validate version format (reusing pattern from sdk-bump-version.yml) if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then @@ -64,66 +64,83 @@ jobs: BRANCH_NAME="v${MAJOR_VERSION}.${MINOR_VERSION}" echo "BRANCH_NAME=${BRANCH_NAME}" >> "${GITHUB_ENV}" - # Function to extract the latest version from changelog - extract_latest_version() { - local changelog_file="$1" - if [ -f "$changelog_file" ]; then - # Extract the first version entry (most recent) from changelog - # Format: ## [version] (1.2.3) or ## [vversion] (v1.2.3) - local version=$(grep -m 1 '^## \[' "$changelog_file" | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]') - echo "$version" - else - echo "" - fi - } - - # Read actual versions from changelogs (source of truth) - UI_VERSION=$(extract_latest_version "ui/CHANGELOG.md") - API_VERSION=$(extract_latest_version "api/CHANGELOG.md") - SDK_VERSION=$(extract_latest_version "prowler/CHANGELOG.md") - MCP_VERSION=$(extract_latest_version "mcp_server/CHANGELOG.md") - - echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}" - echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}" - echo "SDK_VERSION=${SDK_VERSION}" >> "${GITHUB_ENV}" - echo "MCP_VERSION=${MCP_VERSION}" >> "${GITHUB_ENV}" - - if [ -n "$UI_VERSION" ]; then - echo "Read UI version from changelog: $UI_VERSION" - else - echo "Warning: No UI version found in ui/CHANGELOG.md" - fi - - if [ -n "$API_VERSION" ]; then - echo "Read API version from changelog: $API_VERSION" - else - echo "Warning: No API version found in api/CHANGELOG.md" - fi - - if [ -n "$SDK_VERSION" ]; then - echo "Read SDK version from changelog: $SDK_VERSION" - else - echo "Warning: No SDK version found in prowler/CHANGELOG.md" - fi - - if [ -n "$MCP_VERSION" ]; then - echo "Read MCP version from changelog: $MCP_VERSION" - else - echo "Warning: No MCP version found in mcp_server/CHANGELOG.md" - fi - echo "Prowler version: $PROWLER_VERSION" echo "Branch name: $BRANCH_NAME" - echo "UI version: $UI_VERSION" - echo "API version: $API_VERSION" - echo "SDK version: $SDK_VERSION" - echo "MCP version: $MCP_VERSION" echo "Is minor release: $([ $PATCH_VERSION -eq 0 ] && echo 'true' || echo 'false')" else echo "Invalid version syntax: '$PROWLER_VERSION' (must be N.N.N)" >&2 exit 1 fi + - name: Checkout release branch + run: | + echo "Checking out branch $BRANCH_NAME for release $PROWLER_VERSION..." + if git show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then + echo "Branch $BRANCH_NAME exists locally, checking out..." + git checkout "$BRANCH_NAME" + elif git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then + echo "Branch $BRANCH_NAME exists remotely, checking out..." + git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME" + else + echo "ERROR: Branch $BRANCH_NAME does not exist. For minor releases (X.Y.0), create it manually first. For patch releases (X.Y.Z), the branch should already exist." + exit 1 + fi + + - name: Read changelog versions from release branch + run: | + # Function to extract the latest version from changelog + extract_latest_version() { + local changelog_file="$1" + if [ -f "$changelog_file" ]; then + # Extract the first version entry (most recent) from changelog + # Format: ## [version] (1.2.3) or ## [vversion] (v1.2.3) + local version=$(grep -m 1 '^## \[' "$changelog_file" | sed 's/^## \[\(.*\)\].*/\1/' | sed 's/^v//' | tr -d '[:space:]') + echo "$version" + else + echo "" + fi + } + + # Read actual versions from changelogs (source of truth) + UI_VERSION=$(extract_latest_version "ui/CHANGELOG.md") + API_VERSION=$(extract_latest_version "api/CHANGELOG.md") + SDK_VERSION=$(extract_latest_version "prowler/CHANGELOG.md") + MCP_VERSION=$(extract_latest_version "mcp_server/CHANGELOG.md") + + echo "UI_VERSION=${UI_VERSION}" >> "${GITHUB_ENV}" + echo "API_VERSION=${API_VERSION}" >> "${GITHUB_ENV}" + echo "SDK_VERSION=${SDK_VERSION}" >> "${GITHUB_ENV}" + echo "MCP_VERSION=${MCP_VERSION}" >> "${GITHUB_ENV}" + + if [ -n "$UI_VERSION" ]; then + echo "Read UI version from changelog: $UI_VERSION" + else + echo "Warning: No UI version found in ui/CHANGELOG.md" + fi + + if [ -n "$API_VERSION" ]; then + echo "Read API version from changelog: $API_VERSION" + else + echo "Warning: No API version found in api/CHANGELOG.md" + fi + + if [ -n "$SDK_VERSION" ]; then + echo "Read SDK version from changelog: $SDK_VERSION" + else + echo "Warning: No SDK version found in prowler/CHANGELOG.md" + fi + + if [ -n "$MCP_VERSION" ]; then + echo "Read MCP version from changelog: $MCP_VERSION" + else + echo "Warning: No MCP version found in mcp_server/CHANGELOG.md" + fi + + echo "UI version: $UI_VERSION" + echo "API version: $API_VERSION" + echo "SDK version: $SDK_VERSION" + echo "MCP version: $MCP_VERSION" + - name: Extract and combine changelog entries run: | set -e @@ -255,21 +272,6 @@ jobs: echo "Combined changelog preview:" cat combined_changelog.md - - name: Checkout release branch for patch release - if: ${{ env.PATCH_VERSION != '0' }} - run: | - echo "Patch release detected, checking out existing branch $BRANCH_NAME..." - if git show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then - echo "Branch $BRANCH_NAME exists locally, checking out..." - git checkout "$BRANCH_NAME" - elif git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then - echo "Branch $BRANCH_NAME exists remotely, checking out..." - git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME" - else - echo "ERROR: Branch $BRANCH_NAME should exist for patch release $PROWLER_VERSION" - exit 1 - fi - - name: Verify SDK version in pyproject.toml run: | CURRENT_VERSION=$(grep '^version = ' pyproject.toml | sed -E 's/version = "([^"]+)"/\1/' | tr -d '[:space:]') @@ -323,18 +325,6 @@ jobs: fi echo "✓ api/src/backend/api/v1/views.py version: $CURRENT_API_VERSION" - - name: Checkout release branch for minor release - if: ${{ env.PATCH_VERSION == '0' }} - run: | - echo "Minor release detected (patch = 0), checking out existing branch $BRANCH_NAME..." - if git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then - echo "Branch $BRANCH_NAME exists remotely, checking out..." - git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME" - else - echo "ERROR: Branch $BRANCH_NAME should exist for minor release $PROWLER_VERSION. Please create it manually first." - exit 1 - fi - - name: Update API prowler dependency for minor release if: ${{ env.PATCH_VERSION == '0' }} run: | @@ -392,7 +382,7 @@ jobs: no-changelog - name: Create draft release - uses: softprops/action-gh-release@6cbd405e2c4e67a21c47fa9e383d020e4e28b836 # v2.3.3 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2.4.1 with: tag_name: ${{ env.PROWLER_VERSION }} name: Prowler ${{ env.PROWLER_VERSION }} diff --git a/.github/workflows/sdk-build-lint-push-containers.yml b/.github/workflows/sdk-build-lint-push-containers.yml deleted file mode 100644 index 31669924b7..0000000000 --- a/.github/workflows/sdk-build-lint-push-containers.yml +++ /dev/null @@ -1,202 +0,0 @@ -name: SDK - Build and Push containers - -on: - push: - branches: - # For `v3-latest` - - "v3" - # For `v4-latest` - - "v4.6" - # For `latest` - - "master" - paths-ignore: - - ".github/**" - - "README.md" - - "docs/**" - - "ui/**" - - "api/**" - - release: - types: [published] - -env: - # AWS Configuration - AWS_REGION_STG: eu-west-1 - AWS_REGION_PLATFORM: eu-west-1 - AWS_REGION: us-east-1 - - # Container's configuration - IMAGE_NAME: prowler - DOCKERFILE_PATH: ./Dockerfile - - # Tags - LATEST_TAG: latest - STABLE_TAG: stable - # The RELEASE_TAG is set during runtime in releases - RELEASE_TAG: "" - # The PROWLER_VERSION and PROWLER_VERSION_MAJOR are set during runtime in releases - PROWLER_VERSION: "" - PROWLER_VERSION_MAJOR: "" - # TEMPORARY_TAG: temporary - - # Python configuration - PYTHON_VERSION: 3.12 - - # Container Registries - PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud - PROWLERCLOUD_DOCKERHUB_IMAGE: prowler - -jobs: - # Build Prowler OSS container - container-build-push: - # needs: dockerfile-linter - runs-on: ubuntu-latest - outputs: - prowler_version_major: ${{ steps.get-prowler-version.outputs.PROWLER_VERSION_MAJOR }} - prowler_version: ${{ steps.get-prowler-version.outputs.PROWLER_VERSION }} - env: - POETRY_VIRTUALENVS_CREATE: "false" - - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install Poetry - run: | - pipx install poetry==2.* - pipx inject poetry poetry-bumpversion - - - name: Get Prowler version - id: get-prowler-version - run: | - PROWLER_VERSION="$(poetry version -s 2>/dev/null)" - echo "PROWLER_VERSION=${PROWLER_VERSION}" >> "${GITHUB_ENV}" - echo "PROWLER_VERSION=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}" - - # Store prowler version major just for the release - PROWLER_VERSION_MAJOR="${PROWLER_VERSION%%.*}" - echo "PROWLER_VERSION_MAJOR=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_ENV}" - echo "PROWLER_VERSION_MAJOR=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_OUTPUT}" - - case ${PROWLER_VERSION_MAJOR} in - 3) - echo "LATEST_TAG=v3-latest" >> "${GITHUB_ENV}" - echo "STABLE_TAG=v3-stable" >> "${GITHUB_ENV}" - ;; - - - 4) - echo "LATEST_TAG=v4-latest" >> "${GITHUB_ENV}" - echo "STABLE_TAG=v4-stable" >> "${GITHUB_ENV}" - ;; - - 5) - echo "LATEST_TAG=latest" >> "${GITHUB_ENV}" - echo "STABLE_TAG=stable" >> "${GITHUB_ENV}" - ;; - - *) - # Fallback if any other version is present - echo "Releasing another Prowler major version, aborting..." - exit 1 - ;; - esac - - - name: Login to DockerHub - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to Public ECR - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - registry: public.ecr.aws - username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }} - password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }} - env: - AWS_REGION: ${{ env.AWS_REGION }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Build and push container image (latest) - if: github.event_name == 'push' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - push: true - tags: | - ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }} - ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} - file: ${{ env.DOCKERFILE_PATH }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push container image (release) - if: github.event_name == 'release' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - # Use local context to get changes - # https://github.com/docker/build-push-action#path-context - context: . - push: true - tags: | - ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.PROWLER_VERSION }} - ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.STABLE_TAG }} - ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.PROWLER_VERSION }} - ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ env.STABLE_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.PROWLER_VERSION }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} - file: ${{ env.DOCKERFILE_PATH }} - cache-from: type=gha - cache-to: type=gha,mode=max - -# - name: Push README to Docker Hub (toniblyx) -# uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} -# repository: ${{ env.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }} -# readme-filepath: ./README.md -# -# - name: Push README to Docker Hub (prowlercloud) -# uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} -# repository: ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }} -# readme-filepath: ./README.md - - dispatch-action: - needs: container-build-push - runs-on: ubuntu-latest - steps: - - name: Get latest commit info (latest) - if: github.event_name == 'push' - run: | - LATEST_COMMIT_HASH=$(echo ${{ github.event.after }} | cut -b -7) - echo "LATEST_COMMIT_HASH=${LATEST_COMMIT_HASH}" >> $GITHUB_ENV - - - name: Dispatch event (latest) - if: github.event_name == 'push' && needs.container-build-push.outputs.prowler_version_major == '3' - run: | - curl https://api.github.com/repos/${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}/dispatches \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - --data '{"event_type":"dispatch","client_payload":{"version":"v3-latest", "tag": "${{ env.LATEST_COMMIT_HASH }}"}}' - - - name: Dispatch event (release) - if: github.event_name == 'release' && needs.container-build-push.outputs.prowler_version_major == '3' - run: | - curl https://api.github.com/repos/${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }}/dispatches \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - --data '{"event_type":"dispatch","client_payload":{"version":"release", "tag":"${{ needs.container-build-push.outputs.prowler_version }}"}}' diff --git a/.github/workflows/sdk-bump-version.yml b/.github/workflows/sdk-bump-version.yml index e3463e1788..0291502ab6 100644 --- a/.github/workflows/sdk-bump-version.yml +++ b/.github/workflows/sdk-bump-version.yml @@ -1,146 +1,218 @@ -name: SDK - Bump Version +name: 'SDK: Bump Version' on: release: - types: [published] + types: + - 'published' +concurrency: + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false env: PROWLER_VERSION: ${{ github.event.release.tag_name }} BASE_BRANCH: master jobs: - bump-version: - name: Bump Version + detect-release-type: runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + is_minor: ${{ steps.detect.outputs.is_minor }} + is_patch: ${{ steps.detect.outputs.is_patch }} + major_version: ${{ steps.detect.outputs.major_version }} + minor_version: ${{ steps.detect.outputs.minor_version }} + patch_version: ${{ steps.detect.outputs.patch_version }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Get Prowler version - shell: bash + - name: Detect release type and parse version + id: detect run: | if [[ $PROWLER_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then MAJOR_VERSION=${BASH_REMATCH[1]} MINOR_VERSION=${BASH_REMATCH[2]} - FIX_VERSION=${BASH_REMATCH[3]} + PATCH_VERSION=${BASH_REMATCH[3]} - # Export version components to GitHub environment - echo "MAJOR_VERSION=${MAJOR_VERSION}" >> "${GITHUB_ENV}" - echo "MINOR_VERSION=${MINOR_VERSION}" >> "${GITHUB_ENV}" - echo "FIX_VERSION=${FIX_VERSION}" >> "${GITHUB_ENV}" + echo "major_version=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "minor_version=${MINOR_VERSION}" >> "${GITHUB_OUTPUT}" + echo "patch_version=${PATCH_VERSION}" >> "${GITHUB_OUTPUT}" - if (( MAJOR_VERSION == 5 )); then - if (( FIX_VERSION == 0 )); then - echo "Minor Release: $PROWLER_VERSION" + if (( MAJOR_VERSION != 5 )); then + echo "::error::Releasing another Prowler major version, aborting..." + exit 1 + fi - # Set up next minor version for master - BUMP_VERSION_TO=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).${FIX_VERSION} - echo "BUMP_VERSION_TO=${BUMP_VERSION_TO}" >> "${GITHUB_ENV}" - - TARGET_BRANCH=${BASE_BRANCH} - echo "TARGET_BRANCH=${TARGET_BRANCH}" >> "${GITHUB_ENV}" - - # Set up patch version for version branch - PATCH_VERSION_TO=${MAJOR_VERSION}.${MINOR_VERSION}.1 - echo "PATCH_VERSION_TO=${PATCH_VERSION_TO}" >> "${GITHUB_ENV}" - - VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} - echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" - - echo "Bumping to next minor version: ${BUMP_VERSION_TO} in branch ${TARGET_BRANCH}" - echo "Bumping to next patch version: ${PATCH_VERSION_TO} in branch ${VERSION_BRANCH}" - else - echo "Patch Release: $PROWLER_VERSION" - - BUMP_VERSION_TO=${MAJOR_VERSION}.${MINOR_VERSION}.$((FIX_VERSION + 1)) - echo "BUMP_VERSION_TO=${BUMP_VERSION_TO}" >> "${GITHUB_ENV}" - - TARGET_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} - echo "TARGET_BRANCH=${TARGET_BRANCH}" >> "${GITHUB_ENV}" - - echo "Bumping to next patch version: ${BUMP_VERSION_TO} in branch ${TARGET_BRANCH}" - fi + if (( PATCH_VERSION == 0 )); then + echo "is_minor=true" >> "${GITHUB_OUTPUT}" + echo "is_patch=false" >> "${GITHUB_OUTPUT}" + echo "✓ Minor release detected: $PROWLER_VERSION" else - echo "Releasing another Prowler major version, aborting..." - exit 1 + echo "is_minor=false" >> "${GITHUB_OUTPUT}" + echo "is_patch=true" >> "${GITHUB_OUTPUT}" + echo "✓ Patch release detected: $PROWLER_VERSION" fi else - echo "Invalid version syntax: '$PROWLER_VERSION' (must be N.N.N)" >&2 + echo "::error::Invalid version syntax: '$PROWLER_VERSION' (must be X.Y.Z)" exit 1 fi - - name: Bump versions in files + bump-minor-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_minor == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Calculate next minor version run: | - echo "Using PROWLER_VERSION=$PROWLER_VERSION" - echo "Using BUMP_VERSION_TO=$BUMP_VERSION_TO" + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} - set -e + NEXT_MINOR_VERSION=${MAJOR_VERSION}.$((MINOR_VERSION + 1)).0 + echo "NEXT_MINOR_VERSION=${NEXT_MINOR_VERSION}" >> "${GITHUB_ENV}" - echo "Bumping version in pyproject.toml ..." - sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${BUMP_VERSION_TO}\"|" pyproject.toml + echo "Current version: $PROWLER_VERSION" + echo "Next minor version: $NEXT_MINOR_VERSION" - echo "Bumping version in prowler/config/config.py ..." - sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${BUMP_VERSION_TO}\"|" prowler/config/config.py + - name: Bump versions in files for master + run: | + set -e - echo "Bumping version in .env ..." - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${BUMP_VERSION_TO}|" .env + sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_MINOR_VERSION}\"|" pyproject.toml + sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_MINOR_VERSION}\"|" prowler/config/config.py + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_MINOR_VERSION}|" .env - git --no-pager diff + echo "Files modified:" + git --no-pager diff - - name: Create Pull Request + - name: Create PR for next minor version to master uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: - author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - base: ${{ env.TARGET_BRANCH }} - commit-message: "chore(release): Bump version to v${{ env.BUMP_VERSION_TO }}" - branch: "version-bump-to-v${{ env.BUMP_VERSION_TO }}" - title: "chore(release): Bump version to v${{ env.BUMP_VERSION_TO }}" - labels: no-changelog - body: | - ### Description + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: master + commit-message: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}' + branch: version-bump-to-v${{ env.NEXT_MINOR_VERSION }} + title: 'chore(release): Bump version to v${{ env.NEXT_MINOR_VERSION }}' + labels: no-changelog + body: | + ### Description - Bump Prowler version to v${{ env.BUMP_VERSION_TO }} + Bump Prowler version to v${{ env.NEXT_MINOR_VERSION }} after releasing v${{ env.PROWLER_VERSION }}. - ### License + ### License - By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. - - name: Handle patch version for minor release - if: env.FIX_VERSION == '0' + - name: Checkout version branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: v${{ needs.detect-release-type.outputs.major_version }}.${{ needs.detect-release-type.outputs.minor_version }} + + - name: Calculate first patch version run: | - echo "Using PROWLER_VERSION=$PROWLER_VERSION" - echo "Using PATCH_VERSION_TO=$PATCH_VERSION_TO" + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} - set -e + FIRST_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.1 + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} - echo "Bumping version in pyproject.toml ..." - sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${PATCH_VERSION_TO}\"|" pyproject.toml + echo "FIRST_PATCH_VERSION=${FIRST_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" - echo "Bumping version in prowler/config/config.py ..." - sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${PATCH_VERSION_TO}\"|" prowler/config/config.py + echo "First patch version: $FIRST_PATCH_VERSION" + echo "Version branch: $VERSION_BRANCH" - echo "Bumping version in .env ..." - sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PATCH_VERSION_TO}|" .env + - name: Bump versions in files for version branch + run: | + set -e - git --no-pager diff + sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${FIRST_PATCH_VERSION}\"|" pyproject.toml + sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${FIRST_PATCH_VERSION}\"|" prowler/config/config.py + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${FIRST_PATCH_VERSION}|" .env - - name: Create Pull Request for patch version - if: env.FIX_VERSION == '0' + echo "Files modified:" + git --no-pager diff + + - name: Create PR for first patch version to version branch uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: - author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - base: ${{ env.VERSION_BRANCH }} - commit-message: "chore(release): Bump version to v${{ env.PATCH_VERSION_TO }}" - branch: "version-bump-to-v${{ env.PATCH_VERSION_TO }}" - title: "chore(release): Bump version to v${{ env.PATCH_VERSION_TO }}" - labels: no-changelog - body: | - ### Description + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}' + branch: version-bump-to-v${{ env.FIRST_PATCH_VERSION }} + title: 'chore(release): Bump version to v${{ env.FIRST_PATCH_VERSION }}' + labels: no-changelog + body: | + ### Description - Bump Prowler version to v${{ env.PATCH_VERSION_TO }} + Bump Prowler version to v${{ env.FIRST_PATCH_VERSION }} in version branch after releasing v${{ env.PROWLER_VERSION }}. - ### License + ### License - By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + bump-patch-version: + needs: detect-release-type + if: needs.detect-release-type.outputs.is_patch == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Calculate next patch version + run: | + MAJOR_VERSION=${{ needs.detect-release-type.outputs.major_version }} + MINOR_VERSION=${{ needs.detect-release-type.outputs.minor_version }} + PATCH_VERSION=${{ needs.detect-release-type.outputs.patch_version }} + + NEXT_PATCH_VERSION=${MAJOR_VERSION}.${MINOR_VERSION}.$((PATCH_VERSION + 1)) + VERSION_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} + + echo "NEXT_PATCH_VERSION=${NEXT_PATCH_VERSION}" >> "${GITHUB_ENV}" + echo "VERSION_BRANCH=${VERSION_BRANCH}" >> "${GITHUB_ENV}" + + echo "Current version: $PROWLER_VERSION" + echo "Next patch version: $NEXT_PATCH_VERSION" + echo "Target branch: $VERSION_BRANCH" + + - name: Bump versions in files for version branch + run: | + set -e + + sed -i "s|version = \"${PROWLER_VERSION}\"|version = \"${NEXT_PATCH_VERSION}\"|" pyproject.toml + sed -i "s|prowler_version = \"${PROWLER_VERSION}\"|prowler_version = \"${NEXT_PATCH_VERSION}\"|" prowler/config/config.py + sed -i "s|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${PROWLER_VERSION}|NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${NEXT_PATCH_VERSION}|" .env + + echo "Files modified:" + git --no-pager diff + + - name: Create PR for next patch version to version branch + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + with: + author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + base: ${{ env.VERSION_BRANCH }} + commit-message: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}' + branch: version-bump-to-v${{ env.NEXT_PATCH_VERSION }} + title: 'chore(release): Bump version to v${{ env.NEXT_PATCH_VERSION }}' + labels: no-changelog + body: | + ### Description + + Bump Prowler version to v${{ env.NEXT_PATCH_VERSION }} after releasing v${{ env.PROWLER_VERSION }}. + + ### License + + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. diff --git a/.github/workflows/sdk-code-quality.yml b/.github/workflows/sdk-code-quality.yml new file mode 100644 index 0000000000..b32324d8cb --- /dev/null +++ b/.github/workflows/sdk-code-quality.yml @@ -0,0 +1,90 @@ +name: 'SDK: Code Quality' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + sdk-code-quality: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.9' + - '3.10' + - '3.11' + - '3.12' + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for SDK changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ./** + files_ignore: | + .github/** + prowler/CHANGELOG.md + docs/** + permissions/** + api/** + ui/** + dashboard/** + mcp_server/** + README.md + mkdocs.yml + .backportrc.json + .env + docker-compose* + examples/** + .gitignore + contrib/** + + - name: Install Poetry + if: steps.check-changes.outputs.any_changed == 'true' + run: pipx install poetry==2.1.1 + + - name: Set up Python ${{ matrix.python-version }} + if: steps.check-changes.outputs.any_changed == 'true' + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + + - name: Install dependencies + if: steps.check-changes.outputs.any_changed == 'true' + run: | + poetry install --no-root + poetry run pip list + + - name: Check Poetry lock file + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry check --lock + + - name: Lint with flake8 + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api + + - name: Check format with black + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run black --exclude api ui --check . + + - name: Lint with pylint + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/ diff --git a/.github/workflows/sdk-codeql.yml b/.github/workflows/sdk-codeql.yml index ee9ba9c396..590ba52d34 100644 --- a/.github/workflows/sdk-codeql.yml +++ b/.github/workflows/sdk-codeql.yml @@ -31,7 +31,8 @@ concurrency: cancel-in-progress: true jobs: - analyze: + sdk-analyze: + if: github.repository == 'prowler-cloud/prowler' name: CodeQL Security Analysis runs-on: ubuntu-latest timeout-minutes: 30 @@ -51,12 +52,12 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Initialize CodeQL - uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/sdk-codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/sdk-container-build-push.yml b/.github/workflows/sdk-container-build-push.yml new file mode 100644 index 0000000000..72b57ac246 --- /dev/null +++ b/.github/workflows/sdk-container-build-push.yml @@ -0,0 +1,266 @@ +name: 'SDK: Container Build and Push' + +on: + push: + branches: + - 'v3' # For v3-latest + - 'v4.6' # For v4-latest + - 'master' # For latest + paths-ignore: + - '.github/**' + - '!.github/workflows/sdk-container-build-push.yml' + - 'README.md' + - 'docs/**' + - 'ui/**' + - 'api/**' + release: + types: + - 'published' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + # Container configuration + IMAGE_NAME: prowler + DOCKERFILE_PATH: ./Dockerfile + + # Python configuration + PYTHON_VERSION: '3.12' + + # Tags (dynamically set based on version) + LATEST_TAG: latest + STABLE_TAG: stable + + # Container registries + PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud + PROWLERCLOUD_DOCKERHUB_IMAGE: prowler + + # AWS configuration (for ECR) + AWS_REGION: us-east-1 + +jobs: + container-build-push: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 + timeout-minutes: 45 + permissions: + contents: read + packages: write + outputs: + prowler_version: ${{ steps.get-prowler-version.outputs.prowler_version }} + prowler_version_major: ${{ steps.get-prowler-version.outputs.prowler_version_major }} + latest_tag: ${{ steps.get-prowler-version.outputs.latest_tag }} + stable_tag: ${{ steps.get-prowler-version.outputs.stable_tag }} + env: + POETRY_VIRTUALENVS_CREATE: 'false' + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Poetry + run: | + pipx install poetry==2.1.1 + pipx inject poetry poetry-bumpversion + + - name: Get Prowler version and set tags + id: get-prowler-version + run: | + PROWLER_VERSION="$(poetry version -s 2>/dev/null)" + echo "prowler_version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}" + echo "PROWLER_VERSION=${PROWLER_VERSION}" >> "${GITHUB_ENV}" + + # Extract major version + PROWLER_VERSION_MAJOR="${PROWLER_VERSION%%.*}" + echo "prowler_version_major=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_OUTPUT}" + echo "PROWLER_VERSION_MAJOR=${PROWLER_VERSION_MAJOR}" >> "${GITHUB_ENV}" + + # Set version-specific tags + case ${PROWLER_VERSION_MAJOR} in + 3) + echo "LATEST_TAG=v3-latest" >> "${GITHUB_ENV}" + echo "STABLE_TAG=v3-stable" >> "${GITHUB_ENV}" + echo "latest_tag=v3-latest" >> "${GITHUB_OUTPUT}" + echo "stable_tag=v3-stable" >> "${GITHUB_OUTPUT}" + echo "✓ Prowler v3 detected - tags: v3-latest, v3-stable" + ;; + 4) + echo "LATEST_TAG=v4-latest" >> "${GITHUB_ENV}" + echo "STABLE_TAG=v4-stable" >> "${GITHUB_ENV}" + echo "latest_tag=v4-latest" >> "${GITHUB_OUTPUT}" + echo "stable_tag=v4-stable" >> "${GITHUB_OUTPUT}" + echo "✓ Prowler v4 detected - tags: v4-latest, v4-stable" + ;; + 5) + echo "LATEST_TAG=latest" >> "${GITHUB_ENV}" + echo "STABLE_TAG=stable" >> "${GITHUB_ENV}" + echo "latest_tag=latest" >> "${GITHUB_OUTPUT}" + echo "stable_tag=stable" >> "${GITHUB_OUTPUT}" + echo "✓ Prowler v5 detected - tags: latest, stable" + ;; + *) + echo "::error::Unsupported Prowler major version: ${PROWLER_VERSION_MAJOR}" + exit 1 + ;; + esac + + - name: Login to DockerHub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Public ECR + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: public.ecr.aws + username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }} + password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }} + env: + AWS_REGION: ${{ env.AWS_REGION }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build and push SDK container for ${{ matrix.arch }} + if: github.event_name == 'push' || github.event_name == 'release' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + file: ${{ env.DOCKERFILE_PATH }} + push: true + platforms: ${{ matrix.platform }} + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }}-${{ matrix.arch }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + + - name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: SDK + RELEASE_TAG: ${{ env.PROWLER_VERSION }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + - name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: SDK + RELEASE_TAG: ${{ env.PROWLER_VERSION }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} + + # Create and push multi-architecture manifest + create-manifest: + needs: [container-build-push] + if: github.event_name == 'push' || github.event_name == 'release' + runs-on: ubuntu-latest + + steps: + - name: Login to DockerHub + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Create and push manifests for push event + if: github.event_name == 'push' + run: | + docker buildx imagetools create \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \ + -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \ + -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64 + + - name: Create and push manifests for release event + if: github.event_name == 'release' + run: | + docker buildx imagetools create \ + -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.prowler_version }} \ + -t ${{ secrets.DOCKER_HUB_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.stable_tag }} \ + -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.prowler_version }} \ + -t ${{ secrets.PUBLIC_ECR_REPOSITORY }}/${{ env.IMAGE_NAME }}:${{ needs.container-build-push.outputs.stable_tag }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.prowler_version }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.stable_tag }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64 + + - name: Install regctl + if: always() + uses: regclient/actions/regctl-installer@main + + - name: Cleanup intermediate architecture tags + if: always() + run: | + echo "Cleaning up intermediate tags..." + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-amd64" || true + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.container-build-push.outputs.latest_tag }}-arm64" || true + echo "Cleanup completed" + + dispatch-v3-deployment: + if: needs.container-build-push.outputs.prowler_version_major == '3' + needs: container-build-push + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Calculate short SHA + id: short-sha + run: echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + - name: Dispatch v3 deployment (latest) + if: github.event_name == 'push' + uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + with: + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }} + event-type: dispatch + client-payload: '{"version":"v3-latest","tag":"${{ steps.short-sha.outputs.short_sha }}"}' + + - name: Dispatch v3 deployment (release) + if: github.event_name == 'release' + uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + with: + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + repository: ${{ secrets.DISPATCH_OWNER }}/${{ secrets.DISPATCH_REPO }} + event-type: dispatch + client-payload: '{"version":"release","tag":"${{ needs.container-build-push.outputs.prowler_version }}"}' diff --git a/.github/workflows/sdk-container-checks.yml b/.github/workflows/sdk-container-checks.yml new file mode 100644 index 0000000000..4e8a7cf2ad --- /dev/null +++ b/.github/workflows/sdk-container-checks.yml @@ -0,0 +1,103 @@ +name: 'SDK: Container Checks' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + IMAGE_NAME: prowler + +jobs: + sdk-dockerfile-lint: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check if Dockerfile changed + id: dockerfile-changed + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: Dockerfile + + - name: Lint Dockerfile with Hadolint + if: steps.dockerfile-changed.outputs.any_changed == 'true' + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: Dockerfile + ignore: DL3013 + + sdk-container-build-and-scan: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for SDK changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ./** + files_ignore: | + .github/** + prowler/CHANGELOG.md + docs/** + permissions/** + api/** + ui/** + dashboard/** + mcp_server/** + README.md + mkdocs.yml + .backportrc.json + .env + docker-compose* + examples/** + .gitignore + contrib/** + + - name: Set up Docker Buildx + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build SDK container + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Scan SDK container with Trivy + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/trivy-scan + with: + image-name: ${{ env.IMAGE_NAME }} + image-tag: ${{ github.sha }} + fail-on-critical: 'false' + severity: 'CRITICAL' diff --git a/.github/workflows/sdk-pull-request.yml b/.github/workflows/sdk-pull-request.yml deleted file mode 100644 index 3c20e89c9c..0000000000 --- a/.github/workflows/sdk-pull-request.yml +++ /dev/null @@ -1,286 +0,0 @@ -name: SDK - Pull Request - -on: - push: - branches: - - "master" - - "v3" - - "v4.*" - - "v5.*" - pull_request: - branches: - - "master" - - "v3" - - "v4.*" - - "v5.*" -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Test if changes are in not ignored paths - id: are-non-ignored-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: ./** - files_ignore: | - .github/** - docs/** - permissions/** - api/** - ui/** - prowler/CHANGELOG.md - README.md - mkdocs.yml - .backportrc.json - .env - docker-compose* - examples/** - .gitignore - - - name: Install poetry - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - python -m pip install --upgrade pip - pipx install poetry==2.1.1 - - - name: Set up Python ${{ matrix.python-version }} - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ matrix.python-version }} - cache: "poetry" - - - name: Install dependencies - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry install --no-root - poetry run pip list - VERSION=$(curl --silent "https://api.github.com/repos/hadolint/hadolint/releases/latest" | \ - grep '"tag_name":' | \ - sed -E 's/.*"v([^"]+)".*/\1/' \ - ) && curl -L -o /tmp/hadolint "https://github.com/hadolint/hadolint/releases/download/v${VERSION}/hadolint-Linux-x86_64" \ - && chmod +x /tmp/hadolint - - - name: Poetry check - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry check --lock - - - name: Lint with flake8 - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run flake8 . --ignore=E266,W503,E203,E501,W605,E128 --exclude contrib,ui,api - - - name: Checking format with black - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run black --exclude api ui --check . - - - name: Lint with pylint - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run pylint --disable=W,C,R,E -j 0 -rn -sn prowler/ - - - name: Bandit - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run bandit -q -lll -x '*_test.py,./contrib/,./api/,./ui' -r . - - - name: Safety - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run safety check --ignore 70612 -r pyproject.toml - - - name: Vulture - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run vulture --exclude "contrib,api,ui" --min-confidence 100 . - - - name: Dockerfile - Check if Dockerfile has changed - id: dockerfile-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - Dockerfile - - - name: Hadolint - if: steps.dockerfile-changed-files.outputs.any_changed == 'true' - run: | - /tmp/hadolint Dockerfile --ignore=DL3013 - - # Test AWS - - name: AWS - Check if any file has changed - id: aws-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/aws/** - ./tests/providers/aws/** - ./poetry.lock - - - name: AWS - Test - if: steps.aws-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws - - # Test Azure - - name: Azure - Check if any file has changed - id: azure-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/azure/** - ./tests/providers/azure/** - ./poetry.lock - - - name: Azure - Test - if: steps.azure-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/azure --cov-report=xml:azure_coverage.xml tests/providers/azure - - # Test GCP - - name: GCP - Check if any file has changed - id: gcp-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/gcp/** - ./tests/providers/gcp/** - ./poetry.lock - - - name: GCP - Test - if: steps.gcp-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/gcp --cov-report=xml:gcp_coverage.xml tests/providers/gcp - - # Test Kubernetes - - name: Kubernetes - Check if any file has changed - id: kubernetes-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/kubernetes/** - ./tests/providers/kubernetes/** - ./poetry.lock - - - name: Kubernetes - Test - if: steps.kubernetes-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/kubernetes --cov-report=xml:kubernetes_coverage.xml tests/providers/kubernetes - - # Test GitHub - - name: GitHub - Check if any file has changed - id: github-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/github/** - ./tests/providers/github/** - ./poetry.lock - - - name: GitHub - Test - if: steps.github-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/github --cov-report=xml:github_coverage.xml tests/providers/github - - # Test NHN - - name: NHN - Check if any file has changed - id: nhn-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/nhn/** - ./tests/providers/nhn/** - ./poetry.lock - - - name: NHN - Test - if: steps.nhn-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/nhn --cov-report=xml:nhn_coverage.xml tests/providers/nhn - - # Test M365 - - name: M365 - Check if any file has changed - id: m365-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/m365/** - ./tests/providers/m365/** - ./poetry.lock - - - name: M365 - Test - if: steps.m365-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/m365 --cov-report=xml:m365_coverage.xml tests/providers/m365 - - # Test IaC - - name: IaC - Check if any file has changed - id: iac-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/iac/** - ./tests/providers/iac/** - ./poetry.lock - - - name: IaC - Test - if: steps.iac-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/iac --cov-report=xml:iac_coverage.xml tests/providers/iac - - # Test MongoDB Atlas - - name: MongoDB Atlas - Check if any file has changed - id: mongodb-atlas-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/mongodbatlas/** - ./tests/providers/mongodbatlas/** - .poetry.lock - - - name: MongoDB Atlas - Test - if: steps.mongodb-atlas-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/mongodbatlas --cov-report=xml:mongodb_atlas_coverage.xml tests/providers/mongodbatlas - - # Test OCI - - name: OCI - Check if any file has changed - id: oci-changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ./prowler/providers/oraclecloud/** - ./tests/providers/oraclecloud/** - ./poetry.lock - - - name: OCI - Test - if: steps.oci-changed-files.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/providers/oraclecloud --cov-report=xml:oci_coverage.xml tests/providers/oraclecloud - - # Common Tests - - name: Lib - Test - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/lib --cov-report=xml:lib_coverage.xml tests/lib - - - name: Config - Test - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - run: | - poetry run pytest -n auto --cov=./prowler/config --cov-report=xml:config_coverage.xml tests/config - - # Codecov - - name: Upload coverage reports to Codecov - if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - flags: prowler - files: ./aws_coverage.xml,./azure_coverage.xml,./gcp_coverage.xml,./kubernetes_coverage.xml,./github_coverage.xml,./nhn_coverage.xml,./m365_coverage.xml,./oci_coverage.xml,./lib_coverage.xml,./config_coverage.xml diff --git a/.github/workflows/sdk-pypi-release.yml b/.github/workflows/sdk-pypi-release.yml index 59cc948340..0f74ba054c 100644 --- a/.github/workflows/sdk-pypi-release.yml +++ b/.github/workflows/sdk-pypi-release.yml @@ -1,98 +1,119 @@ -name: SDK - PyPI release +name: 'SDK: PyPI Release' on: release: - types: [published] + types: + - 'published' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false env: RELEASE_TAG: ${{ github.event.release.tag_name }} - PYTHON_VERSION: 3.11 - # CACHE: "poetry" + PYTHON_VERSION: '3.12' jobs: - repository-check: - name: Repository check + validate-release: + if: github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read outputs: - is_repo: ${{ steps.repository_check.outputs.is_repo }} - steps: - - name: Repository check - id: repository_check - working-directory: /tmp - run: | - if [[ ${{ github.repository }} == "prowler-cloud/prowler" ]] - then - echo "is_repo=true" >> "${GITHUB_OUTPUT}" - else - echo "This action only runs for prowler-cloud/prowler" - echo "is_repo=false" >> "${GITHUB_OUTPUT}" - fi + prowler_version: ${{ steps.parse-version.outputs.version }} + major_version: ${{ steps.parse-version.outputs.major }} - release-prowler-job: - runs-on: ubuntu-latest - needs: repository-check - if: needs.repository-check.outputs.is_repo == 'true' - env: - POETRY_VIRTUALENVS_CREATE: "false" - name: Release Prowler to PyPI steps: - - name: Repository check - working-directory: /tmp - run: | - if [[ "${{ github.repository }}" != "prowler-cloud/prowler" ]]; then - echo "This action only runs for prowler-cloud/prowler" - exit 1 - fi - - - name: Get Prowler version + - name: Parse and validate version + id: parse-version run: | PROWLER_VERSION="${{ env.RELEASE_TAG }}" + echo "version=${PROWLER_VERSION}" >> "${GITHUB_OUTPUT}" - case ${PROWLER_VERSION%%.*} in - 3) - echo "Releasing Prowler v3 with tag ${PROWLER_VERSION}" + # Extract major version + MAJOR_VERSION="${PROWLER_VERSION%%.*}" + echo "major=${MAJOR_VERSION}" >> "${GITHUB_OUTPUT}" + + # Validate major version + case ${MAJOR_VERSION} in + 3|4|5) + echo "✓ Releasing Prowler v${MAJOR_VERSION} with tag ${PROWLER_VERSION}" ;; - 4) - echo "Releasing Prowler v4 with tag ${PROWLER_VERSION}" - ;; - 5) - echo "Releasing Prowler v5 with tag ${PROWLER_VERSION}" - ;; - *) - echo "Releasing another Prowler major version, aborting..." + *) + echo "::error::Unsupported Prowler major version: ${MAJOR_VERSION}" exit 1 ;; esac - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + publish-prowler: + needs: validate-release + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + environment: + name: pypi-prowler + url: https://pypi.org/project/prowler/${{ needs.validate-release.outputs.prowler_version }}/ - - name: Install dependencies - run: | - pipx install poetry==2.1.1 + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Setup Python + - name: Install Poetry + run: pipx install poetry==2.1.1 + + - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - # cache: ${{ env.CACHE }} + cache: 'poetry' - name: Build Prowler package - run: | - poetry build + run: poetry build - name: Publish Prowler package to PyPI - run: | - poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }} - poetry publish + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + print-hash: true - - name: Replicate PyPI package + publish-prowler-cloud: + needs: validate-release + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + environment: + name: pypi-prowler-cloud + url: https://pypi.org/project/prowler-cloud/${{ needs.validate-release.outputs.prowler_version }}/ + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Install Poetry + run: pipx install poetry==2.1.1 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'poetry' + + - name: Install toml package + run: pip install toml + + - name: Replicate PyPI package for prowler-cloud run: | - rm -rf ./dist && rm -rf ./build && rm -rf prowler.egg-info - pip install toml + rm -rf ./dist ./build prowler.egg-info python util/replicate_pypi_package.py - poetry build + + - name: Build prowler-cloud package + run: poetry build - name: Publish prowler-cloud package to PyPI - run: | - poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }} - poetry publish + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + print-hash: true diff --git a/.github/workflows/sdk-refresh-aws-services-regions.yml b/.github/workflows/sdk-refresh-aws-services-regions.yml index ded41cdc02..884b677294 100644 --- a/.github/workflows/sdk-refresh-aws-services-regions.yml +++ b/.github/workflows/sdk-refresh-aws-services-regions.yml @@ -1,68 +1,90 @@ -# This is a basic workflow to help you get started with Actions - -name: SDK - Refresh AWS services' regions +name: 'SDK: Refresh AWS Regions' on: schedule: - - cron: "0 9 * * 1" # runs at 09:00 UTC every Monday + - cron: '0 9 * * 1' # Every Monday at 09:00 UTC + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false env: - GITHUB_BRANCH: "master" - AWS_REGION_DEV: us-east-1 + PYTHON_VERSION: '3.12' + AWS_REGION: 'us-east-1' -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on + refresh-aws-regions: + if: github.repository == 'prowler-cloud/prowler' runs-on: ubuntu-latest + timeout-minutes: 15 permissions: id-token: write pull-requests: write contents: write - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ env.GITHUB_BRANCH }} - - name: setup python + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: 'master' + + - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - python-version: 3.9 #install the python needed + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install boto3 + run: pip install boto3 - - name: Configure AWS Credentials -- DEV - uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0 with: - aws-region: ${{ env.AWS_REGION_DEV }} + aws-region: ${{ env.AWS_REGION }} role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }} - role-session-name: refresh-AWS-regions-dev + role-session-name: prowler-refresh-aws-regions - # Runs a single command using the runners shell - - name: Run a one-line script - run: python3 util/update_aws_services_regions.py + - name: Update AWS services regions + run: python util/update_aws_services_regions.py - # Create pull request - - name: Create Pull Request + - name: Create pull request + id: create-pr uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: - author: prowler-bot <179230569+prowler-bot@users.noreply.github.com> token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - commit-message: "feat(regions_update): Update regions for AWS services" - branch: "aws-services-regions-updated-${{ github.sha }}" - labels: "status/waiting-for-revision, severity/low, provider/aws, no-changelog" - title: "chore(regions_update): Changes in regions for AWS services" + author: 'prowler-bot <179230569+prowler-bot@users.noreply.github.com>' + committer: 'github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>' + commit-message: 'feat(aws): update regions for AWS services' + branch: 'aws-regions-update-${{ github.run_number }}' + title: 'feat(aws): Update regions for AWS services' + labels: | + status/waiting-for-revision + severity/low + provider/aws + no-changelog body: | ### Description - This PR updates the regions for AWS services. + Automated update of AWS service regions from the official AWS IP ranges. + + **Trigger:** ${{ github.event_name == 'schedule' && 'Scheduled (weekly)' || github.event_name == 'workflow_dispatch' && 'Manual' || 'Workflow update' }} + **Run:** [#${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + + ### Checklist + + - [x] This is an automated update from AWS official sources + - [x] No manual review of region data required ### License By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. + + - name: PR creation result + run: | + if [[ "${{ steps.create-pr.outputs.pull-request-number }}" ]]; then + echo "✓ Pull request #${{ steps.create-pr.outputs.pull-request-number }} created successfully" + echo "URL: ${{ steps.create-pr.outputs.pull-request-url }}" + else + echo "✓ No changes detected - AWS regions are up to date" + fi diff --git a/.github/workflows/sdk-security.yml b/.github/workflows/sdk-security.yml new file mode 100644 index 0000000000..a15bd2539f --- /dev/null +++ b/.github/workflows/sdk-security.yml @@ -0,0 +1,77 @@ +name: 'SDK: Security' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + sdk-security-scans: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for SDK changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ./** + files_ignore: | + .github/** + prowler/CHANGELOG.md + docs/** + permissions/** + api/** + ui/** + dashboard/** + mcp_server/** + README.md + mkdocs.yml + .backportrc.json + .env + docker-compose* + examples/** + .gitignore + contrib/** + + - name: Install Poetry + if: steps.check-changes.outputs.any_changed == 'true' + run: pipx install poetry==2.1.1 + + - name: Set up Python 3.12 + if: steps.check-changes.outputs.any_changed == 'true' + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.12' + cache: 'poetry' + + - name: Install dependencies + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry install --no-root + + - name: Security scan with Bandit + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run bandit -q -lll -x '*_test.py,./contrib/,./api/,./ui' -r . + + - name: Security scan with Safety + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run safety check --ignore 70612 -r pyproject.toml + + - name: Dead code detection with Vulture + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run vulture --exclude "contrib,api,ui" --min-confidence 100 . diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml new file mode 100644 index 0000000000..b01c5e7778 --- /dev/null +++ b/.github/workflows/sdk-tests.yml @@ -0,0 +1,360 @@ +name: 'SDK: Tests' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + sdk-tests: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.9' + - '3.10' + - '3.11' + - '3.12' + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for SDK changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ./** + files_ignore: | + .github/** + prowler/CHANGELOG.md + docs/** + permissions/** + api/** + ui/** + dashboard/** + mcp_server/** + README.md + mkdocs.yml + .backportrc.json + .env + docker-compose* + examples/** + .gitignore + contrib/** + + - name: Install Poetry + if: steps.check-changes.outputs.any_changed == 'true' + run: pipx install poetry==2.1.1 + + - name: Set up Python ${{ matrix.python-version }} + if: steps.check-changes.outputs.any_changed == 'true' + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + + - name: Install dependencies + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry install --no-root + + # AWS Provider + - name: Check if AWS files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-aws + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/aws/** + ./tests/**/aws/** + ./poetry.lock + + - name: Run AWS tests + if: steps.changed-aws.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/aws --cov-report=xml:aws_coverage.xml tests/providers/aws + + - name: Upload AWS coverage to Codecov + if: steps.changed-aws.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-aws + files: ./aws_coverage.xml + + # Azure Provider + - name: Check if Azure files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-azure + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/azure/** + ./tests/**/azure/** + ./poetry.lock + + - name: Run Azure tests + if: steps.changed-azure.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/azure --cov-report=xml:azure_coverage.xml tests/providers/azure + + - name: Upload Azure coverage to Codecov + if: steps.changed-azure.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-azure + files: ./azure_coverage.xml + + # GCP Provider + - name: Check if GCP files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-gcp + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/gcp/** + ./tests/**/gcp/** + ./poetry.lock + + - name: Run GCP tests + if: steps.changed-gcp.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/gcp --cov-report=xml:gcp_coverage.xml tests/providers/gcp + + - name: Upload GCP coverage to Codecov + if: steps.changed-gcp.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-gcp + files: ./gcp_coverage.xml + + # Kubernetes Provider + - name: Check if Kubernetes files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-kubernetes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/kubernetes/** + ./tests/**/kubernetes/** + ./poetry.lock + + - name: Run Kubernetes tests + if: steps.changed-kubernetes.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/kubernetes --cov-report=xml:kubernetes_coverage.xml tests/providers/kubernetes + + - name: Upload Kubernetes coverage to Codecov + if: steps.changed-kubernetes.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-kubernetes + files: ./kubernetes_coverage.xml + + # GitHub Provider + - name: Check if GitHub files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-github + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/github/** + ./tests/**/github/** + ./poetry.lock + + - name: Run GitHub tests + if: steps.changed-github.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/github --cov-report=xml:github_coverage.xml tests/providers/github + + - name: Upload GitHub coverage to Codecov + if: steps.changed-github.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-github + files: ./github_coverage.xml + + # NHN Provider + - name: Check if NHN files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-nhn + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/nhn/** + ./tests/**/nhn/** + ./poetry.lock + + - name: Run NHN tests + if: steps.changed-nhn.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/nhn --cov-report=xml:nhn_coverage.xml tests/providers/nhn + + - name: Upload NHN coverage to Codecov + if: steps.changed-nhn.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-nhn + files: ./nhn_coverage.xml + + # M365 Provider + - name: Check if M365 files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-m365 + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/m365/** + ./tests/**/m365/** + ./poetry.lock + + - name: Run M365 tests + if: steps.changed-m365.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/m365 --cov-report=xml:m365_coverage.xml tests/providers/m365 + + - name: Upload M365 coverage to Codecov + if: steps.changed-m365.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-m365 + files: ./m365_coverage.xml + + # IaC Provider + - name: Check if IaC files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-iac + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/iac/** + ./tests/**/iac/** + ./poetry.lock + + - name: Run IaC tests + if: steps.changed-iac.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/iac --cov-report=xml:iac_coverage.xml tests/providers/iac + + - name: Upload IaC coverage to Codecov + if: steps.changed-iac.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-iac + files: ./iac_coverage.xml + + # MongoDB Atlas Provider + - name: Check if MongoDB Atlas files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-mongodbatlas + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/mongodbatlas/** + ./tests/**/mongodbatlas/** + ./poetry.lock + + - name: Run MongoDB Atlas tests + if: steps.changed-mongodbatlas.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/mongodbatlas --cov-report=xml:mongodbatlas_coverage.xml tests/providers/mongodbatlas + + - name: Upload MongoDB Atlas coverage to Codecov + if: steps.changed-mongodbatlas.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-mongodbatlas + files: ./mongodbatlas_coverage.xml + + # OCI Provider + - name: Check if OCI files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-oraclecloud + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/**/oraclecloud/** + ./tests/**/oraclecloud/** + ./poetry.lock + + - name: Run OCI tests + if: steps.changed-oraclecloud.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/providers/oraclecloud --cov-report=xml:oraclecloud_coverage.xml tests/providers/oraclecloud + + - name: Upload OCI coverage to Codecov + if: steps.changed-oraclecloud.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-oraclecloud + files: ./oraclecloud_coverage.xml + + # Lib + - name: Check if Lib files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-lib + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/lib/** + ./tests/lib/** + ./poetry.lock + + - name: Run Lib tests + if: steps.changed-lib.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/lib --cov-report=xml:lib_coverage.xml tests/lib + + - name: Upload Lib coverage to Codecov + if: steps.changed-lib.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-lib + files: ./lib_coverage.xml + + # Config + - name: Check if Config files changed + if: steps.check-changes.outputs.any_changed == 'true' + id: changed-config + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ./prowler/config/** + ./tests/config/** + ./poetry.lock + + - name: Run Config tests + if: steps.changed-config.outputs.any_changed == 'true' + run: poetry run pytest -n auto --cov=./prowler/config --cov-report=xml:config_coverage.xml tests/config + + - name: Upload Config coverage to Codecov + if: steps.changed-config.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: prowler-py${{ matrix.python-version }}-config + files: ./config_coverage.xml diff --git a/.github/workflows/ui-build-lint-push-containers.yml b/.github/workflows/ui-build-lint-push-containers.yml deleted file mode 100644 index 16d9d80fbb..0000000000 --- a/.github/workflows/ui-build-lint-push-containers.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: UI - Build and Push containers - -on: - push: - branches: - - "master" - paths: - - "ui/**" - - ".github/workflows/ui-build-lint-push-containers.yml" - - # Uncomment the below code to test this action on PRs - # pull_request: - # branches: - # - "master" - # paths: - # - "ui/**" - # - ".github/workflows/ui-build-lint-push-containers.yml" - - release: - types: [published] - -env: - # Tags - LATEST_TAG: latest - RELEASE_TAG: ${{ github.event.release.tag_name }} - STABLE_TAG: stable - - WORKING_DIRECTORY: ./ui - - # Container Registries - PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud - PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-ui - NEXT_PUBLIC_API_BASE_URL: http://prowler-api:8080/api/v1 - -jobs: - repository-check: - name: Repository check - runs-on: ubuntu-latest - outputs: - is_repo: ${{ steps.repository_check.outputs.is_repo }} - steps: - - name: Repository check - id: repository_check - working-directory: /tmp - run: | - if [[ ${{ github.repository }} == "prowler-cloud/prowler" ]] - then - echo "is_repo=true" >> "${GITHUB_OUTPUT}" - else - echo "This action only runs for prowler-cloud/prowler" - echo "is_repo=false" >> "${GITHUB_OUTPUT}" - fi - - # Build Prowler OSS container - container-build-push: - needs: repository-check - if: needs.repository-check.outputs.is_repo == 'true' - runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ env.WORKING_DIRECTORY }} - - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Set short git commit SHA - id: vars - run: | - shortSha=$(git rev-parse --short ${{ github.sha }}) - echo "SHORT_SHA=${shortSha}" >> $GITHUB_ENV - - - name: Login to DockerHub - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Build and push container image (latest) - # Comment the following line for testing - if: github.event_name == 'push' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.WORKING_DIRECTORY }} - build-args: | - NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ env.SHORT_SHA }} - NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }} - # Set push: false for testing - push: true - tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.SHORT_SHA }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push container image (release) - if: github.event_name == 'release' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.WORKING_DIRECTORY }} - build-args: | - NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v${{ env.RELEASE_TAG }} - NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }} - push: true - tags: | - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} - ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Trigger deployment - if: github.event_name == 'push' - uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 - with: - token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} - repository: ${{ secrets.CLOUD_DISPATCH }} - event-type: prowler-ui-deploy - client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ env.SHORT_SHA }}"}' diff --git a/.github/workflows/ui-codeql.yml b/.github/workflows/ui-codeql.yml index d51b904959..2b55cf673c 100644 --- a/.github/workflows/ui-codeql.yml +++ b/.github/workflows/ui-codeql.yml @@ -27,7 +27,8 @@ concurrency: cancel-in-progress: true jobs: - analyze: + ui-analyze: + if: github.repository == 'prowler-cloud/prowler' name: CodeQL Security Analysis runs-on: ubuntu-latest timeout-minutes: 30 @@ -47,12 +48,12 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Initialize CodeQL - uses: github/codeql-action/init@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/ui-codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@3599b3baa15b485a2e49ef411a7a4bb2452e7f93 # v3.30.5 + uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v4.31.2 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/ui-container-build-push.yml b/.github/workflows/ui-container-build-push.yml new file mode 100644 index 0000000000..34cddc900d --- /dev/null +++ b/.github/workflows/ui-container-build-push.yml @@ -0,0 +1,196 @@ +name: 'UI: Container Build and Push' + +on: + push: + branches: + - 'master' + paths: + - 'ui/**' + - '.github/workflows/ui-container-build-push.yml' + release: + types: + - 'published' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + # Tags + LATEST_TAG: latest + RELEASE_TAG: ${{ github.event.release.tag_name }} + STABLE_TAG: stable + WORKING_DIRECTORY: ./ui + + # Container registries + PROWLERCLOUD_DOCKERHUB_REPOSITORY: prowlercloud + PROWLERCLOUD_DOCKERHUB_IMAGE: prowler-ui + + # Build args + NEXT_PUBLIC_API_BASE_URL: http://prowler-api:8080/api/v1 + +jobs: + setup: + if: github.repository == 'prowler-cloud/prowler' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + short-sha: ${{ steps.set-short-sha.outputs.short-sha }} + steps: + - name: Calculate short SHA + id: set-short-sha + run: echo "short-sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + container-build-push: + needs: setup + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 + timeout-minutes: 30 + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Login to DockerHub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build and push UI container for ${{ matrix.arch }} + if: github.event_name == 'push' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.WORKING_DIRECTORY }} + build-args: | + NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ needs.setup.outputs.short-sha }} + NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }} + push: true + platforms: ${{ matrix.platform }} + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-${{ matrix.arch }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + + - name: Build and push UI container for ${{ matrix.arch }} + if: github.event_name == 'push' || github.event_name == 'release' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.WORKING_DIRECTORY }} + build-args: | + NEXT_PUBLIC_PROWLER_RELEASE_VERSION=${{ github.event_name == 'release' && format('v{0}', env.RELEASE_TAG) || needs.setup.outputs.short_sha }} + NEXT_PUBLIC_API_BASE_URL=${{ env.NEXT_PUBLIC_API_BASE_URL }} + push: true + tags: | + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Notify container push started + if: github.event_name == 'release' + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: UI + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-started.json" + + - name: Notify container push completed + if: github.event_name == 'release' && always() + uses: ./.github/actions/slack-notification + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_PLATFORM_DEPLOYMENTS }} + COMPONENT: UI + RELEASE_TAG: ${{ env.RELEASE_TAG }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + with: + slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json" + step-outcome: ${{ steps.container-push.outcome }} + + # Create and push multi-architecture manifest + create-manifest: + needs: [setup, container-build-push] + if: github.event_name == 'push' || github.event_name == 'release' + runs-on: ubuntu-latest + + steps: + - name: Login to DockerHub + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Create and push manifests for push event + if: github.event_name == 'push' + run: | + docker buildx imagetools create \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.LATEST_TAG }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64 + + - name: Create and push manifests for release event + if: github.event_name == 'release' + run: | + docker buildx imagetools create \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.RELEASE_TAG }} \ + -t ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ env.STABLE_TAG }} \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64 \ + ${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64 + + - name: Install regctl + if: always() + uses: regclient/actions/regctl-installer@main + + - name: Cleanup intermediate architecture tags + if: always() + run: | + echo "Cleaning up intermediate tags..." + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-amd64" || true + regctl tag delete "${{ env.PROWLERCLOUD_DOCKERHUB_REPOSITORY }}/${{ env.PROWLERCLOUD_DOCKERHUB_IMAGE }}:${{ needs.setup.outputs.short-sha }}-arm64" || true + echo "Cleanup completed" + + trigger-deployment: + if: github.event_name == 'push' + needs: [setup, container-build-push] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Trigger UI deployment + uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4.0.0 + with: + token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }} + repository: ${{ secrets.CLOUD_DISPATCH }} + event-type: ui-prowler-deployment + client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}' diff --git a/.github/workflows/ui-container-checks.yml b/.github/workflows/ui-container-checks.yml new file mode 100644 index 0000000000..e436c3974a --- /dev/null +++ b/.github/workflows/ui-container-checks.yml @@ -0,0 +1,91 @@ +name: 'UI: Container Checks' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + UI_WORKING_DIR: ./ui + IMAGE_NAME: prowler-ui + +jobs: + ui-dockerfile-lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check if Dockerfile changed + id: dockerfile-changed + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ui/Dockerfile + + - name: Lint Dockerfile with Hadolint + if: steps.dockerfile-changed.outputs.any_changed == 'true' + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: ui/Dockerfile + ignore: DL3018 + + ui-container-build-and-scan: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for UI changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: ui/** + files_ignore: | + ui/CHANGELOG.md + ui/README.md + + - name: Set up Docker Buildx + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build UI container + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.UI_WORKING_DIR }} + target: prod + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX + + - name: Scan UI container with Trivy + if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/trivy-scan + with: + image-name: ${{ env.IMAGE_NAME }} + image-tag: ${{ github.sha }} + fail-on-critical: 'false' + severity: 'CRITICAL' diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index 87be96808d..790a4a6825 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -95,7 +95,7 @@ jobs: echo "All database fixtures loaded successfully!" ' - name: Setup Node.js environment - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 with: node-version: '20.x' cache: 'npm' diff --git a/.github/workflows/ui-pull-request.yml b/.github/workflows/ui-pull-request.yml deleted file mode 100644 index 22af5073a7..0000000000 --- a/.github/workflows/ui-pull-request.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: UI - Pull Request - -on: - push: - branches: - - "master" - - "v5.*" - paths: - - ".github/workflows/ui-pull-request.yml" - - "ui/**" - pull_request: - branches: - - master - - "v5.*" - paths: - - 'ui/**' -env: - UI_WORKING_DIR: ./ui - IMAGE_NAME: prowler-ui - -jobs: - test-and-coverage: - runs-on: ubuntu-latest - strategy: - matrix: - os: [ubuntu-latest] - node-version: [20.x] - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - - name: Setup Node.js ${{ matrix.node-version }} - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 - with: - node-version: ${{ matrix.node-version }} - cache: 'npm' - cache-dependency-path: './ui/package-lock.json' - - name: Install dependencies - working-directory: ./ui - run: npm ci - - name: Run Healthcheck - working-directory: ./ui - run: npm run healthcheck - - name: Build the application - working-directory: ./ui - run: npm run build - - test-container-build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - name: Build Container - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.UI_WORKING_DIR }} - # Always build using `prod` target - target: prod - push: false - tags: ${{ env.IMAGE_NAME }}:latest - outputs: type=docker - build-args: | - NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51LwpXXXX diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml new file mode 100644 index 0000000000..a5a5b5fa17 --- /dev/null +++ b/.github/workflows/ui-tests.yml @@ -0,0 +1,64 @@ +name: 'UI: Tests' + +on: + push: + branches: + - 'master' + - 'v5.*' + pull_request: + branches: + - 'master' + - 'v5.*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + UI_WORKING_DIR: ./ui + NODE_VERSION: '20.x' + +jobs: + ui-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + defaults: + run: + working-directory: ./ui + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for UI changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: | + ui/** + .github/workflows/ui-tests.yml + files_ignore: | + ui/CHANGELOG.md + ui/README.md + + - name: Setup Node.js ${{ env.NODE_VERSION }} + if: steps.check-changes.outputs.any_changed == 'true' + uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: './ui/package-lock.json' + + - name: Install dependencies + if: steps.check-changes.outputs.any_changed == 'true' + run: npm ci + + - name: Run healthcheck + if: steps.check-changes.outputs.any_changed == 'true' + run: npm run healthcheck + + - name: Build application + if: steps.check-changes.outputs.any_changed == 'true' + run: npm run build diff --git a/.gitignore b/.gitignore index bc8c66d39b..7792a031b4 100644 --- a/.gitignore +++ b/.gitignore @@ -45,21 +45,86 @@ pytest_*.xml .coverage htmlcov/ -# VSCode files +# VSCode files and settings .vscode/ +*.code-workspace +.vscode-test/ -# Cursor files +# VSCode extension settings and workspaces +.history/ +.ionide/ + +# MCP Server Settings (various locations) +**/cline_mcp_settings.json +**/mcp_settings.json +**/mcp-config.json +**/mcpServers.json +.mcp/ + +# AI Coding Assistants - Cursor .cursorignore .cursor/ +.cursorrules -# RooCode files +# AI Coding Assistants - RooCode .roo/ .rooignore .roomodes -# Cline files +# AI Coding Assistants - Cline (formerly Claude Dev) .cline/ .clineignore +.clinerules + +# AI Coding Assistants - Continue +.continue/ +continue.json +.continuerc +.continuerc.json + +# AI Coding Assistants - GitHub Copilot +.copilot/ +.github/copilot/ + +# AI Coding Assistants - Amazon Q Developer (formerly CodeWhisperer) +.aws/ +.codewhisperer/ +.amazonq/ +.aws-toolkit/ + +# AI Coding Assistants - Tabnine +.tabnine/ +tabnine_config.json + +# AI Coding Assistants - Kiro +.kiro/ +.kiroignore +kiro.config.json + +# AI Coding Assistants - Aider +.aider/ +.aider.chat.history.md +.aider.input.history +.aider.tags.cache.v3/ + +# AI Coding Assistants - Windsurf +.windsurf/ +.windsurfignore + +# AI Coding Assistants - Replit Agent +.replit +.replitignore + +# AI Coding Assistants - Supermaven +.supermaven/ + +# AI Coding Assistants - Sourcegraph Cody +.cody/ + +# AI Coding Assistants - General +.ai/ +.aiconfig +ai-config.json # Terraform .terraform* @@ -70,7 +135,6 @@ htmlcov/ ui/.env* api/.env* mcp_server/.env* -.env.local # Coverage .coverage* diff --git a/Dockerfile b/Dockerfile index 5884bda9f3..f71c53e008 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,10 @@ LABEL maintainer="https://github.com/prowler-cloud/prowler" LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler" ARG POWERSHELL_VERSION=7.5.0 +ENV POWERSHELL_VERSION=${POWERSHELL_VERSION} + +ARG TRIVY_VERSION=0.66.0 +ENV TRIVY_VERSION=${TRIVY_VERSION} # hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -25,6 +29,24 @@ RUN ARCH=$(uname -m) && \ ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh && \ rm /tmp/powershell.tar.gz +# Install Trivy for IaC scanning +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + TRIVY_ARCH="Linux-64bit" ; \ + elif [ "$ARCH" = "aarch64" ]; then \ + TRIVY_ARCH="Linux-ARM64" ; \ + else \ + echo "Unsupported architecture for Trivy: $ARCH" && exit 1 ; \ + fi && \ + wget --progress=dot:giga "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_${TRIVY_ARCH}.tar.gz" -O /tmp/trivy.tar.gz && \ + tar zxf /tmp/trivy.tar.gz -C /tmp && \ + mv /tmp/trivy /usr/local/bin/trivy && \ + chmod +x /usr/local/bin/trivy && \ + rm /tmp/trivy.tar.gz && \ + # Create trivy cache directory with proper permissions + mkdir -p /tmp/.cache/trivy && \ + chmod 777 /tmp/.cache/trivy + # Add prowler user RUN addgroup --gid 1000 prowler && \ adduser --uid 1000 --gid 1000 --disabled-password --gecos "" prowler diff --git a/README.md b/README.md index f1097816c6..25cc8f6abe 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Prowler includes hundreds of built-in controls to ensure compliance with standar Prowler App is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments. -![Prowler App](docs/products/img/overview.png) +![Prowler App](docs/images/products/overview.png) >For more details, refer to the [Prowler App Documentation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-app-installation) @@ -73,26 +73,26 @@ prowler ```console prowler dashboard ``` -![Prowler Dashboard](docs/products/img/dashboard.png) +![Prowler Dashboard](docs/images/products/dashboard.png) # Prowler at a Glance > [!Tip] > For the most accurate and up-to-date information about checks, services, frameworks, and categories, visit [**Prowler Hub**](https://hub.prowler.com). -| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Stage | Interface | -|---|---|---|---|---|---|---|---| -| AWS | 576 | 82 | 38 | 10 | Official | Stable | UI, API, CLI | -| GCP | 79 | 13 | 11 | 3 | Official | Stable | UI, API, CLI | -| Azure | 162 | 19 | 12 | 4 | Official | Stable | UI, API, CLI | -| Kubernetes | 83 | 7 | 5 | 7 | Official | Stable | UI, API, CLI | +| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface | +|---|---|---|---|---|---|---| +| AWS | 576 | 82 | 39 | 10 | Official | UI, API, CLI | +| GCP | 79 | 13 | 13 | 3 | Official | UI, API, CLI | +| Azure | 162 | 19 | 13 | 4 | Official | UI, API, CLI | +| Kubernetes | 83 | 7 | 5 | 7 | Official | UI, API, CLI | | GitHub | 17 | 2 | 1 | 0 | Official | Stable | UI, API, CLI | -| M365 | 70 | 7 | 3 | 2 | Official | Stable | UI, API, CLI | -| OCI | 51 | 13 | 1 | 10 | Official | Stable | CLI | -| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | Beta | CLI | -| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | Beta | CLI | -| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | Beta | CLI | -| NHN | 6 | 2 | 1 | 0 | Unofficial | Beta | CLI | +| M365 | 70 | 7 | 3 | 2 | Official | UI, API, CLI | +| OCI | 51 | 13 | 1 | 10 | Official | UI, API, CLI | +| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI | +| MongoDB Atlas | 10 | 3 | 0 | 0 | Official | CLI, API | +| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI | +| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI | > [!Note] > The numbers in the table are updated periodically. diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 6c43a2e582..ce5d8eed74 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -5,7 +5,46 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.15.0] (Prowler UNRELEASED) ### Added +- IaC (Infrastructure as Code) provider support for remote repositories [(#8751)](https://github.com/prowler-cloud/prowler/pull/8751) +- Extend `GET /api/v1/providers` with provider-type filters and optional pagination disable to support the new Overview filters [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975) +- New endpoint to retrieve the number of providers grouped by provider type [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975) - Support for configuring multiple LLM providers [(#8772)](https://github.com/prowler-cloud/prowler/pull/8772) +- Support C5 compliance framework for Azure provider [(#9081)](https://github.com/prowler-cloud/prowler/pull/9081) +- Support for Oracle Cloud Infrastructure (OCI) provider [(#8927)](https://github.com/prowler-cloud/prowler/pull/8927) +- Support muting findings based on simple rules with custom reason [(#9051)](https://github.com/prowler-cloud/prowler/pull/9051) +- Support C5 compliance framework for the GCP provider [(#9097)](https://github.com/prowler-cloud/prowler/pull/9097) +- Support for Amazon Bedrock and OpenAI compatible providers in Lighthouse AI [(#8957)](https://github.com/prowler-cloud/prowler/pull/8957) +- Tenant-wide ThreatScore overview aggregation and snapshot persistence with backfill support [(#9148)](https://github.com/prowler-cloud/prowler/pull/9148) +- Added `metadata`, `details`, and `partition` attributes to `/resources` endpoint & `details`, and `partition` to `/findings` endpoint [(#9098)](https://github.com/prowler-cloud/prowler/pull/9098) +- Support for MongoDB Atlas provider [(#9167)](https://github.com/prowler-cloud/prowler/pull/9167) +- Enhanced compliance overview endpoint with provider filtering and latest scan aggregation [(#9244)](https://github.com/prowler-cloud/prowler/pull/9244) + +### Changed +- Optimized database write queries for scan related tasks [(#9190)](https://github.com/prowler-cloud/prowler/pull/9190) +- Date filters are now optional for `GET /api/v1/overviews/services` endpoint; returns latest scan data by default [(#9248)](https://github.com/prowler-cloud/prowler/pull/9248) + +### Fixed +- Scans no longer fail when findings have UIDs exceeding 300 characters; such findings are now skipped with detailed logging [(#9246)](https://github.com/prowler-cloud/prowler/pull/9246) + +### Security +- Django updated to the latest 5.1 security release, 5.1.14, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/113) and [denial-of-service vulnerability](https://github.com/prowler-cloud/prowler/security/dependabot/114) [(#9176)](https://github.com/prowler-cloud/prowler/pull/9176) + +--- + +## [1.14.2] (Prowler 5.13.2) + +### Fixed +- Update unique constraint for `Provider` model to exclude soft-deleted entries, resolving duplicate errors when re-deleting providers [(#9054)](https://github.com/prowler-cloud/prowler/pull/9054) +- Remove compliance generation for providers without compliance frameworks [(#9208)](https://github.com/prowler-cloud/prowler/pull/9208) + +## [1.14.1] (Prowler 5.13.1) + +### Fixed +- `/api/v1/overviews/providers` collapses data by provider type so the UI receives a single aggregated record per cloud family even when multiple accounts exist [(#9053)](https://github.com/prowler-cloud/prowler/pull/9053) +- Added retry logic to database transactions to handle Aurora read replica connection failures during scale-down events [(#9064)](https://github.com/prowler-cloud/prowler/pull/9064) +- Security Hub integrations stop failing when they read relationships via the replica by allowing replica relations and saving updates through the primary [(#9080)](https://github.com/prowler-cloud/prowler/pull/9080) + +--- ## [1.14.0] (Prowler 5.13.0) diff --git a/api/Dockerfile b/api/Dockerfile index a3cfb21782..2d7883a957 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,6 +5,9 @@ LABEL maintainer="https://github.com/prowler-cloud/api" ARG POWERSHELL_VERSION=7.5.0 ENV POWERSHELL_VERSION=${POWERSHELL_VERSION} +ARG TRIVY_VERSION=0.66.0 +ENV TRIVY_VERSION=${TRIVY_VERSION} + # hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ wget \ @@ -36,6 +39,24 @@ RUN ARCH=$(uname -m) && \ ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh && \ rm /tmp/powershell.tar.gz +# Install Trivy for IaC scanning +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + TRIVY_ARCH="Linux-64bit" ; \ + elif [ "$ARCH" = "aarch64" ]; then \ + TRIVY_ARCH="Linux-ARM64" ; \ + else \ + echo "Unsupported architecture for Trivy: $ARCH" && exit 1 ; \ + fi && \ + wget --progress=dot:giga "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_${TRIVY_ARCH}.tar.gz" -O /tmp/trivy.tar.gz && \ + tar zxf /tmp/trivy.tar.gz -C /tmp && \ + mv /tmp/trivy /usr/local/bin/trivy && \ + chmod +x /usr/local/bin/trivy && \ + rm /tmp/trivy.tar.gz && \ + # Create trivy cache directory with proper permissions + mkdir -p /tmp/.cache/trivy && \ + chmod 777 /tmp/.cache/trivy + # Add prowler user RUN addgroup --gid 1000 prowler && \ adduser --uid 1000 --gid 1000 --disabled-password --gecos "" prowler diff --git a/api/poetry.lock b/api/poetry.lock index 40313d9fa5..332d4d9f73 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "about-time" @@ -1671,14 +1671,14 @@ with-social = ["django-allauth[socialaccount] (>=64.0.0)"] [[package]] name = "django" -version = "5.1.13" +version = "5.1.14" description = "A high-level Python web framework that encourages rapid development and clean, pragmatic design." optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "django-5.1.13-py3-none-any.whl", hash = "sha256:06f257f79dc4c17f3f9e23b106a4c5ed1335abecbe731e83c598c941d14fbeed"}, - {file = "django-5.1.13.tar.gz", hash = "sha256:543ff21679f15e80edfc01fe7ea35f8291b6d4ea589433882913626a7c1cf929"}, + {file = "django-5.1.14-py3-none-any.whl", hash = "sha256:2a4b9c20404fd1bf50aaaa5542a19d860594cba1354f688f642feb271b91df27"}, + {file = "django-5.1.14.tar.gz", hash = "sha256:b98409fb31fdd6e8c3a6ba2eef3415cc5c0020057b43b21ba7af6eff5f014831"}, ] [package.dependencies] @@ -6786,4 +6786,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "3c9164d668d37d6373eb5200bbe768232ead934d9312b9c68046b1df922789f3" +content-hash = "943e2cd6b87229704550d4e140b36509fb9f58896ebb5834b9fbabe28a9ee92f" diff --git a/api/pyproject.toml b/api/pyproject.toml index 22ae14e6a1..ffc8413489 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -7,7 +7,7 @@ authors = [{name = "Prowler Engineering", email = "engineering@prowler.com"}] dependencies = [ "celery[pytest] (>=5.4.0,<6.0.0)", "dj-rest-auth[with_social,jwt] (==7.0.1)", - "django (==5.1.13)", + "django (==5.1.14)", "django-allauth[saml] (>=65.8.0,<66.0.0)", "django-celery-beat (>=2.7.0,<3.0.0)", "django-celery-results (>=2.5.1,<3.0.0)", diff --git a/api/src/backend/api/admin.py b/api/src/backend/api/admin.py deleted file mode 100644 index 4185d360e9..0000000000 --- a/api/src/backend/api/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -# from django.contrib import admin - -# Register your models here. diff --git a/api/src/backend/api/compliance.py b/api/src/backend/api/compliance.py index 291cbb757e..da39fc23bb 100644 --- a/api/src/backend/api/compliance.py +++ b/api/src/backend/api/compliance.py @@ -144,6 +144,7 @@ def generate_scan_compliance( Returns: None: This function modifies the compliance_overview in place. """ + for compliance_id in PROWLER_CHECKS[provider_type][check_id]: for requirement in compliance_overview[compliance_id]["requirements"].values(): if check_id in requirement["checks"]: diff --git a/api/src/backend/api/db_router.py b/api/src/backend/api/db_router.py index 76b2ff7452..2a9085746e 100644 --- a/api/src/backend/api/db_router.py +++ b/api/src/backend/api/db_router.py @@ -48,8 +48,9 @@ class MainRouter: return db == self.admin_db def allow_relation(self, obj1, obj2, **hints): # noqa: F841 - # Allow relations if both objects are in either "default" or "admin" db connectors - if {obj1._state.db, obj2._state.db} <= {self.default_db, self.admin_db}: + # Allow relations when both objects originate from allowed connectors + allowed_dbs = {self.default_db, self.admin_db, self.replica_db} + if {obj1._state.db, obj2._state.db} <= allowed_dbs: return True return None diff --git a/api/src/backend/api/db_utils.py b/api/src/backend/api/db_utils.py index 0ef3891728..c6fcaeb43a 100644 --- a/api/src/backend/api/db_utils.py +++ b/api/src/backend/api/db_utils.py @@ -1,18 +1,35 @@ import re import secrets +import time import uuid from contextlib import contextmanager from datetime import datetime, timedelta, timezone +from celery.utils.log import get_task_logger +from config.env import env from django.conf import settings from django.contrib.auth.models import BaseUserManager -from django.db import DEFAULT_DB_ALIAS, connection, connections, models, transaction +from django.db import ( + DEFAULT_DB_ALIAS, + OperationalError, + connection, + connections, + models, + transaction, +) from django_celery_beat.models import PeriodicTask from psycopg2 import connect as psycopg2_connect from psycopg2.extensions import AsIs, new_type, register_adapter, register_type from rest_framework_json_api.serializers import ValidationError -from api.db_router import get_read_db_alias, reset_read_db_alias, set_read_db_alias +from api.db_router import ( + READ_REPLICA_ALIAS, + get_read_db_alias, + reset_read_db_alias, + set_read_db_alias, +) + +logger = get_task_logger(__name__) DB_USER = settings.DATABASES["default"]["USER"] if not settings.TESTING else "test" DB_PASSWORD = ( @@ -28,6 +45,9 @@ TASK_RUNNER_DB_TABLE = "django_celery_results_taskresult" POSTGRES_TENANT_VAR = "api.tenant_id" POSTGRES_USER_VAR = "api.user_id" +REPLICA_MAX_ATTEMPTS = env.int("POSTGRES_REPLICA_MAX_ATTEMPTS", default=3) +REPLICA_RETRY_BASE_DELAY = env.float("POSTGRES_REPLICA_RETRY_BASE_DELAY", default=0.5) + SET_CONFIG_QUERY = "SELECT set_config(%s, %s::text, TRUE);" @@ -71,24 +91,51 @@ def rls_transaction( if db_alias not in connections: db_alias = DEFAULT_DB_ALIAS - router_token = None - try: - if db_alias != DEFAULT_DB_ALIAS: - router_token = set_read_db_alias(db_alias) + alias = db_alias + is_replica = READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS + max_attempts = REPLICA_MAX_ATTEMPTS if is_replica else 1 - with transaction.atomic(using=db_alias): - conn = connections[db_alias] - with conn.cursor() as cursor: - try: - # just in case the value is a UUID object - uuid.UUID(str(value)) - except ValueError: - raise ValidationError("Must be a valid UUID") - cursor.execute(SET_CONFIG_QUERY, [parameter, value]) - yield cursor - finally: - if router_token is not None: - reset_read_db_alias(router_token) + for attempt in range(1, max_attempts + 1): + router_token = None + + # On final attempt, fallback to primary + if attempt == max_attempts and is_replica: + logger.warning( + f"RLS transaction failed after {attempt - 1} attempts on replica, " + f"falling back to primary DB" + ) + alias = DEFAULT_DB_ALIAS + + conn = connections[alias] + try: + if alias != DEFAULT_DB_ALIAS: + router_token = set_read_db_alias(alias) + + with transaction.atomic(using=alias): + with conn.cursor() as cursor: + try: + # just in case the value is a UUID object + uuid.UUID(str(value)) + except ValueError: + raise ValidationError("Must be a valid UUID") + cursor.execute(SET_CONFIG_QUERY, [parameter, value]) + yield cursor + return + except OperationalError as e: + # If on primary or max attempts reached, raise + if not is_replica or attempt == max_attempts: + raise + + # Retry with exponential backoff + delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1)) + logger.info( + f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), " + f"retrying in {delay}s. Error: {e}" + ) + time.sleep(delay) + finally: + if router_token is not None: + reset_read_db_alias(router_token) class CustomUserManager(BaseUserManager): diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index 26e5baa782..de1397b117 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -30,6 +30,7 @@ from api.models import ( LighthouseProviderConfiguration, LighthouseProviderModels, Membership, + MuteRule, OverviewStatusChoices, PermissionChoices, Processor, @@ -46,6 +47,7 @@ from api.models import ( StatusChoices, Task, TenantAPIKey, + ThreatScoreSnapshot, User, ) from api.rls import Tenant @@ -247,6 +249,14 @@ class ProviderFilter(FilterSet): choices=Provider.ProviderChoices.choices, lookup_expr="in", ) + provider_type = ChoiceFilter( + choices=Provider.ProviderChoices.choices, field_name="provider" + ) + provider_type__in = ChoiceInFilter( + field_name="provider", + choices=Provider.ProviderChoices.choices, + lookup_expr="in", + ) class Meta: model = Provider @@ -751,6 +761,14 @@ class RoleFilter(FilterSet): class ComplianceOverviewFilter(FilterSet): inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") scan_id = UUIDFilter(field_name="scan_id") + provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") + provider_id__in = UUIDInFilter(field_name="scan__provider__id", lookup_expr="in") + provider_type = ChoiceFilter( + field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices + ) + provider_type__in = ChoiceInFilter( + field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices + ) region = CharFilter(field_name="region") class Meta: @@ -850,26 +868,6 @@ class ScanSummarySeverityFilter(ScanSummaryFilter): } -class ServiceOverviewFilter(ScanSummaryFilter): - def is_valid(self): - # Check if at least one of the inserted_at filters is present - inserted_at_filters = [ - self.data.get("inserted_at"), - self.data.get("inserted_at__gte"), - self.data.get("inserted_at__lte"), - ] - if not any(inserted_at_filters): - raise ValidationError( - { - "inserted_at": [ - "At least one of filter[inserted_at], filter[inserted_at__gte], or " - "filter[inserted_at__lte] is required." - ] - } - ) - return super().is_valid() - - class IntegrationFilter(FilterSet): inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") integration_type = ChoiceFilter(choices=Integration.IntegrationChoices.choices) @@ -972,3 +970,53 @@ class LighthouseProviderModelsFilter(FilterSet): fields = { "model_id": ["exact", "icontains", "in"], } + + +class MuteRuleFilter(FilterSet): + inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") + updated_at = DateFilter(field_name="updated_at", lookup_expr="date") + created_by = UUIDFilter(field_name="created_by__id", lookup_expr="exact") + + class Meta: + model = MuteRule + fields = { + "id": ["exact", "in"], + "name": ["exact", "icontains"], + "reason": ["icontains"], + "enabled": ["exact"], + "inserted_at": ["gte", "lte"], + "updated_at": ["gte", "lte"], + } + + +class ThreatScoreSnapshotFilter(FilterSet): + """ + Filter for ThreatScore snapshots. + Allows filtering by scan, provider, compliance_id, and date ranges. + """ + + inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") + scan_id = UUIDFilter(field_name="scan__id", lookup_expr="exact") + scan_id__in = UUIDInFilter(field_name="scan__id", lookup_expr="in") + provider_id = UUIDFilter(field_name="provider__id", lookup_expr="exact") + provider_id__in = UUIDInFilter(field_name="provider__id", lookup_expr="in") + provider_type = ChoiceFilter( + field_name="provider__provider", choices=Provider.ProviderChoices.choices + ) + provider_type__in = ChoiceInFilter( + field_name="provider__provider", + choices=Provider.ProviderChoices.choices, + lookup_expr="in", + ) + compliance_id = CharFilter(field_name="compliance_id", lookup_expr="exact") + compliance_id__in = CharInFilter(field_name="compliance_id", lookup_expr="in") + + class Meta: + model = ThreatScoreSnapshot + fields = { + "scan": ["exact", "in"], + "provider": ["exact", "in"], + "compliance_id": ["exact", "in"], + "inserted_at": ["date", "gte", "lte"], + "overall_score": ["exact", "gte", "lte"], + } diff --git a/api/src/backend/api/migrations/0051_oraclecloud_provider.py b/api/src/backend/api/migrations/0051_oraclecloud_provider.py new file mode 100644 index 0000000000..022c022ea6 --- /dev/null +++ b/api/src/backend/api/migrations/0051_oraclecloud_provider.py @@ -0,0 +1,34 @@ +# Generated by Django 5.1.7 on 2025-10-14 00:00 + +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0050_lighthouse_multi_llm"), + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("oraclecloud", "Oracle Cloud Infrastructure"), + ], + default="aws", + ), + ), + migrations.RunSQL( + "ALTER TYPE provider ADD VALUE IF NOT EXISTS 'oraclecloud';", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/api/src/backend/api/migrations/0052_mute_rules.py b/api/src/backend/api/migrations/0052_mute_rules.py new file mode 100644 index 0000000000..56a3ff516f --- /dev/null +++ b/api/src/backend/api/migrations/0052_mute_rules.py @@ -0,0 +1,117 @@ +# Generated by Django 5.1.13 on 2025-10-22 11:56 + +import uuid + +import django.contrib.postgres.fields +import django.core.validators +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0051_oraclecloud_provider"), + ] + + operations = [ + migrations.CreateModel( + name="MuteRule", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "name", + models.CharField( + help_text="Human-readable name for this rule", + max_length=100, + validators=[django.core.validators.MinLengthValidator(3)], + ), + ), + ( + "reason", + models.TextField( + help_text="Reason for muting", + max_length=500, + validators=[django.core.validators.MinLengthValidator(3)], + ), + ), + ( + "enabled", + models.BooleanField( + default=True, help_text="Whether this rule is currently enabled" + ), + ), + ( + "finding_uids", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=255), + help_text="List of finding UIDs to mute", + size=None, + ), + ), + ], + options={ + "db_table": "mute_rules", + "abstract": False, + }, + ), + migrations.AddField( + model_name="finding", + name="muted_at", + field=models.DateTimeField( + blank=True, help_text="Timestamp when this finding was muted", null=True + ), + ), + migrations.AlterField( + model_name="tenantapikey", + name="name", + field=models.CharField( + max_length=100, + validators=[django.core.validators.MinLengthValidator(3)], + ), + ), + migrations.AddField( + model_name="muterule", + name="created_by", + field=models.ForeignKey( + help_text="User who created this rule", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="created_mute_rules", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddField( + model_name="muterule", + name="tenant", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + migrations.AddConstraint( + model_name="muterule", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_muterule", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + migrations.AddConstraint( + model_name="muterule", + constraint=models.UniqueConstraint( + fields=("tenant_id", "name"), name="unique_mute_rule_name_per_tenant" + ), + ), + ] diff --git a/api/src/backend/api/migrations/0053_lighthouse_bedrock_openai_compatible.py b/api/src/backend/api/migrations/0053_lighthouse_bedrock_openai_compatible.py new file mode 100644 index 0000000000..d7054d6443 --- /dev/null +++ b/api/src/backend/api/migrations/0053_lighthouse_bedrock_openai_compatible.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1.12 on 2025-10-14 11:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0052_mute_rules"), + ] + + operations = [ + migrations.AlterField( + model_name="lighthouseproviderconfiguration", + name="provider_type", + field=models.CharField( + choices=[ + ("openai", "OpenAI"), + ("bedrock", "AWS Bedrock"), + ("openai_compatible", "OpenAI Compatible"), + ], + help_text="LLM provider name", + max_length=50, + ), + ) + ] diff --git a/api/src/backend/api/migrations/0054_iac_provider.py b/api/src/backend/api/migrations/0054_iac_provider.py new file mode 100644 index 0000000000..3ba456b0cf --- /dev/null +++ b/api/src/backend/api/migrations/0054_iac_provider.py @@ -0,0 +1,35 @@ +# Generated by Django 5.1.10 on 2025-09-09 09:25 + +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0053_lighthouse_bedrock_openai_compatible"), + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("oci", "Oracle Cloud Infrastructure"), + ("iac", "IaC"), + ], + default="aws", + ), + ), + migrations.RunSQL( + "ALTER TYPE provider ADD VALUE IF NOT EXISTS 'iac';", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/api/src/backend/api/migrations/0055_mongodbatlas_provider.py b/api/src/backend/api/migrations/0055_mongodbatlas_provider.py new file mode 100644 index 0000000000..beb7a3946a --- /dev/null +++ b/api/src/backend/api/migrations/0055_mongodbatlas_provider.py @@ -0,0 +1,32 @@ +# Generated by Django 5.1.13 on 2025-11-05 08:37 + +from django.db import migrations + +import api.db_utils + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0054_iac_provider"), + ] + + operations = [ + migrations.AlterField( + model_name="provider", + name="provider", + field=api.db_utils.ProviderEnumField( + choices=[ + ("aws", "AWS"), + ("azure", "Azure"), + ("gcp", "GCP"), + ("kubernetes", "Kubernetes"), + ("m365", "M365"), + ("github", "GitHub"), + ("mongodbatlas", "MongoDB Atlas"), + ("iac", "IaC"), + ("oraclecloud", "Oracle Cloud Infrastructure"), + ], + default="aws", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0056_remove_provider_unique_provider_uids_and_more.py b/api/src/backend/api/migrations/0056_remove_provider_unique_provider_uids_and_more.py new file mode 100644 index 0000000000..14cbc4fa47 --- /dev/null +++ b/api/src/backend/api/migrations/0056_remove_provider_unique_provider_uids_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.13 on 2025-11-06 09:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0055_mongodbatlas_provider"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="provider", + name="unique_provider_uids", + ), + migrations.AddConstraint( + model_name="provider", + constraint=models.UniqueConstraint( + condition=models.Q(("is_deleted", False)), + fields=("tenant_id", "provider", "uid"), + name="unique_provider_uids", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0057_threatscoresnapshot.py b/api/src/backend/api/migrations/0057_threatscoresnapshot.py new file mode 100644 index 0000000000..ee3530a5b6 --- /dev/null +++ b/api/src/backend/api/migrations/0057_threatscoresnapshot.py @@ -0,0 +1,170 @@ +# Generated by Django 5.1.13 on 2025-10-31 09:04 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0056_remove_provider_unique_provider_uids_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="ThreatScoreSnapshot", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ( + "compliance_id", + models.CharField( + help_text="Compliance framework ID (e.g., 'prowler_threatscore_aws')", + max_length=100, + ), + ), + ( + "overall_score", + models.DecimalField( + decimal_places=2, + help_text="Overall ThreatScore percentage (0-100)", + max_digits=5, + ), + ), + ( + "score_delta", + models.DecimalField( + blank=True, + decimal_places=2, + help_text="Score change compared to previous snapshot (positive = improvement)", + max_digits=5, + null=True, + ), + ), + ( + "section_scores", + models.JSONField( + blank=True, + default=dict, + help_text="ThreatScore breakdown by section", + ), + ), + ( + "critical_requirements", + models.JSONField( + blank=True, + default=list, + help_text="List of critical failed requirements (risk >= 4)", + ), + ), + ( + "total_requirements", + models.IntegerField( + default=0, help_text="Total number of requirements evaluated" + ), + ), + ( + "passed_requirements", + models.IntegerField( + default=0, help_text="Number of requirements with PASS status" + ), + ), + ( + "failed_requirements", + models.IntegerField( + default=0, help_text="Number of requirements with FAIL status" + ), + ), + ( + "manual_requirements", + models.IntegerField( + default=0, help_text="Number of requirements with MANUAL status" + ), + ), + ( + "total_findings", + models.IntegerField( + default=0, + help_text="Total number of findings across all requirements", + ), + ), + ( + "passed_findings", + models.IntegerField( + default=0, help_text="Number of findings with PASS status" + ), + ), + ( + "failed_findings", + models.IntegerField( + default=0, help_text="Number of findings with FAIL status" + ), + ), + ( + "provider", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="threatscore_snapshots", + related_query_name="threatscore_snapshot", + to="api.provider", + ), + ), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="threatscore_snapshots", + related_query_name="threatscore_snapshot", + to="api.scan", + ), + ), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + ], + options={ + "db_table": "threatscore_snapshots", + "abstract": False, + }, + ), + migrations.AddIndex( + model_name="threatscoresnapshot", + index=models.Index( + fields=["tenant_id", "scan_id"], name="threatscore_snap_t_scan_idx" + ), + ), + migrations.AddIndex( + model_name="threatscoresnapshot", + index=models.Index( + fields=["tenant_id", "provider_id"], name="threatscore_snap_t_prov_idx" + ), + ), + migrations.AddIndex( + model_name="threatscoresnapshot", + index=models.Index( + fields=["tenant_id", "inserted_at"], name="threatscore_snap_t_time_idx" + ), + ), + migrations.AddConstraint( + model_name="threatscoresnapshot", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_threatscoresnapshot", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + ] diff --git a/api/src/backend/api/migrations/0058_drop_redundant_compliance_requirement_indexes.py b/api/src/backend/api/migrations/0058_drop_redundant_compliance_requirement_indexes.py new file mode 100644 index 0000000000..1d2ee35f4d --- /dev/null +++ b/api/src/backend/api/migrations/0058_drop_redundant_compliance_requirement_indexes.py @@ -0,0 +1,25 @@ +from django.contrib.postgres.operations import RemoveIndexConcurrently +from django.db import migrations + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0057_threatscoresnapshot"), + ] + + operations = [ + RemoveIndexConcurrently( + model_name="compliancerequirementoverview", + name="cro_tenant_scan_idx", + ), + RemoveIndexConcurrently( + model_name="compliancerequirementoverview", + name="cro_scan_comp_idx", + ), + RemoveIndexConcurrently( + model_name="compliancerequirementoverview", + name="cro_scan_comp_req_idx", + ), + ] diff --git a/api/src/backend/api/migrations/0059_compliance_overview_summary.py b/api/src/backend/api/migrations/0059_compliance_overview_summary.py new file mode 100644 index 0000000000..d2d57a34ae --- /dev/null +++ b/api/src/backend/api/migrations/0059_compliance_overview_summary.py @@ -0,0 +1,75 @@ +# Generated by Django 5.1.13 on 2025-10-30 15:23 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + +import api.rls + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0058_drop_redundant_compliance_requirement_indexes"), + ] + + operations = [ + migrations.CreateModel( + name="ComplianceOverviewSummary", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ("compliance_id", models.TextField()), + ("requirements_passed", models.IntegerField(default=0)), + ("requirements_failed", models.IntegerField(default=0)), + ("requirements_manual", models.IntegerField(default=0)), + ("total_requirements", models.IntegerField(default=0)), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="compliance_summaries", + related_query_name="compliance_summary", + to="api.scan", + ), + ), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + ], + options={ + "db_table": "compliance_overview_summaries", + "abstract": False, + "indexes": [ + models.Index( + fields=["tenant_id", "scan_id"], name="cos_tenant_scan_idx" + ) + ], + "constraints": [ + models.UniqueConstraint( + fields=("tenant_id", "scan_id", "compliance_id"), + name="unique_compliance_summary_per_scan", + ) + ], + }, + ), + migrations.AddConstraint( + model_name="complianceoverviewsummary", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_complianceoverviewsummary", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index c2c0dbcb47..1191350ac6 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -284,6 +284,9 @@ class Provider(RowLevelSecurityProtectedModel): KUBERNETES = "kubernetes", _("Kubernetes") M365 = "m365", _("M365") GITHUB = "github", _("GitHub") + MONGODBATLAS = "mongodbatlas", _("MongoDB Atlas") + IAC = "iac", _("IaC") + ORACLECLOUD = "oraclecloud", _("Oracle Cloud Infrastructure") @staticmethod def validate_aws_uid(value): @@ -354,6 +357,40 @@ class Provider(RowLevelSecurityProtectedModel): pointer="/data/attributes/uid", ) + @staticmethod + def validate_iac_uid(value): + # Validate that it's a valid repository URL (git URL format) + if not re.match( + r"^(https?://|git@|ssh://)[^\s/]+[^\s]*\.git$|^(https?://)[^\s/]+[^\s]*$", + value, + ): + raise ModelValidationError( + detail="IaC provider ID must be a valid repository URL (e.g., https://github.com/user/repo or https://github.com/user/repo.git).", + code="iac-uid", + pointer="/data/attributes/uid", + ) + + @staticmethod + def validate_oraclecloud_uid(value): + if not re.match( + r"^ocid1\.([a-z0-9_-]+)\.([a-z0-9_-]+)\.([a-z0-9_-]*)\.([a-z0-9]+)$", value + ): + raise ModelValidationError( + detail="Oracle Cloud Infrastructure provider ID must be a valid tenancy OCID in the format: " + "ocid1....", + code="oraclecloud-uid", + pointer="/data/attributes/uid", + ) + + @staticmethod + def validate_mongodbatlas_uid(value): + if not re.match(r"^[0-9a-fA-F]{24}$", value): + raise ModelValidationError( + detail="MongoDB Atlas organization ID must be a 24-character hexadecimal string.", + code="mongodbatlas-uid", + pointer="/data/attributes/uid", + ) + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) inserted_at = models.DateTimeField(auto_now_add=True, editable=False) updated_at = models.DateTimeField(auto_now=True, editable=False) @@ -388,7 +425,8 @@ class Provider(RowLevelSecurityProtectedModel): constraints = [ models.UniqueConstraint( - fields=("tenant_id", "provider", "uid", "is_deleted"), + fields=("tenant_id", "provider", "uid"), + condition=Q(is_deleted=False), name="unique_provider_uids", ), RowLevelSecurityConstraint( @@ -810,6 +848,9 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): muted_reason = models.TextField( blank=True, null=True, validators=[MinLengthValidator(3)], max_length=500 ) + muted_at = models.DateTimeField( + null=True, blank=True, help_text="Timestamp when this finding was muted" + ) compliance = models.JSONField(default=dict, null=True, blank=True) # Denormalize resource data for performance @@ -1330,35 +1371,70 @@ class ComplianceRequirementOverview(RowLevelSecurityProtectedModel): ), ] indexes = [ - models.Index(fields=["tenant_id", "scan_id"], name="cro_tenant_scan_idx"), - models.Index( - fields=["tenant_id", "scan_id", "compliance_id"], - name="cro_scan_comp_idx", - ), models.Index( fields=["tenant_id", "scan_id", "compliance_id", "region"], name="cro_scan_comp_reg_idx", ), - models.Index( - fields=["tenant_id", "scan_id", "compliance_id", "requirement_id"], - name="cro_scan_comp_req_idx", - ), - models.Index( - fields=[ - "tenant_id", - "scan_id", - "compliance_id", - "requirement_id", - "region", - ], - name="cro_scan_comp_req_reg_idx", - ), ] class JSONAPIMeta: resource_name = "compliance-requirements-overviews" +class ComplianceOverviewSummary(RowLevelSecurityProtectedModel): + """ + Pre-aggregated compliance overview aggregated across ALL regions. + One row per (scan_id, compliance_id) combination. + + This table optimizes the common case where users view overall compliance + without filtering by region. For region-specific views, the detailed + ComplianceRequirementOverview table is used instead. + """ + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + + scan = models.ForeignKey( + Scan, + on_delete=models.CASCADE, + related_name="compliance_summaries", + related_query_name="compliance_summary", + ) + + compliance_id = models.TextField(blank=False) + + # Pre-aggregated scores (computed across ALL regions) + requirements_passed = models.IntegerField(default=0) + requirements_failed = models.IntegerField(default=0) + requirements_manual = models.IntegerField(default=0) + total_requirements = models.IntegerField(default=0) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "compliance_overview_summaries" + + constraints = [ + models.UniqueConstraint( + fields=("tenant_id", "scan_id", "compliance_id"), + name="unique_compliance_summary_per_scan", + ), + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "DELETE"], + ), + ] + + indexes = [ + models.Index( + fields=["tenant_id", "scan_id"], + name="cos_tenant_scan_idx", + ), + ] + + class JSONAPIMeta: + resource_name = "compliance-overview-summaries" + + class ScanSummary(RowLevelSecurityProtectedModel): objects = ActiveProviderManager() all_objects = models.Manager() @@ -1922,6 +1998,59 @@ class LighthouseConfiguration(RowLevelSecurityProtectedModel): resource_name = "lighthouse-configurations" +class MuteRule(RowLevelSecurityProtectedModel): + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + updated_at = models.DateTimeField(auto_now=True, editable=False) + + # Rule metadata + name = models.CharField( + max_length=100, + validators=[MinLengthValidator(3)], + help_text="Human-readable name for this rule", + ) + reason = models.TextField( + validators=[MinLengthValidator(3)], + max_length=500, + help_text="Reason for muting", + ) + enabled = models.BooleanField( + default=True, help_text="Whether this rule is currently enabled" + ) + + # Audit fields + created_by = models.ForeignKey( + User, + on_delete=models.SET_NULL, + null=True, + related_name="created_mute_rules", + help_text="User who created this rule", + ) + + # Rule criteria - array of finding UIDs + finding_uids = ArrayField( + models.CharField(max_length=255), help_text="List of finding UIDs to mute" + ) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "mute_rules" + + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + models.UniqueConstraint( + fields=("tenant_id", "name"), + name="unique_mute_rule_name_per_tenant", + ), + ] + + class JSONAPIMeta: + resource_name = "mute-rules" + + class Processor(RowLevelSecurityProtectedModel): class ProcessorChoices(models.TextChoices): MUTELIST = "mutelist", _("Mutelist") @@ -1970,6 +2099,8 @@ class LighthouseProviderConfiguration(RowLevelSecurityProtectedModel): class LLMProviderChoices(models.TextChoices): OPENAI = "openai", _("OpenAI") + BEDROCK = "bedrock", _("AWS Bedrock") + OPENAI_COMPATIBLE = "openai_compatible", _("OpenAI Compatible") id = models.UUIDField(primary_key=True, default=uuid4, editable=False) inserted_at = models.DateTimeField(auto_now_add=True, editable=False) @@ -2091,7 +2222,7 @@ class LighthouseTenantConfiguration(RowLevelSecurityProtectedModel): ] class JSONAPIMeta: - resource_name = "lighthouse-config" + resource_name = "lighthouse-configurations" class LighthouseProviderModels(RowLevelSecurityProtectedModel): @@ -2140,3 +2271,137 @@ class LighthouseProviderModels(RowLevelSecurityProtectedModel): name="lh_prov_models_cfg_idx", ), ] + + class JSONAPIMeta: + resource_name = "lighthouse-models" + + +class ThreatScoreSnapshot(RowLevelSecurityProtectedModel): + """ + Stores historical ThreatScore metrics for a given scan. + Snapshots are created automatically after each ThreatScore report generation. + """ + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + + scan = models.ForeignKey( + Scan, + on_delete=models.CASCADE, + related_name="threatscore_snapshots", + related_query_name="threatscore_snapshot", + ) + + provider = models.ForeignKey( + Provider, + on_delete=models.CASCADE, + related_name="threatscore_snapshots", + related_query_name="threatscore_snapshot", + ) + + compliance_id = models.CharField( + max_length=100, + blank=False, + null=False, + help_text="Compliance framework ID (e.g., 'prowler_threatscore_aws')", + ) + + # Overall ThreatScore metrics + overall_score = models.DecimalField( + max_digits=5, + decimal_places=2, + help_text="Overall ThreatScore percentage (0-100)", + ) + + # Score improvement/degradation compared to previous snapshot + score_delta = models.DecimalField( + max_digits=5, + decimal_places=2, + null=True, + blank=True, + help_text="Score change compared to previous snapshot (positive = improvement)", + ) + + # Section breakdown stored as JSON + # Format: {"1. IAM": 85.5, "2. Attack Surface": 92.3, ...} + section_scores = models.JSONField( + default=dict, + blank=True, + help_text="ThreatScore breakdown by section", + ) + + # Critical requirements metadata stored as JSON + # Format: [{"requirement_id": "...", "risk_level": 5, "weight": 150, ...}, ...] + critical_requirements = models.JSONField( + default=list, + blank=True, + help_text="List of critical failed requirements (risk >= 4)", + ) + + # Summary statistics + total_requirements = models.IntegerField( + default=0, + help_text="Total number of requirements evaluated", + ) + + passed_requirements = models.IntegerField( + default=0, + help_text="Number of requirements with PASS status", + ) + + failed_requirements = models.IntegerField( + default=0, + help_text="Number of requirements with FAIL status", + ) + + manual_requirements = models.IntegerField( + default=0, + help_text="Number of requirements with MANUAL status", + ) + + total_findings = models.IntegerField( + default=0, + help_text="Total number of findings across all requirements", + ) + + passed_findings = models.IntegerField( + default=0, + help_text="Number of findings with PASS status", + ) + + failed_findings = models.IntegerField( + default=0, + help_text="Number of findings with FAIL status", + ) + + def __str__(self): + return f"ThreatScore {self.overall_score}% for scan {self.scan_id} ({self.inserted_at})" + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "threatscore_snapshots" + + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + + indexes = [ + models.Index( + fields=["tenant_id", "scan_id"], + name="threatscore_snap_t_scan_idx", + ), + models.Index( + fields=["tenant_id", "provider_id"], + name="threatscore_snap_t_prov_idx", + ), + models.Index( + fields=["tenant_id", "inserted_at"], + name="threatscore_snap_t_time_idx", + ), + ] + + class JSONAPIMeta: + resource_name = "threatscore-snapshots" diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index d3f68719a1..6227d2500e 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -283,8 +283,11 @@ paths: /api/v1/compliance-overviews: get: operationId: compliance_overviews_list - description: Retrieve an overview of all the compliance in a given scan. - summary: List compliance overviews for a scan + description: Retrieve an overview of all compliance frameworks. If scan_id is + provided, returns compliance data for that specific scan. If scan_id is omitted, + returns compliance data aggregated from the latest completed scan of each + provider. + summary: List compliance overviews parameters: - in: query name: fields[compliance-overviews] @@ -343,6 +346,32 @@ paths: schema: type: string format: date-time + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + description: Filter by specific provider ID. + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Filter by multiple provider IDs (comma-separated). + - in: query + name: filter[provider_type] + schema: + type: string + description: Filter by provider type (e.g., aws, azure, gcp). + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + description: Filter by multiple provider types (comma-separated). - in: query name: filter[region] schema: @@ -365,8 +394,8 @@ paths: schema: type: string format: uuid - description: Related scan ID. - required: true + description: Optional scan ID. If provided, returns compliance for that scan. + If omitted, returns compliance for the latest completed scan per provider. - name: filter[search] required: false in: query @@ -606,6 +635,77 @@ paths: schema: type: string format: date-time + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + x-spec-enum-id: eca8c51e6bd28935 + enum: + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + x-spec-enum-id: eca8c51e6bd28935 + enum: + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + explode: false + style: form - in: query name: filter[region] schema: @@ -869,14 +969,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -884,20 +987,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -907,6 +1016,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -1396,14 +1508,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -1411,20 +1526,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -1434,6 +1555,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -1831,14 +1955,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -1846,20 +1973,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -1869,6 +2002,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -2264,14 +2400,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -2279,20 +2418,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -2302,6 +2447,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -2685,14 +2833,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -2700,20 +2851,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -2723,6 +2880,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -3565,7 +3725,7 @@ paths: summary: Get Lighthouse AI Tenant config parameters: - in: query - name: fields[lighthouse-config] + name: fields[lighthouse-configurations] schema: type: array items: @@ -3670,7 +3830,7 @@ paths: summary: List all LLM models parameters: - in: query - name: fields[LighthouseProviderModels] + name: fields[lighthouse-models] schema: type: array items: @@ -3707,26 +3867,34 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 80350f3ab09becd9 + x-spec-enum-id: 30bc523cb9ec0e8b enum: + - bedrock - openai + - openai_compatible description: |- LLM provider name * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 80350f3ab09becd9 + x-spec-enum-id: 30bc523cb9ec0e8b enum: + - bedrock - openai + - openai_compatible description: |- Multiple values may be separated by commas. * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible explode: false style: form - name: filter[search] @@ -3791,7 +3959,7 @@ paths: summary: Retrieve LLM model details parameters: - in: query - name: fields[LighthouseProviderModels] + name: fields[lighthouse-models] schema: type: array items: @@ -3829,7 +3997,7 @@ paths: get: operationId: lighthouse_providers_list description: Retrieve all LLM provider configurations for the current tenant - summary: List all LLM provider configs + summary: List all LLM provider configurations parameters: - in: query name: fields[lighthouse-providers] @@ -3856,26 +4024,34 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 80350f3ab09becd9 + x-spec-enum-id: 30bc523cb9ec0e8b enum: + - bedrock - openai + - openai_compatible description: |- LLM provider name * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 80350f3ab09becd9 + x-spec-enum-id: 30bc523cb9ec0e8b enum: + - bedrock - openai + - openai_compatible description: |- Multiple values may be separated by commas. * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible explode: false style: form - name: filter[search] @@ -3937,7 +4113,7 @@ paths: operationId: lighthouse_providers_create description: Create a per-tenant configuration for an LLM provider. Only one configuration per provider type is allowed per tenant. - summary: Create LLM provider config + summary: Create LLM provider configuration tags: - Lighthouse AI requestBody: @@ -3966,7 +4142,7 @@ paths: operationId: lighthouse_providers_retrieve description: Get details for a specific provider configuration in the current tenant. - summary: Retrieve LLM provider config + summary: Retrieve LLM provider configuration parameters: - in: query name: fields[lighthouse-providers] @@ -4006,7 +4182,7 @@ paths: patch: operationId: lighthouse_providers_partial_update description: Partially update a provider configuration (e.g., base_url, is_active). - summary: Update LLM provider config + summary: Update LLM provider configuration parameters: - in: path name: id @@ -4042,7 +4218,7 @@ paths: operationId: lighthouse_providers_destroy description: Delete a provider configuration. Any tenant defaults that reference this provider are cleared during deletion. - summary: Delete LLM provider config + summary: Delete LLM provider configuration parameters: - in: path name: id @@ -4101,7 +4277,7 @@ paths: post: operationId: lighthouse_providers_refresh_models_create description: Fetch available models for this provider configuration and upsert - into catalog. + into catalog. Supports OpenAI, OpenAI-compatible, and AWS Bedrock providers. summary: Refresh LLM models catalog parameters: - in: path @@ -4137,6 +4313,290 @@ paths: task_args: null metadata: null description: '' + /api/v1/mute-rules: + get: + operationId: mute_rules_list + description: Retrieve a list of all mute rules with filtering options. + summary: List all mute rules + parameters: + - in: query + name: fields[mute-rules] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - name + - reason + - enabled + - created_by + - finding_uids + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[created_by] + schema: + type: string + format: uuid + - in: query + name: filter[enabled] + schema: + type: boolean + - in: query + name: filter[id] + schema: + type: string + format: uuid + - in: query + name: filter[id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[inserted_at] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[inserted_at__lte] + schema: + type: string + format: date-time + - in: query + name: filter[name] + schema: + type: string + - in: query + name: filter[name__icontains] + schema: + type: string + - in: query + name: filter[reason__icontains] + schema: + type: string + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[updated_at] + schema: + type: string + format: date + - in: query + name: filter[updated_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[updated_at__lte] + schema: + type: string + format: date-time + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - created_by + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - name + - -name + - enabled + - -enabled + - inserted_at + - -inserted_at + - updated_at + - -updated_at + explode: false + tags: + - Mute Rules + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedMuteRuleList' + description: '' + post: + operationId: mute_rules_create + description: Create a new mute rule by providing finding IDs, name, and reason. + The rule will immediately mute the selected findings and launch a background + task to mute all historical findings with matching UIDs. + summary: Create a new mute rule + tags: + - Mute Rules + requestBody: + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/MuteRuleCreateRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/MuteRuleCreateRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/MuteRuleCreateRequest' + required: true + security: + - JWT or API Key: [] + responses: + '201': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/MuteRuleCreateResponse' + description: '' + /api/v1/mute-rules/{id}: + get: + operationId: mute_rules_retrieve + description: Fetch detailed information about a specific mute rule by ID. + summary: Retrieve a mute rule + parameters: + - in: query + name: fields[mute-rules] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - name + - reason + - enabled + - created_by + - finding_uids + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this mute rule. + required: true + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - created_by + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + tags: + - Mute Rules + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/MuteRuleResponse' + description: '' + patch: + operationId: mute_rules_partial_update + description: Update certain fields of an existing mute rule (e.g., name, reason, + enabled). + summary: Partially update a mute rule + parameters: + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this mute rule. + required: true + tags: + - Mute Rules + requestBody: + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PatchedMuteRuleUpdateRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PatchedMuteRuleUpdateRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/PatchedMuteRuleUpdateRequest' + required: true + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/SerializerMetaclassResponse' + description: '' + delete: + operationId: mute_rules_destroy + description: 'Remove a mute rule from the system. Note: Previously muted findings + remain muted.' + summary: Delete a mute rule + parameters: + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this mute rule. + required: true + tags: + - Mute Rules + security: + - JWT or API Key: [] + responses: + '204': + description: No response body /api/v1/overviews/findings: get: operationId: overviews_findings_retrieve @@ -4210,14 +4670,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -4225,20 +4688,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -4248,6 +4717,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -4387,14 +4859,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -4402,20 +4877,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -4425,6 +4906,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -4536,13 +5020,43 @@ paths: schema: $ref: '#/components/schemas/OverviewProviderResponse' description: '' + /api/v1/overviews/providers/count: + get: + operationId: overviews_providers_count_retrieve + description: Retrieve the number of providers grouped by provider type. This + endpoint counts every provider in the tenant, including those without completed + scans. + summary: Get provider counts grouped by type + parameters: + - in: query + name: fields[providers-count-overview] + schema: + type: array + items: + type: string + enum: + - id + - count + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + tags: + - Overview + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/OverviewProviderCountResponse' + description: '' /api/v1/overviews/services: get: operationId: overviews_services_retrieve description: Retrieve an aggregated summary of findings grouped by service. The response includes the total count of findings for each service, as long - as there are at least one finding for that service. At least one of the `inserted_at` - filters must be provided. + as there are at least one finding for that service. summary: Get findings data by service parameters: - in: query @@ -4599,14 +5113,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -4614,20 +5131,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -4637,6 +5160,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -4693,6 +5219,90 @@ paths: schema: $ref: '#/components/schemas/OverviewServiceResponse' description: '' + /api/v1/overviews/threatscore: + get: + operationId: overviews_threatscore_retrieve + description: Retrieve ThreatScore metrics. By default, returns the latest snapshot + for each provider. Use snapshot_id to retrieve a specific historical snapshot. + summary: Get ThreatScore snapshots + parameters: + - in: query + name: fields[threatscore-snapshots] + schema: + type: array + items: + type: string + enum: + - id + - inserted_at + - scan + - provider + - compliance_id + - overall_score + - score_delta + - section_scores + - critical_requirements + - total_requirements + - passed_requirements + - failed_requirements + - manual_requirements + - total_findings + - passed_findings + - failed_findings + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - scan + - provider + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + - in: query + name: provider_id + schema: + type: string + format: uuid + description: Filter by specific provider ID + - in: query + name: provider_id__in + schema: + type: string + description: Filter by multiple provider IDs (comma-separated UUIDs) + - in: query + name: provider_type + schema: + type: string + description: Filter by provider type (aws, azure, gcp, etc.) + - in: query + name: provider_type__in + schema: + type: string + description: Filter by multiple provider types (comma-separated) + - in: query + name: snapshot_id + schema: + type: string + format: uuid + description: Retrieve a specific snapshot by ID. If not provided, returns + latest snapshots. + tags: + - Overviews + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/ThreatScoreSnapshotResponse' + description: '' /api/v1/processors: get: operationId: processors_list @@ -5322,14 +5932,17 @@ paths: name: filter[provider] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -5337,20 +5950,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -5360,6 +5979,65 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + x-spec-enum-id: eca8c51e6bd28935 + enum: + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + x-spec-enum-id: eca8c51e6bd28935 + enum: + - aws + - azure + - gcp + - github + - iac + - kubernetes + - m365 + - mongodbatlas + - oraclecloud + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - name: filter[search] @@ -5894,6 +6572,9 @@ paths: - findings - failed_findings_count - url + - metadata + - details + - partition - type description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. @@ -5957,14 +6638,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -5972,20 +6656,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -5995,6 +6685,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -6215,6 +6908,9 @@ paths: - findings - failed_findings_count - url + - metadata + - details + - partition - type description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. @@ -6274,6 +6970,9 @@ paths: - findings - failed_findings_count - url + - metadata + - details + - partition - type description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. @@ -6322,14 +7021,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -6337,20 +7039,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -6360,6 +7068,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -6588,14 +7299,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -6603,20 +7317,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -6626,6 +7346,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -6860,14 +7583,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -6875,20 +7601,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -6898,6 +7630,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -7695,14 +8430,17 @@ paths: name: filter[provider_type] schema: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- * `aws` - AWS * `azure` - Azure @@ -7710,20 +8448,26 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure - in: query name: filter[provider_type__in] schema: type: array items: type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 enum: - aws - azure - gcp - github + - iac - kubernetes - m365 + - mongodbatlas + - oraclecloud description: |- Multiple values may be separated by commas. @@ -7733,6 +8477,9 @@ paths: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure explode: false style: form - in: query @@ -8106,6 +8853,72 @@ paths: '404': description: The scan has no reports, or the report generation task has not started yet + /api/v1/scans/{id}/threatscore: + get: + operationId: scans_threatscore_retrieve + description: Download a specific threatscore report (e.g., 'prowler_threatscore_aws') + as a PDF file. + summary: Retrieve threatscore report + parameters: + - in: query + name: fields[scans] + schema: + type: array + items: + type: string + enum: + - name + - trigger + - state + - unique_resource_count + - progress + - duration + - provider + - task + - inserted_at + - started_at + - completed_at + - scheduled_at + - next_scan_at + - processor + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this scan. + required: true + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - provider + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + tags: + - Scan + security: + - JWT or API Key: [] + responses: + '200': + description: PDF file containing the threatscore report + '202': + description: The task is in progress + '401': + description: API key missing or user not Authenticated + '403': + description: There is a problem with credentials + '404': + description: The scan has no threatscore reports, or the threatscore report + generation task has not started yet /api/v1/schedules/daily: post: operationId: schedules_daily_create @@ -11469,12 +12282,16 @@ components: provider_type: enum: - openai + - bedrock + - openai_compatible type: string - x-spec-enum-id: 80350f3ab09becd9 + x-spec-enum-id: 30bc523cb9ec0e8b description: |- LLM provider name * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible base_url: type: string format: uri @@ -11506,18 +12323,64 @@ components: provider_type: enum: - openai + - bedrock + - openai_compatible type: string - x-spec-enum-id: 80350f3ab09becd9 + x-spec-enum-id: 30bc523cb9ec0e8b description: |- - LLM provider name + LLM provider type. Determines which credential format to use. See 'credentials' field documentation for provider-specific requirements. * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible base_url: type: string format: uri nullable: true - maxLength: 200 + description: Base URL for the LLM provider API. Required for 'openai_compatible' + provider type. credentials: + oneOf: + - type: object + title: OpenAI Credentials + properties: + api_key: + type: string + description: OpenAI API key. Must start with 'sk-' followed by + alphanumeric characters, hyphens, or underscores. + pattern: ^sk-[\w-]+$ + required: + - api_key + - type: object + title: AWS Bedrock Credentials + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - type: object + title: OpenAI Compatible Credentials + properties: + api_key: + type: string + description: 'API key for OpenAI-compatible provider. The format + varies by provider. Note: The ''base_url'' field (separate from + credentials) is required when using this provider type.' + required: + - api_key writeOnly: true is_active: type: boolean @@ -11546,18 +12409,66 @@ components: provider_type: enum: - openai + - bedrock + - openai_compatible type: string - x-spec-enum-id: 80350f3ab09becd9 + x-spec-enum-id: 30bc523cb9ec0e8b description: |- - LLM provider name + LLM provider type. Determines which credential format to use. See 'credentials' field documentation for provider-specific requirements. * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible base_url: type: string format: uri nullable: true - maxLength: 200 + minLength: 1 + description: Base URL for the LLM provider API. Required for 'openai_compatible' + provider type. credentials: + oneOf: + - type: object + title: OpenAI Credentials + properties: + api_key: + type: string + description: OpenAI API key. Must start with 'sk-' followed + by alphanumeric characters, hyphens, or underscores. + pattern: ^sk-[\w-]+$ + required: + - api_key + - type: object + title: AWS Bedrock Credentials + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - type: object + title: OpenAI Compatible Credentials + properties: + api_key: + type: string + description: 'API key for OpenAI-compatible provider. The + format varies by provider. Note: The ''base_url'' field + (separate from credentials) is required when using this + provider type.' + required: + - api_key writeOnly: true is_active: type: boolean @@ -11603,19 +12514,65 @@ components: provider_type: enum: - openai + - bedrock + - openai_compatible type: string - x-spec-enum-id: 80350f3ab09becd9 + x-spec-enum-id: 30bc523cb9ec0e8b readOnly: true description: |- LLM provider name * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible base_url: type: string format: uri nullable: true - maxLength: 200 + description: Base URL for the LLM provider API. Required for 'openai_compatible' + provider type. credentials: + oneOf: + - type: object + title: OpenAI Credentials + properties: + api_key: + type: string + description: OpenAI API key. Must start with 'sk-' followed by + alphanumeric characters, hyphens, or underscores. + pattern: ^sk-[\w-]+$ + required: + - api_key + - type: object + title: AWS Bedrock Credentials + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - type: object + title: OpenAI Compatible Credentials + properties: + api_key: + type: string + description: 'API key for OpenAI-compatible provider. The format + varies by provider. Note: The ''base_url'' field (separate from + credentials) is required when using this provider type.' + required: + - api_key writeOnly: true is_active: type: boolean @@ -11639,7 +12596,7 @@ components: member is used to describe resource objects that share common attributes and relationships. enum: - - LighthouseProviderModels + - lighthouse-models id: type: string format: uuid @@ -11712,7 +12669,7 @@ components: member is used to describe resource objects that share common attributes and relationships. enum: - - lighthouse-config + - lighthouse-configurations id: type: string format: uuid @@ -11749,7 +12706,7 @@ components: member is used to describe resource objects that share common attributes and relationships. enum: - - lighthouse-config + - lighthouse-configurations id: type: string format: uuid @@ -11860,6 +12817,272 @@ components: $ref: '#/components/schemas/Membership' required: - data + MuteRule: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - mute-rules + id: + type: string + format: uuid + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + description: Human-readable name for this rule + maxLength: 100 + minLength: 3 + reason: + type: string + description: Reason for muting + minLength: 3 + maxLength: 500 + enabled: + type: boolean + description: Whether this rule is currently enabled + finding_uids: + type: array + items: + type: string + readOnly: true + description: List of finding UIDs that are muted by this rule + required: + - name + - reason + relationships: + type: object + properties: + created_by: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - users + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + nullable: true + MuteRuleCreate: + type: object + required: + - type + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - mute-rules + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + description: Human-readable name for this rule + maxLength: 100 + minLength: 3 + reason: + type: string + description: Reason for muting + minLength: 3 + maxLength: 500 + enabled: + type: boolean + readOnly: true + description: Whether this rule is currently enabled + finding_ids: + type: array + items: + type: string + format: uuid + writeOnly: true + description: List of Finding IDs to mute (will be converted to UIDs) + finding_uids: + type: array + items: + type: string + readOnly: true + description: List of finding UIDs that are muted by this rule + required: + - name + - reason + - finding_ids + relationships: + type: object + properties: + created_by: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - users + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + nullable: true + MuteRuleCreateRequest: + type: object + properties: + data: + type: object + required: + - type + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - mute-rules + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + minLength: 3 + description: Human-readable name for this rule + maxLength: 100 + reason: + type: string + minLength: 3 + description: Reason for muting + maxLength: 500 + enabled: + type: boolean + readOnly: true + description: Whether this rule is currently enabled + finding_ids: + type: array + items: + type: string + format: uuid + writeOnly: true + description: List of Finding IDs to mute (will be converted to UIDs) + finding_uids: + type: array + items: + type: string + minLength: 1 + readOnly: true + description: List of finding UIDs that are muted by this rule + required: + - name + - reason + - finding_ids + relationships: + type: object + properties: + created_by: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - users + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share + common attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + nullable: true + required: + - data + MuteRuleCreateResponse: + type: object + properties: + data: + $ref: '#/components/schemas/MuteRuleCreate' + required: + - data + MuteRuleResponse: + type: object + properties: + data: + $ref: '#/components/schemas/MuteRule' + required: + - data OpenApiResponseResponse: type: object properties: @@ -11975,6 +13198,38 @@ components: readOnly: true required: - id + OverviewProviderCount: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - providers-count-overview + id: {} + attributes: + type: object + properties: + id: + type: string + count: + type: integer + required: + - id + - count + OverviewProviderCountResponse: + type: object + properties: + data: + $ref: '#/components/schemas/OverviewProviderCount' + required: + - data OverviewProviderResponse: type: object properties: @@ -12166,6 +13421,15 @@ components: $ref: '#/components/schemas/Membership' required: - data + PaginatedMuteRuleList: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/MuteRule' + required: + - data PaginatedProcessorList: type: object properties: @@ -12644,19 +13908,67 @@ components: provider_type: enum: - openai + - bedrock + - openai_compatible type: string - x-spec-enum-id: 80350f3ab09becd9 + x-spec-enum-id: 30bc523cb9ec0e8b readOnly: true description: |- LLM provider name * `openai` - OpenAI + * `bedrock` - AWS Bedrock + * `openai_compatible` - OpenAI Compatible base_url: type: string format: uri nullable: true - maxLength: 200 + minLength: 1 + description: Base URL for the LLM provider API. Required for 'openai_compatible' + provider type. credentials: + oneOf: + - type: object + title: OpenAI Credentials + properties: + api_key: + type: string + description: OpenAI API key. Must start with 'sk-' followed + by alphanumeric characters, hyphens, or underscores. + pattern: ^sk-[\w-]+$ + required: + - api_key + - type: object + title: AWS Bedrock Credentials + properties: + access_key_id: + type: string + description: AWS access key ID. + pattern: ^AKIA[0-9A-Z]{16}$ + secret_access_key: + type: string + description: AWS secret access key. + pattern: ^[A-Za-z0-9/+=]{40}$ + region: + type: string + description: 'AWS region identifier where Bedrock is available. + Examples: us-east-1, us-west-2, eu-west-1, ap-northeast-1.' + pattern: ^[a-z]{2}-[a-z]+-\d+$ + required: + - access_key_id + - secret_access_key + - region + - type: object + title: OpenAI Compatible Credentials + properties: + api_key: + type: string + description: 'API key for OpenAI-compatible provider. The + format varies by provider. Note: The ''base_url'' field + (separate from credentials) is required when using this + provider type.' + required: + - api_key writeOnly: true is_active: type: boolean @@ -12678,7 +13990,7 @@ components: member is used to describe resource objects that share common attributes and relationships. enum: - - lighthouse-config + - lighthouse-configurations id: type: string format: uuid @@ -12693,6 +14005,44 @@ components: default_models: {} required: - data + PatchedMuteRuleUpdateRequest: + type: object + properties: + data: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - mute-rules + id: + type: string + format: uuid + attributes: + type: object + properties: + name: + type: string + minLength: 3 + description: Human-readable name for this rule + maxLength: 100 + reason: + type: string + minLength: 3 + description: Reason for muting + maxLength: 500 + enabled: + type: boolean + description: Whether this rule is currently enabled + required: + - data PatchedProcessorUpdateRequest: type: object properties: @@ -13163,6 +14513,62 @@ components: required: - github_app_id - github_app_key + - type: object + title: IaC Repository Credentials + properties: + repository_url: + type: string + description: Repository URL to scan for IaC files. + access_token: + type: string + description: Optional access token for private repositories. + required: + - repository_url + - type: object + title: Oracle Cloud Infrastructure (OCI) API Key Credentials + properties: + user: + type: string + description: The OCID of the user to authenticate with. + fingerprint: + type: string + description: The fingerprint of the API signing key. + key_file: + type: string + description: The path to the private key file for API signing. + Either key_file or key_content must be provided. + key_content: + type: string + description: The content of the private key for API signing + (base64 encoded). Either key_file or key_content must be + provided. + tenancy: + type: string + description: The OCID of the tenancy. + region: + type: string + description: The OCI region identifier (e.g., us-ashburn-1, + us-phoenix-1). + pass_phrase: + type: string + description: The passphrase for the private key, if encrypted. + required: + - user + - fingerprint + - tenancy + - region + - type: object + title: MongoDB Atlas API Key + properties: + atlas_public_key: + type: string + description: MongoDB Atlas API public key. + atlas_private_key: + type: string + description: MongoDB Atlas API private key. + required: + - atlas_public_key + - atlas_private_key writeOnly: true required: - secret @@ -14157,6 +15563,9 @@ components: - kubernetes - m365 - github + - mongodbatlas + - iac + - oraclecloud type: string description: |- * `aws` - AWS @@ -14165,7 +15574,10 @@ components: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub - x-spec-enum-id: 4c1e219dad1cc0e7 + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + x-spec-enum-id: eca8c51e6bd28935 uid: type: string title: Unique identifier for the provider, set by the provider @@ -14277,8 +15689,11 @@ components: - kubernetes - m365 - github + - mongodbatlas + - iac + - oraclecloud type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 description: |- Type of provider to create. @@ -14288,6 +15703,9 @@ components: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure uid: type: string title: Unique identifier for the provider, set by the provider @@ -14331,8 +15749,11 @@ components: - kubernetes - m365 - github + - mongodbatlas + - iac + - oraclecloud type: string - x-spec-enum-id: 4c1e219dad1cc0e7 + x-spec-enum-id: eca8c51e6bd28935 description: |- Type of provider to create. @@ -14342,6 +15763,9 @@ components: * `kubernetes` - Kubernetes * `m365` - M365 * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure uid: type: string minLength: 3 @@ -15050,6 +16474,60 @@ components: required: - github_app_id - github_app_key + - type: object + title: IaC Repository Credentials + properties: + repository_url: + type: string + description: Repository URL to scan for IaC files. + access_token: + type: string + description: Optional access token for private repositories. + required: + - repository_url + - type: object + title: Oracle Cloud Infrastructure (OCI) API Key Credentials + properties: + user: + type: string + description: The OCID of the user to authenticate with. + fingerprint: + type: string + description: The fingerprint of the API signing key. + key_file: + type: string + description: The path to the private key file for API signing. + Either key_file or key_content must be provided. + key_content: + type: string + description: The content of the private key for API signing (base64 + encoded). Either key_file or key_content must be provided. + tenancy: + type: string + description: The OCID of the tenancy. + region: + type: string + description: The OCI region identifier (e.g., us-ashburn-1, us-phoenix-1). + pass_phrase: + type: string + description: The passphrase for the private key, if encrypted. + required: + - user + - fingerprint + - tenancy + - region + - type: object + title: MongoDB Atlas API Key + properties: + atlas_public_key: + type: string + description: MongoDB Atlas API public key. + atlas_private_key: + type: string + description: MongoDB Atlas API private key. + required: + - atlas_public_key + - atlas_private_key writeOnly: true required: - secret_type @@ -15319,6 +16797,62 @@ components: required: - github_app_id - github_app_key + - type: object + title: IaC Repository Credentials + properties: + repository_url: + type: string + description: Repository URL to scan for IaC files. + access_token: + type: string + description: Optional access token for private repositories. + required: + - repository_url + - type: object + title: Oracle Cloud Infrastructure (OCI) API Key Credentials + properties: + user: + type: string + description: The OCID of the user to authenticate with. + fingerprint: + type: string + description: The fingerprint of the API signing key. + key_file: + type: string + description: The path to the private key file for API signing. + Either key_file or key_content must be provided. + key_content: + type: string + description: The content of the private key for API signing + (base64 encoded). Either key_file or key_content must be + provided. + tenancy: + type: string + description: The OCID of the tenancy. + region: + type: string + description: The OCI region identifier (e.g., us-ashburn-1, + us-phoenix-1). + pass_phrase: + type: string + description: The passphrase for the private key, if encrypted. + required: + - user + - fingerprint + - tenancy + - region + - type: object + title: MongoDB Atlas API Key + properties: + atlas_public_key: + type: string + description: MongoDB Atlas API public key. + atlas_private_key: + type: string + description: MongoDB Atlas API private key. + required: + - atlas_public_key + - atlas_private_key writeOnly: true required: - secret_type @@ -15604,6 +17138,60 @@ components: required: - github_app_id - github_app_key + - type: object + title: IaC Repository Credentials + properties: + repository_url: + type: string + description: Repository URL to scan for IaC files. + access_token: + type: string + description: Optional access token for private repositories. + required: + - repository_url + - type: object + title: Oracle Cloud Infrastructure (OCI) API Key Credentials + properties: + user: + type: string + description: The OCID of the user to authenticate with. + fingerprint: + type: string + description: The fingerprint of the API signing key. + key_file: + type: string + description: The path to the private key file for API signing. + Either key_file or key_content must be provided. + key_content: + type: string + description: The content of the private key for API signing (base64 + encoded). Either key_file or key_content must be provided. + tenancy: + type: string + description: The OCID of the tenancy. + region: + type: string + description: The OCI region identifier (e.g., us-ashburn-1, us-phoenix-1). + pass_phrase: + type: string + description: The passphrase for the private key, if encrypted. + required: + - user + - fingerprint + - tenancy + - region + - type: object + title: MongoDB Atlas API Key + properties: + atlas_public_key: + type: string + description: MongoDB Atlas API public key. + atlas_private_key: + type: string + description: MongoDB Atlas API private key. + required: + - atlas_public_key + - atlas_private_key writeOnly: true required: - secret @@ -15692,6 +17280,18 @@ components: failed_findings_count: type: integer readOnly: true + metadata: + type: string + readOnly: true + nullable: true + details: + type: string + readOnly: true + nullable: true + partition: + type: string + readOnly: true + nullable: true type: type: string readOnly: true @@ -16657,7 +18257,7 @@ components: type: object properties: data: - $ref: '#/components/schemas/ProviderGroup' + $ref: '#/components/schemas/MuteRule' required: - data Task: @@ -17164,6 +18764,143 @@ components: $ref: '#/components/schemas/Tenant' required: - data + ThreatScoreSnapshot: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - threatscore-snapshots + id: {} + attributes: + type: object + properties: + id: + type: string + readOnly: true + inserted_at: + type: string + format: date-time + readOnly: true + compliance_id: + type: string + readOnly: true + description: Compliance framework ID (e.g., 'prowler_threatscore_aws') + overall_score: + type: string + format: decimal + pattern: ^-?\d{0,3}(?:\.\d{0,2})?$ + readOnly: true + description: Overall ThreatScore percentage (0-100) + score_delta: + type: string + format: decimal + pattern: ^-?\d{0,3}(?:\.\d{0,2})?$ + readOnly: true + nullable: true + description: Score change compared to previous snapshot (positive = + improvement) + section_scores: + readOnly: true + description: ThreatScore breakdown by section + critical_requirements: + readOnly: true + description: List of critical failed requirements (risk >= 4) + total_requirements: + type: integer + readOnly: true + description: Total number of requirements evaluated + passed_requirements: + type: integer + readOnly: true + description: Number of requirements with PASS status + failed_requirements: + type: integer + readOnly: true + description: Number of requirements with FAIL status + manual_requirements: + type: integer + readOnly: true + description: Number of requirements with MANUAL status + total_findings: + type: integer + readOnly: true + description: Total number of findings across all requirements + passed_findings: + type: integer + readOnly: true + description: Number of findings with PASS status + failed_findings: + type: integer + readOnly: true + description: Number of findings with FAIL status + relationships: + type: object + properties: + scan: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - scans + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + provider: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - providers + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + ThreatScoreSnapshotResponse: + type: object + properties: + data: + $ref: '#/components/schemas/ThreatScoreSnapshot' + required: + - data Token: type: object required: @@ -17701,3 +19438,7 @@ tags: - name: API Keys description: Endpoints for API keys management. These can be used as an alternative to JWT authorization. +- name: Mute Rules + description: Endpoints for simple mute rules management. These can be used as an + alternative to the Mutelist Processor if you need to mute specific findings across + your tenant with a specific reason. diff --git a/api/src/backend/api/tests/integration/test_rls_transaction.py b/api/src/backend/api/tests/integration/test_rls_transaction.py new file mode 100644 index 0000000000..6731b39d71 --- /dev/null +++ b/api/src/backend/api/tests/integration/test_rls_transaction.py @@ -0,0 +1,39 @@ +"""Tests for rls_transaction retry and fallback logic.""" + +import pytest +from django.db import DEFAULT_DB_ALIAS +from rest_framework_json_api.serializers import ValidationError + +from api.db_utils import rls_transaction + + +@pytest.mark.django_db +class TestRLSTransaction: + """Simple integration tests for rls_transaction using real DB.""" + + @pytest.fixture + def tenant(self, tenants_fixture): + return tenants_fixture[0] + + def test_success_on_primary(self, tenant): + """Basic: transaction succeeds on primary database.""" + with rls_transaction(str(tenant.id), using=DEFAULT_DB_ALIAS) as cursor: + cursor.execute("SELECT 1") + result = cursor.fetchone() + assert result == (1,) + + def test_invalid_uuid_raises_validation_error(self): + """Invalid UUID raises ValidationError before DB operations.""" + with pytest.raises(ValidationError, match="Must be a valid UUID"): + with rls_transaction("not-a-uuid", using=DEFAULT_DB_ALIAS): + pass + + def test_custom_parameter_name(self, tenant): + """Test custom RLS parameter name.""" + custom_param = "api.custom_id" + with rls_transaction( + str(tenant.id), parameter=custom_param, using=DEFAULT_DB_ALIAS + ) as cursor: + cursor.execute("SELECT current_setting(%s, true)", [custom_param]) + result = cursor.fetchone() + assert result == (str(tenant.id),) diff --git a/api/src/backend/api/tests/test_db_utils.py b/api/src/backend/api/tests/test_db_utils.py index b4ec73e715..f706ab8c71 100644 --- a/api/src/backend/api/tests/test_db_utils.py +++ b/api/src/backend/api/tests/test_db_utils.py @@ -1,12 +1,15 @@ from datetime import datetime, timezone from enum import Enum -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from django.conf import settings +from django.db import DEFAULT_DB_ALIAS, OperationalError from freezegun import freeze_time +from rest_framework_json_api.serializers import ValidationError from api.db_utils import ( + POSTGRES_TENANT_VAR, _should_create_index_on_partition, batch_delete, create_objects_in_batches, @@ -14,11 +17,22 @@ from api.db_utils import ( generate_api_key_prefix, generate_random_token, one_week_from_now, + rls_transaction, update_objects_in_batches, ) from api.models import Provider +@pytest.fixture +def enable_read_replica(): + """ + Fixture to enable READ_REPLICA_ALIAS for tests that need replica functionality. + This avoids polluting the global test configuration. + """ + with patch("api.db_utils.READ_REPLICA_ALIAS", "replica"): + yield "replica" + + class TestEnumToChoices: def test_enum_to_choices_simple(self): class Color(Enum): @@ -339,3 +353,498 @@ class TestGenerateApiKeyPrefix: prefix = generate_api_key_prefix() random_part = prefix[3:] # Strip 'pk_' assert all(char in allowed_chars for char in random_part) + + +@pytest.mark.django_db +class TestRlsTransaction: + def test_rls_transaction_valid_uuid_string(self, tenants_fixture): + """Test rls_transaction with valid UUID string.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with rls_transaction(tenant_id) as cursor: + assert cursor is not None + cursor.execute("SELECT current_setting(%s)", [POSTGRES_TENANT_VAR]) + result = cursor.fetchone() + assert result[0] == tenant_id + + def test_rls_transaction_valid_uuid_object(self, tenants_fixture): + """Test rls_transaction with UUID object.""" + tenant = tenants_fixture[0] + + with rls_transaction(tenant.id) as cursor: + assert cursor is not None + cursor.execute("SELECT current_setting(%s)", [POSTGRES_TENANT_VAR]) + result = cursor.fetchone() + assert result[0] == str(tenant.id) + + def test_rls_transaction_invalid_uuid_raises_validation_error(self): + """Test rls_transaction raises ValidationError for invalid UUID.""" + invalid_uuid = "not-a-valid-uuid" + + with pytest.raises(ValidationError, match="Must be a valid UUID"): + with rls_transaction(invalid_uuid): + pass + + def test_rls_transaction_uses_default_database_when_no_alias(self, tenants_fixture): + """Test rls_transaction uses DEFAULT_DB_ALIAS when no alias specified.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=None): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with rls_transaction(tenant_id): + pass + + mock_connections.__getitem__.assert_called_with(DEFAULT_DB_ALIAS) + + def test_rls_transaction_uses_specified_alias(self, tenants_fixture): + """Test rls_transaction uses specified database alias via using parameter.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + custom_alias = "custom_db" + + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias") as mock_set_alias: + with patch("api.db_utils.reset_read_db_alias") as mock_reset_alias: + mock_set_alias.return_value = "test_token" + with rls_transaction(tenant_id, using=custom_alias): + pass + + mock_connections.__getitem__.assert_called_with(custom_alias) + mock_set_alias.assert_called_once_with(custom_alias) + mock_reset_alias.assert_called_once_with("test_token") + + def test_rls_transaction_uses_read_replica_from_router( + self, tenants_fixture, enable_read_replica + ): + """Test rls_transaction uses read replica alias from router.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias") as mock_set_alias: + with patch( + "api.db_utils.reset_read_db_alias" + ) as mock_reset_alias: + mock_set_alias.return_value = "test_token" + with rls_transaction(tenant_id): + pass + + mock_connections.__getitem__.assert_called() + mock_set_alias.assert_called_once() + mock_reset_alias.assert_called_once() + + def test_rls_transaction_fallback_to_default_when_alias_not_in_connections( + self, tenants_fixture + ): + """Test rls_transaction falls back to DEFAULT_DB_ALIAS when alias not in connections.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + invalid_alias = "nonexistent_db" + + with patch("api.db_utils.get_read_db_alias", return_value=invalid_alias): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + + def contains_check(alias): + return alias == DEFAULT_DB_ALIAS + + mock_connections.__contains__.side_effect = contains_check + mock_connections.__getitem__.return_value = mock_conn + + with patch("api.db_utils.transaction.atomic"): + with rls_transaction(tenant_id): + pass + + mock_connections.__getitem__.assert_called_with(DEFAULT_DB_ALIAS) + + def test_rls_transaction_successful_execution_on_replica_no_retries( + self, tenants_fixture, enable_read_replica + ): + """Test successful execution on replica without retries.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias", return_value="token"): + with patch("api.db_utils.reset_read_db_alias"): + with rls_transaction(tenant_id): + pass + + assert mock_cursor.execute.call_count == 1 + + def test_rls_transaction_retry_with_exponential_backoff_on_operational_error( + self, tenants_fixture, enable_read_replica + ): + """Test retry with exponential backoff on OperationalError on replica.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + call_count = 0 + + def atomic_side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise OperationalError("Connection error") + return MagicMock( + __enter__=MagicMock(return_value=None), + __exit__=MagicMock(return_value=False), + ) + + with patch( + "api.db_utils.transaction.atomic", side_effect=atomic_side_effect + ): + with patch("api.db_utils.time.sleep") as mock_sleep: + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with patch("api.db_utils.logger") as mock_logger: + with rls_transaction(tenant_id): + pass + + assert mock_sleep.call_count == 2 + mock_sleep.assert_any_call(0.5) + mock_sleep.assert_any_call(1.0) + assert mock_logger.info.call_count == 2 + + def test_rls_transaction_max_three_attempts_for_replica( + self, tenants_fixture, enable_read_replica + ): + """Test maximum 3 attempts for replica database.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = OperationalError("Persistent error") + + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with pytest.raises(OperationalError): + with rls_transaction(tenant_id): + pass + + assert mock_atomic.call_count == 3 + + def test_rls_transaction_only_one_attempt_for_primary(self, tenants_fixture): + """Test only 1 attempt for primary database.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=None): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = OperationalError("Primary error") + + with pytest.raises(OperationalError): + with rls_transaction(tenant_id): + pass + + assert mock_atomic.call_count == 1 + + def test_rls_transaction_fallback_to_primary_after_max_attempts( + self, tenants_fixture, enable_read_replica + ): + """Test fallback to primary DB after max attempts on replica.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + call_count = 0 + + def atomic_side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise OperationalError("Replica error") + return MagicMock( + __enter__=MagicMock(return_value=None), + __exit__=MagicMock(return_value=False), + ) + + with patch( + "api.db_utils.transaction.atomic", side_effect=atomic_side_effect + ): + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with patch("api.db_utils.logger") as mock_logger: + with rls_transaction(tenant_id): + pass + + mock_logger.warning.assert_called_once() + warning_msg = mock_logger.warning.call_args[0][0] + assert "falling back to primary DB" in warning_msg + + def test_rls_transaction_logger_warning_on_fallback( + self, tenants_fixture, enable_read_replica + ): + """Test logger warnings are emitted on fallback to primary.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + call_count = 0 + + def atomic_side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise OperationalError("Replica error") + return MagicMock( + __enter__=MagicMock(return_value=None), + __exit__=MagicMock(return_value=False), + ) + + with patch( + "api.db_utils.transaction.atomic", side_effect=atomic_side_effect + ): + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with patch("api.db_utils.logger") as mock_logger: + with rls_transaction(tenant_id): + pass + + assert mock_logger.info.call_count == 2 + assert mock_logger.warning.call_count == 1 + + def test_rls_transaction_operational_error_raised_immediately_on_primary( + self, tenants_fixture + ): + """Test OperationalError raised immediately on primary without retry.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=None): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = OperationalError("Primary error") + + with patch("api.db_utils.time.sleep") as mock_sleep: + with pytest.raises(OperationalError): + with rls_transaction(tenant_id): + pass + + mock_sleep.assert_not_called() + + def test_rls_transaction_operational_error_raised_after_max_attempts( + self, tenants_fixture, enable_read_replica + ): + """Test OperationalError raised after max attempts on replica.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=enable_read_replica): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = OperationalError( + "Persistent replica error" + ) + + with patch("api.db_utils.time.sleep"): + with patch( + "api.db_utils.set_read_db_alias", return_value="token" + ): + with patch("api.db_utils.reset_read_db_alias"): + with pytest.raises(OperationalError): + with rls_transaction(tenant_id): + pass + + def test_rls_transaction_router_token_set_for_non_default_alias( + self, tenants_fixture + ): + """Test router token is set when using non-default alias.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + custom_alias = "custom_db" + + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias") as mock_set_alias: + with patch("api.db_utils.reset_read_db_alias") as mock_reset_alias: + mock_set_alias.return_value = "test_token" + with rls_transaction(tenant_id, using=custom_alias): + pass + + mock_set_alias.assert_called_once_with(custom_alias) + mock_reset_alias.assert_called_once_with("test_token") + + def test_rls_transaction_router_token_reset_in_finally_block(self, tenants_fixture): + """Test router token is reset in finally block even on error.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + custom_alias = "custom_db" + + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic") as mock_atomic: + mock_atomic.side_effect = Exception("Unexpected error") + + with patch("api.db_utils.set_read_db_alias", return_value="test_token"): + with patch("api.db_utils.reset_read_db_alias") as mock_reset_alias: + with pytest.raises(Exception): + with rls_transaction(tenant_id, using=custom_alias): + pass + + mock_reset_alias.assert_called_once_with("test_token") + + def test_rls_transaction_router_token_not_set_for_default_alias( + self, tenants_fixture + ): + """Test router token is not set when using default alias.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with patch("api.db_utils.get_read_db_alias", return_value=None): + with patch("api.db_utils.connections") as mock_connections: + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_connections.__getitem__.return_value = mock_conn + mock_connections.__contains__.return_value = True + + with patch("api.db_utils.transaction.atomic"): + with patch("api.db_utils.set_read_db_alias") as mock_set_alias: + with patch( + "api.db_utils.reset_read_db_alias" + ) as mock_reset_alias: + with rls_transaction(tenant_id): + pass + + mock_set_alias.assert_not_called() + mock_reset_alias.assert_not_called() + + def test_rls_transaction_set_config_query_executed_with_correct_params( + self, tenants_fixture + ): + """Test SET_CONFIG_QUERY executed with correct parameters.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with rls_transaction(tenant_id) as cursor: + cursor.execute("SELECT current_setting(%s)", [POSTGRES_TENANT_VAR]) + result = cursor.fetchone() + assert result[0] == tenant_id + + def test_rls_transaction_custom_parameter(self, tenants_fixture): + """Test rls_transaction with custom parameter name.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + custom_param = "api.user_id" + + with rls_transaction(tenant_id, parameter=custom_param) as cursor: + cursor.execute("SELECT current_setting(%s)", [custom_param]) + result = cursor.fetchone() + assert result[0] == tenant_id + + def test_rls_transaction_cursor_yielded_correctly(self, tenants_fixture): + """Test cursor is yielded correctly.""" + tenant = tenants_fixture[0] + tenant_id = str(tenant.id) + + with rls_transaction(tenant_id) as cursor: + assert cursor is not None + cursor.execute("SELECT 1") + result = cursor.fetchone() + assert result[0] == 1 diff --git a/api/src/backend/api/tests/test_utils.py b/api/src/backend/api/tests/test_utils.py index 0b36cdf9fe..d67df93ff5 100644 --- a/api/src/backend/api/tests/test_utils.py +++ b/api/src/backend/api/tests/test_utils.py @@ -20,8 +20,11 @@ from prowler.providers.aws.aws_provider import AwsProvider from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection from prowler.providers.azure.azure_provider import AzureProvider from prowler.providers.gcp.gcp_provider import GcpProvider +from prowler.providers.github.github_provider import GithubProvider from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider from prowler.providers.m365.m365_provider import M365Provider +from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider +from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider class TestMergeDicts: @@ -108,6 +111,9 @@ class TestReturnProwlerProvider: (Provider.ProviderChoices.AZURE.value, AzureProvider), (Provider.ProviderChoices.KUBERNETES.value, KubernetesProvider), (Provider.ProviderChoices.M365.value, M365Provider), + (Provider.ProviderChoices.GITHUB.value, GithubProvider), + (Provider.ProviderChoices.MONGODBATLAS.value, MongodbatlasProvider), + (Provider.ProviderChoices.ORACLECLOUD.value, OraclecloudProvider), ], ) def test_return_prowler_provider(self, provider_type, expected_provider): @@ -203,6 +209,14 @@ class TestGetProwlerProviderKwargs: Provider.ProviderChoices.GITHUB.value, {"organizations": ["provider_uid"]}, ), + ( + Provider.ProviderChoices.ORACLECLOUD.value, + {}, + ), + ( + Provider.ProviderChoices.MONGODBATLAS.value, + {"atlas_organization_id": "provider_uid"}, + ), ], ) def test_get_prowler_provider_kwargs(self, provider_type, expected_extra_kwargs): diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 94ce2d0ed2..e5eaa2f836 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -4,6 +4,7 @@ import json import os import tempfile from datetime import datetime, timedelta, timezone +from decimal import Decimal from pathlib import Path from types import SimpleNamespace from unittest.mock import ANY, MagicMock, Mock, patch @@ -23,6 +24,7 @@ from conftest import ( today_after_n_days, ) from django.conf import settings +from django.db.models import Count from django.http import JsonResponse from django.test import RequestFactory from django.urls import reverse @@ -33,6 +35,9 @@ from rest_framework.response import Response from api.compliance import get_compliance_frameworks from api.db_router import MainRouter from api.models import ( + ComplianceOverviewSummary, + ComplianceRequirementOverview, + Finding, Integration, Invitation, LighthouseProviderConfiguration, @@ -44,6 +49,7 @@ from api.models import ( ProviderGroup, ProviderGroupMembership, ProviderSecret, + Resource, Role, RoleProviderGroupRelationship, SAMLConfiguration, @@ -51,14 +57,18 @@ from api.models import ( Scan, ScanSummary, StateChoices, + StatusChoices, Task, TenantAPIKey, + ThreatScoreSnapshot, User, UserRoleRelationship, ) from api.rls import Tenant from api.v1.serializers import TokenSerializer from api.v1.views import ComplianceOverviewViewSet, TenantFinishACSView +from prowler.lib.check.models import Severity +from prowler.lib.outputs.finding import Status class TestViewSet: @@ -946,6 +956,74 @@ class TestProviderViewSet: assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == len(providers_fixture) + def test_providers_filter_provider_type( + self, authenticated_client, providers_fixture + ): + response = authenticated_client.get( + reverse("provider-list"), {"filter[provider_type]": "aws"} + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + assert all(item["attributes"]["provider"] == "aws" for item in data) + + def test_providers_filter_provider_type_in( + self, authenticated_client, providers_fixture + ): + response = authenticated_client.get( + reverse("provider-list"), {"filter[provider_type__in]": "aws,gcp"} + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 3 + assert {"aws", "gcp"} >= {item["attributes"]["provider"] for item in data} + + def test_providers_filter_provider_type_invalid( + self, authenticated_client, providers_fixture + ): + response = authenticated_client.get( + reverse("provider-list"), {"filter[provider_type]": "invalid"} + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_providers_disable_pagination( + self, authenticated_client, providers_fixture, tenants_fixture + ): + tenant, *_ = tenants_fixture + existing_count = Provider.objects.filter(tenant_id=tenant.id).count() + target_total = settings.REST_FRAMEWORK["PAGE_SIZE"] + 1 + additional_needed = max(0, target_total - existing_count) + + base_uid = 200000000000 + for index in range(additional_needed): + Provider.objects.create( + tenant_id=tenant.id, + provider=Provider.ProviderChoices.AWS, + uid=f"{base_uid + index:012d}", + alias=f"aws_extra_{index}", + ) + + total_providers = Provider.objects.filter(tenant_id=tenant.id).count() + + paginated_response = authenticated_client.get(reverse("provider-list")) + assert paginated_response.status_code == status.HTTP_200_OK + paginated_data = paginated_response.json()["data"] + assert len(paginated_data) == min( + settings.REST_FRAMEWORK["PAGE_SIZE"], total_providers + ) + paginated_meta = paginated_response.json().get("meta", {}) + assert "pagination" in paginated_meta + assert paginated_meta["pagination"]["count"] == total_providers + + unpaginated_response = authenticated_client.get( + reverse("provider-list"), {"page[disable]": "true"} + ) + assert unpaginated_response.status_code == status.HTTP_200_OK + unpaginated_data = unpaginated_response.json()["data"] + assert len(unpaginated_data) == total_providers + unpaginated_meta = unpaginated_response.json().get("meta", {}) + assert "pagination" not in unpaginated_meta + @pytest.mark.parametrize( "include_values, expected_resources", [ @@ -1070,6 +1148,21 @@ class TestProviderViewSet: "uid": "a12345678901234567890123456789012345678", "alias": "Long Username", }, + { + "provider": "iac", + "uid": "https://github.com/user/repo.git", + "alias": "Git Repo", + }, + { + "provider": "iac", + "uid": "https://gitlab.com/user/project", + "alias": "GitLab Repo", + }, + { + "provider": "mongodbatlas", + "uid": "64b1d3c0e4b03b1234567890", + "alias": "Atlas Organization", + }, ] ), ) @@ -1083,6 +1176,161 @@ class TestProviderViewSet: assert Provider.objects.get().uid == provider_json_payload["uid"] assert Provider.objects.get().alias == provider_json_payload["alias"] + @pytest.mark.parametrize( + "provider_json_payload", + ( + [ + {"provider": "aws", "uid": "111111111111", "alias": "test"}, + {"provider": "gcp", "uid": "a12322-test54321", "alias": "test"}, + { + "provider": "kubernetes", + "uid": "kubernetes-test-123456789", + "alias": "test", + }, + { + "provider": "kubernetes", + "uid": "arn:aws:eks:us-east-1:111122223333:cluster/test-cluster-long-name-123456789", + "alias": "EKS", + }, + { + "provider": "kubernetes", + "uid": "gke_aaaa-dev_europe-test1_dev-aaaa-test-cluster-long-name-123456789", + "alias": "GKE", + }, + { + "provider": "kubernetes", + "uid": "gke_project/cluster-name", + "alias": "GKE", + }, + { + "provider": "kubernetes", + "uid": "admin@k8s-demo", + "alias": "test", + }, + { + "provider": "azure", + "uid": "8851db6b-42e5-4533-aa9e-30a32d67e875", + "alias": "test", + }, + { + "provider": "m365", + "uid": "TestingPro.onmicrosoft.com", + "alias": "test", + }, + { + "provider": "m365", + "uid": "subdomain.domain.es", + "alias": "test", + }, + { + "provider": "m365", + "uid": "microsoft.net", + "alias": "test", + }, + { + "provider": "m365", + "uid": "subdomain1.subdomain2.subdomain3.subdomain4.domain.net", + "alias": "test", + }, + { + "provider": "github", + "uid": "test-user", + "alias": "test", + }, + { + "provider": "github", + "uid": "test-organization", + "alias": "GitHub Org", + }, + { + "provider": "github", + "uid": "prowler-cloud", + "alias": "Prowler", + }, + { + "provider": "github", + "uid": "microsoft", + "alias": "Microsoft", + }, + { + "provider": "github", + "uid": "a12345678901234567890123456789012345678", + "alias": "Long Username", + }, + ] + ), + ) + @patch("api.v1.views.Task.objects.get") + @patch("api.v1.views.delete_provider_task.delay") + def test_providers_soft_delete( + self, + mock_delete_task, + mock_task_get, + authenticated_client, + provider_json_payload, + tasks_fixture, + ): + # Mock the Celery task response + prowler_task = tasks_fixture[0] + task_mock = Mock() + task_mock.id = prowler_task.id + mock_delete_task.return_value = task_mock + mock_task_get.return_value = prowler_task + + # 1.Create a provider + response = authenticated_client.post( + reverse("provider-list"), data=provider_json_payload, format="json" + ) + assert response.status_code == status.HTTP_201_CREATED + assert Provider.objects.count() == 1 + provider_id = response.json()["data"]["id"] + + # 2. Soft delete the provider using the actual API endpoint + response = authenticated_client.delete( + reverse("provider-detail", kwargs={"pk": provider_id}) + ) + assert response.status_code == status.HTTP_202_ACCEPTED + assert Provider.objects.count() == 0 + assert Provider.all_objects.count() == 1 + + mock_delete_task.assert_called_once_with( + provider_id=str(provider_id), tenant_id=ANY + ) + + # 3. Create a provider with the same UID should succeed (since the old one is soft deleted) + response = authenticated_client.post( + reverse("provider-list"), data=provider_json_payload, format="json" + ) + assert response.status_code == status.HTTP_201_CREATED + assert Provider.objects.count() == 1 + assert Provider.all_objects.count() == 2 + provider_id = response.json()["data"]["id"] + + # 4. Creating another provider with the same UID should fail (duplicate) + response = authenticated_client.post( + reverse("provider-list"), data=provider_json_payload, format="json" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + mock_delete_task.reset_mock() + mock_delete_task.return_value = task_mock + + # 5. Delete the second provider + response = authenticated_client.delete( + reverse("provider-detail", kwargs={"pk": provider_id}) + ) + assert response.status_code == status.HTTP_202_ACCEPTED + assert Provider.objects.count() == 0 + assert Provider.all_objects.count() == 2 + + # 6. Creating a provider with the same UID should succeed again + response = authenticated_client.post( + reverse("provider-list"), data=provider_json_payload, format="json" + ) + assert response.status_code == status.HTTP_201_CREATED + assert Provider.objects.count() == 1 + assert Provider.all_objects.count() == 3 + @pytest.mark.parametrize( "provider_json_payload, error_code, error_pointer", ( @@ -1219,6 +1467,51 @@ class TestProviderViewSet: "github-uid", "uid", ), + ( + { + "provider": "iac", + "uid": "not-a-url", + "alias": "test", + }, + "iac-uid", + "uid", + ), + ( + { + "provider": "iac", + "uid": "ftp://invalid-protocol.com/repo", + "alias": "test", + }, + "iac-uid", + "uid", + ), + ( + { + "provider": "iac", + "uid": "http://", + "alias": "test", + }, + "iac-uid", + "uid", + ), + ( + { + "provider": "mongodbatlas", + "uid": "64b1d3c0e4b03b123456789g", + "alias": "test", + }, + "mongodbatlas-uid", + "uid", + ), + ( + { + "provider": "mongodbatlas", + "uid": "1234", + "alias": "test", + }, + "mongodbatlas-uid", + "uid", + ), ] ), ) @@ -1389,13 +1682,25 @@ class TestProviderViewSet: ("provider", "aws", 2), ("provider.in", "azure,gcp", 2), ("uid", "123456789012", 1), - ("uid.icontains", "1", 5), + ( + "uid.icontains", + "1", + 7, + ), ("alias", "aws_testing_1", 1), ("alias.icontains", "aws", 2), - ("inserted_at", TODAY, 6), - ("inserted_at.gte", "2024-01-01", 6), + ("inserted_at", TODAY, 8), + ( + "inserted_at.gte", + "2024-01-01", + 8, + ), ("inserted_at.lte", "2024-01-01", 0), - ("updated_at.gte", "2024-01-01", 6), + ( + "updated_at.gte", + "2024-01-01", + 8, + ), ("updated_at.lte", "2024-01-01", 0), ] ), @@ -1898,6 +2203,52 @@ class TestProviderSecretViewSet: "password": "supersecret", }, ), + # OCI with API key credentials (with key_content) + ( + Provider.ProviderChoices.ORACLECLOUD.value, + ProviderSecret.TypeChoices.STATIC, + { + "user": "ocid1.user.oc1..aaaaaaaakldibrbov4ubh25aqdeiroklxjngwka7u6w7no3glmdq3n5sxtkq", + "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", + "key_content": "-----BEGIN RSA PRIVATE KEY-----\ntest-key-content\n-----END RSA PRIVATE KEY-----", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", + "region": "us-ashburn-1", + }, + ), + # OCI with API key credentials (with key_file) + ( + Provider.ProviderChoices.ORACLECLOUD.value, + ProviderSecret.TypeChoices.STATIC, + { + "user": "ocid1.user.oc1..aaaaaaaakldibrbov4ubh25aqdeiroklxjngwka7u6w7no3glmdq3n5sxtkq", + "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", + "key_file": "/path/to/oci_api_key.pem", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", + "region": "us-ashburn-1", + }, + ), + # OCI with API key credentials (with passphrase) + ( + Provider.ProviderChoices.ORACLECLOUD.value, + ProviderSecret.TypeChoices.STATIC, + { + "user": "ocid1.user.oc1..aaaaaaaakldibrbov4ubh25aqdeiroklxjngwka7u6w7no3glmdq3n5sxtkq", + "fingerprint": "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99", + "key_content": "-----BEGIN RSA PRIVATE KEY-----\ntest-encrypted-key\n-----END RSA PRIVATE KEY-----", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", + "region": "us-ashburn-1", + "pass_phrase": "my-secure-passphrase", + }, + ), + # MongoDB Atlas credentials + ( + Provider.ProviderChoices.MONGODBATLAS.value, + ProviderSecret.TypeChoices.STATIC, + { + "atlas_public_key": "public-key", + "atlas_private_key": "private-key", + }, + ), ], ) def test_provider_secrets_create_valid( @@ -3194,6 +3545,9 @@ class TestResourceViewSet: ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == len(resources_fixture) + assert "metadata" in response.json()["data"][0]["attributes"] + assert "details" in response.json()["data"][0]["attributes"] + assert "partition" in response.json()["data"][0]["attributes"] @pytest.mark.parametrize( "include_values, expected_resources", @@ -5466,16 +5820,44 @@ class TestProviderGroupMembershipViewSet: @pytest.mark.django_db class TestComplianceOverviewViewSet: - def test_compliance_overview_list_none(self, authenticated_client): + @pytest.fixture(autouse=True) + def mock_backfill_task(self): + with patch("api.v1.views.backfill_compliance_summaries_task.delay") as mock: + yield mock + + def test_compliance_overview_list_none( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + mock_backfill_task, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + scan = Scan.objects.create( + name="empty-compliance-scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + response = authenticated_client.get( reverse("complianceoverview-list"), - {"filter[scan_id]": "8d20ac7d-4cbc-435e-85f4-359be37af821"}, + {"filter[scan_id]": str(scan.id)}, ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == 0 + mock_backfill_task.assert_called_once() + _, kwargs = mock_backfill_task.call_args + assert kwargs["scan_id"] == str(scan.id) + assert str(kwargs["tenant_id"]) == str(tenant.id) def test_compliance_overview_list( - self, authenticated_client, compliance_requirements_overviews_fixture + self, + authenticated_client, + compliance_requirements_overviews_fixture, + mock_backfill_task, ): # List compliance overviews with existing data requirement_overview1 = compliance_requirements_overviews_fixture[0] @@ -5505,6 +5887,112 @@ class TestComplianceOverviewViewSet: assert "requirements_failed" in attributes assert "requirements_manual" in attributes assert "total_requirements" in attributes + mock_backfill_task.assert_called_once() + _, kwargs = mock_backfill_task.call_args + assert kwargs["scan_id"] == scan_id + + def test_compliance_overview_list_uses_preaggregated_summaries( + self, + authenticated_client, + tenants_fixture, + providers_fixture, + mock_backfill_task, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + scan = Scan.objects.create( + name="preaggregated-scan", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + ComplianceRequirementOverview.objects.create( + tenant=tenant, + scan=scan, + compliance_id="cis_1.4_aws", + framework="CIS-1.4-AWS", + version="1.4", + description="CIS AWS Foundations Benchmark v1.4.0", + region="eu-west-1", + requirement_id="framework-metadata", + requirement_status=StatusChoices.PASS, + passed_checks=1, + failed_checks=0, + total_checks=1, + ) + + ComplianceOverviewSummary.objects.create( + tenant=tenant, + scan=scan, + compliance_id="cis_1.4_aws", + requirements_passed=5, + requirements_failed=1, + requirements_manual=2, + total_requirements=8, + ) + + response = authenticated_client.get( + reverse("complianceoverview-list"), + {"filter[scan_id]": str(scan.id)}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + overview = data[0] + assert overview["id"] == "cis_1.4_aws" + assert overview["attributes"]["requirements_passed"] == 5 + assert overview["attributes"]["requirements_failed"] == 1 + assert overview["attributes"]["requirements_manual"] == 2 + assert overview["attributes"]["total_requirements"] == 8 + assert "framework" in overview["attributes"] + assert "version" in overview["attributes"] + mock_backfill_task.assert_not_called() + + def test_compliance_overview_region_filter_skips_backfill( + self, + authenticated_client, + compliance_requirements_overviews_fixture, + mock_backfill_task, + ): + requirement_overview = compliance_requirements_overviews_fixture[0] + scan_id = str(requirement_overview.scan.id) + + response = authenticated_client.get( + reverse("complianceoverview-list"), + { + "filter[scan_id]": scan_id, + "filter[region]": requirement_overview.region, + }, + ) + + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) >= 1 + mock_backfill_task.assert_not_called() + + def test_compliance_overview_list_without_scan_id( + self, authenticated_client, compliance_requirements_overviews_fixture + ): + # Ensure the endpoint works without passing a scan filter + response = authenticated_client.get(reverse("complianceoverview-list")) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 3 + + # Validate payload structure + first_item = data[0] + assert "id" in first_item + assert "attributes" in first_item + attributes = first_item["attributes"] + assert "framework" in attributes + assert "version" in attributes + assert "requirements_passed" in attributes + assert "requirements_failed" in attributes + assert "requirements_manual" in attributes + assert "total_requirements" in attributes def test_compliance_overview_metadata( self, authenticated_client, compliance_requirements_overviews_fixture @@ -5658,6 +6146,11 @@ class TestComplianceOverviewViewSet: requirement_overview1 = compliance_requirements_overviews_fixture[0] scan_id = str(requirement_overview1.scan.id) + # Remove existing compliance data so the view falls back to task checks + scan = requirement_overview1.scan + ComplianceOverviewSummary.objects.filter(scan=scan).delete() + ComplianceRequirementOverview.objects.filter(scan=scan).delete() + # Mock a running task with patch.object( ComplianceOverviewViewSet, "get_task_response_if_running" @@ -5685,6 +6178,11 @@ class TestComplianceOverviewViewSet: requirement_overview1 = compliance_requirements_overviews_fixture[0] scan_id = str(requirement_overview1.scan.id) + # Remove existing compliance data so the view falls back to task checks + scan = requirement_overview1.scan + ComplianceOverviewSummary.objects.filter(scan=scan).delete() + ComplianceRequirementOverview.objects.filter(scan=scan).delete() + # Mock a failed task with patch.object( ComplianceOverviewViewSet, "get_task_response_if_running" @@ -5708,6 +6206,8 @@ class TestComplianceOverviewViewSet: ("framework", "framework", 1), ("version", "version", 1), ("region", "region", 1), + ("region__in", "region", 1), + ("region.in", "region", 1), ], ) def test_compliance_overview_filters( @@ -5784,14 +6284,505 @@ class TestOverviewViewSet: assert response.json()["data"][0]["attributes"]["findings"]["pass"] == 2 assert response.json()["data"][0]["attributes"]["findings"]["fail"] == 1 assert response.json()["data"][0]["attributes"]["findings"]["muted"] == 1 - # Since we rely on completed scans, there are only 2 resources now - assert response.json()["data"][0]["attributes"]["resources"]["total"] == 2 + # Aggregated resources include all AWS providers present in the tenant + assert response.json()["data"][0]["attributes"]["resources"]["total"] == 3 + + def test_overview_providers_aggregates_same_provider_type( + self, + authenticated_client, + scan_summaries_fixture, + resources_fixture, + providers_fixture, + tenants_fixture, + ): + tenant = tenants_fixture[0] + _provider1, provider2, *_ = providers_fixture + + scan = Scan.objects.create( + name="overview scan aws account 2", + provider=provider2, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant=tenant, + ) + + ScanSummary.objects.create( + tenant=tenant, + scan=scan, + check_id="check-aws-two", + service="service-extra", + severity="medium", + region="region-extra", + _pass=3, + fail=2, + muted=1, + total=6, + ) + + Resource.objects.create( + tenant_id=tenant.id, + provider=provider2, + uid="arn:aws:ec2:us-west-2:123456789013:instance/i-aggregation", + name="Aggregated Instance", + region="us-west-2", + service="ec2", + type="prowler-test", + ) + + response = authenticated_client.get(reverse("overview-providers")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + attributes = data[0]["attributes"] + + assert attributes["findings"]["total"] == 10 + assert attributes["findings"]["pass"] == 5 + assert attributes["findings"]["fail"] == 3 + assert attributes["findings"]["muted"] == 2 + assert attributes["resources"]["total"] == 4 + + def test_overview_providers_count( + self, + authenticated_client, + scan_summaries_fixture, + resources_fixture, + providers_fixture, + tenants_fixture, + ): + tenant = tenants_fixture[0] + + default_response = authenticated_client.get(reverse("overview-providers")) + assert default_response.status_code == status.HTTP_200_OK + default_data = default_response.json()["data"] + assert len(default_data) == 1 + assert all("count" not in item["attributes"] for item in default_data) + grouped_response = authenticated_client.get(reverse("overview-providers-count")) + assert grouped_response.status_code == status.HTTP_200_OK + grouped_data = grouped_response.json()["data"] + assert len(grouped_data) >= 1 + + aggregated = { + entry["id"]: entry["attributes"]["count"] for entry in grouped_data + } + db_counts = ( + Provider.objects.filter(tenant_id=tenant.id, is_deleted=False) + .values("provider") + .annotate(count=Count("id")) + ) + expected = {row["provider"]: row["count"] for row in db_counts} + + assert aggregated == expected + for entry in grouped_data: + assert "findings" not in entry["attributes"] + + def _create_scan(self, tenant, provider, name, started_at=None): + scan_started = started_at or datetime.now(timezone.utc) - timedelta(hours=1) + return Scan.objects.create( + tenant=tenant, + provider=provider, + name=name, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + started_at=scan_started, + completed_at=scan_started + timedelta(minutes=30), + ) + + def _create_threatscore_snapshot( + self, + tenant, + scan, + provider, + *, + compliance_id, + overall_score, + score_delta, + section_scores, + critical_requirements, + total_requirements, + passed_requirements, + failed_requirements, + manual_requirements, + total_findings, + passed_findings, + failed_findings, + ): + return ThreatScoreSnapshot.objects.create( + tenant=tenant, + scan=scan, + provider=provider, + compliance_id=compliance_id, + overall_score=Decimal(overall_score), + score_delta=Decimal(score_delta) if score_delta is not None else None, + section_scores=section_scores, + critical_requirements=critical_requirements, + total_requirements=total_requirements, + passed_requirements=passed_requirements, + failed_requirements=failed_requirements, + manual_requirements=manual_requirements, + total_findings=total_findings, + passed_findings=passed_findings, + failed_findings=failed_findings, + ) + + def test_overview_threatscore_returns_weighted_aggregate_snapshot( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = self._create_scan(tenant, provider1, "agg-scan-one") + scan2 = self._create_scan(tenant, provider2, "agg-scan-two") + + snapshot1 = self._create_threatscore_snapshot( + tenant, + scan1, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="80.00", + score_delta="5.00", + section_scores={"1. IAM": "70.00", "2. Attack Surface": "60.00"}, + critical_requirements=[ + { + "requirement_id": "req_shared", + "title": "Shared requirement (preferred)", + "section": "1. IAM", + "subsection": "Sub IAM", + "risk_level": 5, + "weight": 150, + "passed_findings": 14, + "total_findings": 20, + "description": "Higher risk duplicate", + }, + { + "requirement_id": "req_unique_one", + "title": "Unique provider one", + "section": "2. Attack Surface", + "subsection": "Sub Attack", + "risk_level": 4, + "weight": 90, + "passed_findings": 20, + "total_findings": 30, + "description": "Lower risk", + }, + ], + total_requirements=120, + passed_requirements=90, + failed_requirements=30, + manual_requirements=0, + total_findings=100, + passed_findings=70, + failed_findings=30, + ) + + snapshot2 = self._create_threatscore_snapshot( + tenant, + scan2, + provider2, + compliance_id="prowler_threatscore_aws", + overall_score="20.00", + score_delta="-2.00", + section_scores={ + "1. IAM": "10.00", + "2. Attack Surface": "40.00", + "3. Logging": "30.00", + }, + critical_requirements=[ + { + "requirement_id": "req_shared", + "title": "Shared requirement (secondary)", + "section": "1. IAM", + "subsection": "Sub IAM", + "risk_level": 4, + "weight": 120, + "passed_findings": 8, + "total_findings": 12, + "description": "Lower risk duplicate", + }, + { + "requirement_id": "req_unique_two", + "title": "Unique provider two", + "section": "3. Logging", + "subsection": "Sub Logging", + "risk_level": 5, + "weight": 110, + "passed_findings": 6, + "total_findings": 10, + "description": "Another critical requirement", + }, + ], + total_requirements=80, + passed_requirements=30, + failed_requirements=50, + manual_requirements=0, + total_findings=50, + passed_findings=15, + failed_findings=35, + ) + + older_inserted = datetime(2025, 1, 1, 12, 0, tzinfo=timezone.utc) + newer_inserted = datetime(2025, 1, 2, 12, 0, tzinfo=timezone.utc) + ThreatScoreSnapshot.objects.filter(id=snapshot1.id).update( + inserted_at=older_inserted + ) + ThreatScoreSnapshot.objects.filter(id=snapshot2.id).update( + inserted_at=newer_inserted + ) + snapshot2.refresh_from_db() + + response = authenticated_client.get(reverse("overview-threatscore")) + + assert response.status_code == status.HTTP_200_OK + body = response.json() + assert len(body["data"]) == 1 + aggregated = body["data"][0] + + assert aggregated["id"] == "n/a" + assert aggregated["relationships"]["scan"]["data"] is None + assert aggregated["relationships"]["provider"]["data"] is None + + attrs = aggregated["attributes"] + assert Decimal(attrs["overall_score"]) == Decimal("60.00") + assert Decimal(attrs["score_delta"]) == Decimal("2.67") + assert attrs["inserted_at"] == snapshot2.inserted_at.isoformat().replace( + "+00:00", "Z" + ) + assert attrs["total_findings"] == 150 + assert attrs["passed_findings"] == 85 + assert attrs["failed_findings"] == 65 + assert attrs["total_requirements"] == 200 + assert attrs["passed_requirements"] == 120 + assert attrs["failed_requirements"] == 80 + assert attrs["manual_requirements"] == 0 + + assert attrs["section_scores"] == { + "1. IAM": "50.00", + "2. Attack Surface": "53.33", + "3. Logging": "30.00", + } + + expected_critical = [ + { + "requirement_id": "req_shared", + "title": "Shared requirement (preferred)", + "section": "1. IAM", + "subsection": "Sub IAM", + "risk_level": 5, + "weight": 150, + "passed_findings": 14, + "total_findings": 20, + "description": "Higher risk duplicate", + }, + { + "requirement_id": "req_unique_two", + "title": "Unique provider two", + "section": "3. Logging", + "subsection": "Sub Logging", + "risk_level": 5, + "weight": 110, + "passed_findings": 6, + "total_findings": 10, + "description": "Another critical requirement", + }, + { + "requirement_id": "req_unique_one", + "title": "Unique provider one", + "section": "2. Attack Surface", + "subsection": "Sub Attack", + "risk_level": 4, + "weight": 90, + "passed_findings": 20, + "total_findings": 30, + "description": "Lower risk", + }, + ] + assert attrs["critical_requirements"] == expected_critical + + def test_overview_threatscore_weight_fallback_to_requirements( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = self._create_scan(tenant, provider1, "fallback-scan-1") + scan2 = self._create_scan(tenant, provider2, "fallback-scan-2") + + self._create_threatscore_snapshot( + tenant, + scan1, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="90.00", + score_delta="4.00", + section_scores={"1. IAM": "90.00"}, + critical_requirements=[], + total_requirements=10, + passed_requirements=8, + failed_requirements=0, + manual_requirements=2, + total_findings=0, + passed_findings=0, + failed_findings=0, + ) + self._create_threatscore_snapshot( + tenant, + scan2, + provider2, + compliance_id="prowler_threatscore_aws", + overall_score="50.00", + score_delta="1.00", + section_scores={"1. IAM": "40.00"}, + critical_requirements=[], + total_requirements=12, + passed_requirements=5, + failed_requirements=7, + manual_requirements=0, + total_findings=10, + passed_findings=4, + failed_findings=6, + ) + + response = authenticated_client.get(reverse("overview-threatscore")) + assert response.status_code == status.HTTP_200_OK + aggregate = response.json()["data"][0]["attributes"] + + assert Decimal(aggregate["overall_score"]) == Decimal("67.78") + assert Decimal(aggregate["score_delta"]) == Decimal("2.33") + assert aggregate["total_findings"] == 10 + assert aggregate["total_requirements"] == 22 + assert aggregate["manual_requirements"] == 2 + assert aggregate["section_scores"] == {"1. IAM": "62.22"} + + def test_overview_threatscore_filter_by_scan_id_returns_snapshot( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, *_ = providers_fixture + scan = self._create_scan(tenant, provider1, "filter-scan") + + snapshot = self._create_threatscore_snapshot( + tenant, + scan, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="75.00", + score_delta="3.00", + section_scores={"1. IAM": "70.00"}, + critical_requirements=[], + total_requirements=50, + passed_requirements=30, + failed_requirements=20, + manual_requirements=0, + total_findings=25, + passed_findings=15, + failed_findings=10, + ) + + response = authenticated_client.get( + reverse("overview-threatscore"), {"filter[scan_id]": str(scan.id)} + ) + + assert response.status_code == status.HTTP_200_OK + body = response.json() + assert len(body["data"]) == 1 + assert body["data"][0]["id"] == str(snapshot.id) + assert body["data"][0]["attributes"]["overall_score"] == "75.00" + + def test_overview_threatscore_snapshot_id_returns_specific_snapshot( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, *_ = providers_fixture + scan = self._create_scan(tenant, provider1, "snapshot-id-scan") + + snapshot = self._create_threatscore_snapshot( + tenant, + scan, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="88.50", + score_delta=None, + section_scores={"1. IAM": "80.00"}, + critical_requirements=[], + total_requirements=60, + passed_requirements=45, + failed_requirements=15, + manual_requirements=0, + total_findings=30, + passed_findings=25, + failed_findings=5, + ) + + response = authenticated_client.get( + reverse("overview-threatscore"), {"snapshot_id": str(snapshot.id)} + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["data"]["id"] == str(snapshot.id) + assert data["data"]["attributes"]["score_delta"] is None + + def test_overview_threatscore_provider_filter_returns_unaggregated_snapshot( + self, authenticated_client, tenants_fixture, providers_fixture + ): + tenant = tenants_fixture[0] + provider1, provider2, *_ = providers_fixture + + scan1 = self._create_scan(tenant, provider1, "provider-filter-scan-1") + scan2 = self._create_scan(tenant, provider2, "provider-filter-scan-2") + + snapshot1 = self._create_threatscore_snapshot( + tenant, + scan1, + provider1, + compliance_id="prowler_threatscore_aws", + overall_score="55.55", + score_delta="1.10", + section_scores={"1. IAM": "50.00"}, + critical_requirements=[], + total_requirements=40, + passed_requirements=25, + failed_requirements=15, + manual_requirements=0, + total_findings=12, + passed_findings=7, + failed_findings=5, + ) + self._create_threatscore_snapshot( + tenant, + scan2, + provider2, + compliance_id="prowler_threatscore_aws", + overall_score="44.44", + score_delta="0.80", + section_scores={"1. IAM": "40.00"}, + critical_requirements=[], + total_requirements=30, + passed_requirements=18, + failed_requirements=12, + manual_requirements=0, + total_findings=10, + passed_findings=6, + failed_findings=4, + ) + + response = authenticated_client.get( + reverse("overview-threatscore"), + {"filter[provider_id__in]": str(provider1.id)}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 1 + assert data[0]["id"] == str(snapshot1.id) + assert data[0]["attributes"]["overall_score"] == "55.55" def test_overview_services_list_no_required_filters( self, authenticated_client, scan_summaries_fixture ): response = authenticated_client.get(reverse("overview-services")) - assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.status_code == status.HTTP_200_OK + # Should return services from latest scans + assert len(response.json()["data"]) == 2 def test_overview_services_list(self, authenticated_client, scan_summaries_fixture): response = authenticated_client.get( @@ -8716,7 +9707,7 @@ class TestLighthouseTenantConfigViewSet: """Test creating a tenant config successfully via PATCH (upsert)""" payload = { "data": { - "type": "lighthouse-config", + "type": "lighthouse-configurations", "attributes": { "business_context": "Test business context for security analysis", "default_provider": "", @@ -8725,7 +9716,7 @@ class TestLighthouseTenantConfigViewSet: } } response = authenticated_client.patch( - reverse("lighthouse-config"), + reverse("lighthouse-configurations"), data=payload, content_type=API_JSON_CONTENT_TYPE, ) @@ -8742,7 +9733,7 @@ class TestLighthouseTenantConfigViewSet: """Test that PATCH creates config if not exists and updates if exists (upsert)""" payload = { "data": { - "type": "lighthouse-config", + "type": "lighthouse-configurations", "attributes": { "business_context": "First config", }, @@ -8751,7 +9742,7 @@ class TestLighthouseTenantConfigViewSet: # First PATCH creates the config response = authenticated_client.patch( - reverse("lighthouse-config"), + reverse("lighthouse-configurations"), data=payload, content_type=API_JSON_CONTENT_TYPE, ) @@ -8762,7 +9753,7 @@ class TestLighthouseTenantConfigViewSet: # Second PATCH updates the same config (not creating a duplicate) payload["data"]["attributes"]["business_context"] = "Updated config" response = authenticated_client.patch( - reverse("lighthouse-config"), + reverse("lighthouse-configurations"), data=payload, content_type=API_JSON_CONTENT_TYPE, ) @@ -8818,7 +9809,7 @@ class TestLighthouseTenantConfigViewSet: ) # Retrieve and verify the configuration - response = authenticated_client.get(reverse("lighthouse-config")) + response = authenticated_client.get(reverse("lighthouse-configurations")) assert response.status_code == status.HTTP_200_OK data = response.json()["data"] assert data["id"] == str(config.id) @@ -8828,7 +9819,7 @@ class TestLighthouseTenantConfigViewSet: def test_lighthouse_tenant_config_retrieve_not_found(self, authenticated_client): """Test GET when config doesn't exist returns 404""" - response = authenticated_client.get(reverse("lighthouse-config")) + response = authenticated_client.get(reverse("lighthouse-configurations")) assert response.status_code == status.HTTP_404_NOT_FOUND assert "not found" in response.json()["errors"][0]["detail"].lower() @@ -8849,14 +9840,14 @@ class TestLighthouseTenantConfigViewSet: # Update it payload = { "data": { - "type": "lighthouse-config", + "type": "lighthouse-configurations", "attributes": { "business_context": "Updated context for cloud security", }, } } response = authenticated_client.patch( - reverse("lighthouse-config"), + reverse("lighthouse-configurations"), data=payload, content_type=API_JSON_CONTENT_TYPE, ) @@ -8881,14 +9872,14 @@ class TestLighthouseTenantConfigViewSet: # Try to set invalid provider payload = { "data": { - "type": "lighthouse-config", + "type": "lighthouse-configurations", "attributes": { "default_provider": "nonexistent-provider", }, } } response = authenticated_client.patch( - reverse("lighthouse-config"), + reverse("lighthouse-configurations"), data=payload, content_type=API_JSON_CONTENT_TYPE, ) @@ -8909,7 +9900,7 @@ class TestLighthouseTenantConfigViewSet: # Send invalid JSON response = authenticated_client.patch( - reverse("lighthouse-config"), + reverse("lighthouse-configurations"), data="invalid json", content_type=API_JSON_CONTENT_TYPE, ) @@ -9186,3 +10177,934 @@ class TestLighthouseProviderConfigViewSet: # Unrelated entries should remain untouched assert cfg.default_models.get("other") == "model-x" + + +@pytest.mark.django_db +class TestMuteRuleViewSet: + """Tests for MuteRule endpoints.""" + + def test_mute_rules_list(self, authenticated_client, mute_rules_fixture): + """Test listing all mute rules for the tenant.""" + response = authenticated_client.get(reverse("mute-rule-list")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == len(mute_rules_fixture) + + def test_mute_rules_list_empty(self, authenticated_client, tenants_fixture): + """Test listing mute rules when none exist returns empty list.""" + response = authenticated_client.get(reverse("mute-rule-list")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 0 + assert isinstance(data, list) + + def test_mute_rules_list_default_ordering( + self, authenticated_client, mute_rules_fixture + ): + """Test that mute rules are ordered by -inserted_at by default.""" + response = authenticated_client.get(reverse("mute-rule-list")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + + if len(data) >= 2: + first_date = data[0]["attributes"]["inserted_at"] + second_date = data[1]["attributes"]["inserted_at"] + assert first_date >= second_date + + def test_mute_rules_retrieve(self, authenticated_client, mute_rules_fixture): + """Test retrieving a single mute rule by ID.""" + mute_rule = mute_rules_fixture[0] + response = authenticated_client.get( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}) + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data["id"] == str(mute_rule.id) + assert data["attributes"]["name"] == mute_rule.name + assert data["attributes"]["reason"] == mute_rule.reason + assert data["attributes"]["enabled"] == mute_rule.enabled + assert "finding_uids" in data["attributes"] + assert "inserted_at" in data["attributes"] + assert "updated_at" in data["attributes"] + + def test_mute_rules_retrieve_invalid(self, authenticated_client): + """Test retrieving non-existent mute rule returns 404.""" + response = authenticated_client.get( + reverse( + "mute-rule-detail", + kwargs={"pk": "f498b103-c760-4785-9a3e-e23fafbb7b02"}, + ) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + @pytest.mark.parametrize( + "filter_name, filter_value, expected_count", + ( + [ + ("name", "Test Rule 1", 1), + ("name.icontains", "rule", 2), + ("reason.icontains", "security", 1), + ("enabled", True, 1), + ("enabled", False, 1), + ] + ), + ) + def test_mute_rule_filters( + self, + authenticated_client, + mute_rules_fixture, + filter_name, + filter_value, + expected_count, + ): + """Test filtering mute rules by various fields.""" + filters = {f"filter[{filter_name}]": filter_value} + response = authenticated_client.get(reverse("mute-rule-list"), filters) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == expected_count + + def test_mute_rule_filter_by_created_by( + self, authenticated_client, mute_rules_fixture, create_test_user + ): + """Test filtering mute rules by creator.""" + response = authenticated_client.get( + reverse("mute-rule-list"), + {"filter[created_by]": create_test_user.id}, + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + + def test_mute_rule_search(self, authenticated_client, mute_rules_fixture): + """Test searching mute rules by name and reason.""" + response = authenticated_client.get( + reverse("mute-rule-list"), {"filter[search]": "Rule 1"} + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 1 + + @pytest.mark.parametrize( + "sort_field, first_index", + ( + [ + ("name", 0), + ("-name", 1), + ("inserted_at", 0), + ("-inserted_at", 1), + ] + ), + ) + def test_mute_rule_ordering( + self, authenticated_client, mute_rules_fixture, sort_field, first_index + ): + """Test ordering mute rules by various fields.""" + response = authenticated_client.get( + reverse("mute-rule-list"), {"sort": sort_field} + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == 2 + assert data[0]["id"] == str(mute_rules_fixture[first_index].id) + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_valid( + self, + mock_task, + authenticated_client, + findings_fixture, + create_test_user, + ): + """Test creating a valid mute rule.""" + finding_ids = [str(findings_fixture[0].id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "New Mute Rule", + "reason": "Security exception approved", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_201_CREATED + + # Verify response contains the created mute rule + response_data = response.json()["data"] + assert response_data["type"] == "mute-rules" + assert response_data["attributes"]["name"] == "New Mute Rule" + assert response_data["attributes"]["reason"] == "Security exception approved" + + # Verify the finding was immediately muted + from api.models import Finding + + finding = Finding.objects.get(id=findings_fixture[0].id) + assert finding.muted is True + assert finding.muted_at is not None + assert finding.muted_reason == "Security exception approved" + + # Verify background task was called + mock_task.assert_called_once() + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_converts_finding_ids_to_uids( + self, + mock_task, + authenticated_client, + findings_fixture, + ): + """Test that finding_ids are converted to finding UIDs.""" + finding_ids = [str(findings_fixture[0].id), str(findings_fixture[1].id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "UID Conversion Test", + "reason": "Testing UID conversion", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_201_CREATED + + # Verify finding_uids contains the UIDs, not IDs + from api.models import MuteRule + + mute_rule = MuteRule.objects.get(name="UID Conversion Test") + expected_uids = [ + findings_fixture[0].uid, + findings_fixture[1].uid, + ] + assert set(mute_rule.finding_uids) == set(expected_uids) + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_deduplicates_uids( + self, + mock_task, + authenticated_client, + tenants_fixture, + providers_fixture, + scans_fixture, + ): + """Test that multiple findings with same UID result in only one UID in the rule.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + shared_uid = "prowler-aws-dedupe-test-001" + + finding1 = Finding.objects.create( + tenant=tenant, + uid=shared_uid, + scan=scan, + status=Status.FAIL, + status_extended="test", + severity=Severity.high, + impact=Severity.high, + check_id="test_check", + check_metadata={"CheckId": "test_check"}, + raw_result={}, + ) + + finding2 = Finding.objects.create( + tenant=tenant, + uid=shared_uid, + scan=scan, + status=Status.FAIL, + status_extended="test", + severity=Severity.high, + impact=Severity.high, + check_id="test_check", + check_metadata={"CheckId": "test_check"}, + raw_result={}, + ) + + finding_ids = [str(finding1.id), str(finding2.id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Dedupe Test Rule", + "reason": "Testing UID deduplication", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_201_CREATED + + from api.models import MuteRule + + mute_rule = MuteRule.objects.get(name="Dedupe Test Rule") + assert len(mute_rule.finding_uids) == 1 + assert mute_rule.finding_uids[0] == shared_uid + + finding1.refresh_from_db() + finding2.refresh_from_db() + assert finding1.muted is True + assert finding2.muted is True + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_overlap_detection_active( + self, + mock_task, + authenticated_client, + mute_rules_fixture, + findings_fixture, + ): + """Test that creating a rule with overlapping UIDs in active rule fails.""" + # mute_rules_fixture[0] is active and has findings_fixture[0] UID + finding_ids = [str(findings_fixture[0].id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Overlapping Rule", + "reason": "This should fail", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_409_CONFLICT + assert "errors" in response.json() + error_detail = response.json()["errors"][0]["detail"] + assert ( + "already muted" in error_detail.lower() or "overlap" in error_detail.lower() + ) + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_no_overlap_with_inactive( + self, + mock_task, + authenticated_client, + mute_rules_fixture, + findings_fixture, + ): + """Test that disabled rules don't prevent new rules with same UIDs.""" + # mute_rules_fixture[1] is disabled + # Disable the enabled rule first + mute_rules_fixture[0].enabled = False + mute_rules_fixture[0].save() + + finding_ids = [str(findings_fixture[0].id)] + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Non-overlapping Rule", + "reason": "Inactive rules don't block", + "finding_ids": finding_ids, + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_201_CREATED + + def test_mute_rules_create_invalid_empty_finding_ids(self, authenticated_client): + """Test creating mute rule with empty finding_ids fails.""" + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Valid", + "reason": "Valid", + "finding_ids": [], + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + assert ( + response.json()["errors"][0]["source"]["pointer"] + == "/data/attributes/finding_ids" + ) + + @patch("tasks.tasks.mute_historical_findings_task.apply_async") + def test_mute_rules_create_invalid_finding_ids( + self, mock_task, authenticated_client + ): + """Test creating mute rule with non-existent finding IDs fails.""" + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": "Invalid Findings", + "reason": "This should fail", + "finding_ids": ["f498b103-c760-4785-9a3e-e23fafbb7b02"], + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + + def test_mute_rules_create_duplicate_name( + self, authenticated_client, mute_rules_fixture + ): + """Test creating a mute rule with duplicate name fails.""" + existing_name = mute_rules_fixture[0].name + data = { + "data": { + "type": "mute-rules", + "attributes": { + "name": existing_name, + "reason": "Duplicate name test", + "finding_ids": ["f498b103-c760-4785-9a3e-e23fafbb7b02"], + }, + } + } + response = authenticated_client.post( + reverse("mute-rule-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + + def test_mute_rules_update_name(self, authenticated_client, mute_rules_fixture): + """Test updating mute rule name.""" + mute_rule = mute_rules_fixture[0] + data = { + "data": { + "type": "mute-rules", + "id": str(mute_rule.id), + "attributes": { + "name": "Updated Name", + }, + } + } + response = authenticated_client.patch( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_200_OK + response_data = response.json()["data"] + assert response_data["attributes"]["name"] == "Updated Name" + + # Verify database was updated + mute_rule.refresh_from_db() + assert mute_rule.name == "Updated Name" + + def test_mute_rules_update_reason(self, authenticated_client, mute_rules_fixture): + """Test updating mute rule reason.""" + mute_rule = mute_rules_fixture[0] + data = { + "data": { + "type": "mute-rules", + "id": str(mute_rule.id), + "attributes": { + "reason": "Updated reason for muting", + }, + } + } + response = authenticated_client.patch( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_200_OK + response_data = response.json()["data"] + assert response_data["attributes"]["reason"] == "Updated reason for muting" + + mute_rule.refresh_from_db() + assert mute_rule.reason == "Updated reason for muting" + + def test_mute_rules_update_enabled(self, authenticated_client, mute_rules_fixture): + """Test disabling a mute rule.""" + mute_rule = mute_rules_fixture[0] + assert mute_rule.enabled is True + + data = { + "data": { + "type": "mute-rules", + "id": str(mute_rule.id), + "attributes": { + "enabled": False, + }, + } + } + response = authenticated_client.patch( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_200_OK + response_data = response.json()["data"] + assert response_data["attributes"]["enabled"] is False + + mute_rule.refresh_from_db() + assert mute_rule.enabled is False + + def test_mute_rules_update_duplicate_name( + self, authenticated_client, mute_rules_fixture + ): + """Test updating mute rule with duplicate name fails.""" + first_rule = mute_rules_fixture[0] + second_rule = mute_rules_fixture[1] + + data = { + "data": { + "type": "mute-rules", + "id": str(second_rule.id), + "attributes": { + "name": first_rule.name, + }, + } + } + response = authenticated_client.patch( + reverse("mute-rule-detail", kwargs={"pk": second_rule.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "errors" in response.json() + + def test_mute_rules_delete(self, authenticated_client, mute_rules_fixture): + """Test deleting a mute rule.""" + mute_rule = mute_rules_fixture[0] + response = authenticated_client.delete( + reverse("mute-rule-detail", kwargs={"pk": mute_rule.id}) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + + # Verify rule was deleted + from api.models import MuteRule + + assert not MuteRule.objects.filter(id=mute_rule.id).exists() + + def test_mute_rules_tenant_isolation( + self, authenticated_client, mute_rules_fixture, tenants_fixture + ): + """Test that users can only access mute rules from their tenant.""" + # Create a second tenant with a mute rule + from api.models import MuteRule, Tenant + + other_tenant = Tenant.objects.create(name="Other Tenant") + other_rule = MuteRule.objects.create( + tenant=other_tenant, + name="Other Tenant Rule", + reason="Should not be visible", + finding_uids=["test-uid"], + ) + + # Try to access other tenant's rule + response = authenticated_client.get( + reverse("mute-rule-detail", kwargs={"pk": other_rule.id}) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + # List should only show current tenant's rules + response = authenticated_client.get(reverse("mute-rule-list")) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert len(data) == len(mute_rules_fixture) + for rule_data in data: + assert rule_data["id"] != str(other_rule.id) + + @pytest.mark.parametrize( + "credentials", + [ + {}, # empty credentials + { + "access_key_id": "AKIAIOSFODNN7EXAMPLE" + }, # missing secret_access_key and region + { + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + }, # missing access_key_id and region + { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, # missing region + { # invalid access_key_id format (not starting with AKIA) + "access_key_id": "ABCD0123456789ABCDEF", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + }, + { # invalid access_key_id format (wrong length) + "access_key_id": "AKIAIOSFODNN7EXAMPL", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + }, + { # invalid secret_access_key format (wrong length) + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEK", + "region": "us-east-1", + }, + { # invalid region format + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "invalid-region", + }, + { # invalid region format (uppercase) + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "US-EAST-1", + }, + ], + ) + def test_bedrock_invalid_credentials(self, authenticated_client, credentials): + """Bedrock provider with invalid credentials should error""" + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": credentials, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + + def test_bedrock_valid_credentials_success(self, authenticated_client): + """Bedrock provider with valid AWS credentials should succeed and mask credentials""" + valid_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + } + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": valid_credentials, + }, + } + } + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_201_CREATED + data = resp.json()["data"] + + # Verify credentials are returned masked + masked_creds = data["attributes"].get("credentials") + assert masked_creds is not None + assert "access_key_id" in masked_creds + assert "secret_access_key" in masked_creds + assert "region" in masked_creds + # Verify all characters are masked with asterisks + assert all(c == "*" for c in masked_creds["access_key_id"]) + assert all(c == "*" for c in masked_creds["secret_access_key"]) + + def test_bedrock_provider_duplicate_per_tenant(self, authenticated_client): + """Creating a second Bedrock provider for same tenant should fail""" + valid_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-west-2", + } + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": valid_credentials, + }, + } + } + # First creation succeeds + resp1 = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp1.status_code == status.HTTP_201_CREATED + + # Second creation should fail with validation error + resp2 = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp2.status_code == status.HTTP_400_BAD_REQUEST + assert "already exists" in str(resp2.json()).lower() + + def test_bedrock_patch_credentials_and_fields_filter(self, authenticated_client): + """PATCH credentials and verify fields filter returns decrypted values""" + valid_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "eu-west-1", + } + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": valid_credentials, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Update credentials with new valid ones + new_credentials = { + "access_key_id": "AKIAZZZZZZZZZZZZZZZZ", + "secret_access_key": "aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789+/==", + "region": "ap-south-1", + } + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "credentials": new_credentials, + "is_active": False, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + updated = patch_resp.json()["data"]["attributes"] + assert updated["is_active"] is False + + # Default GET should return masked credentials + get_resp = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + ) + assert get_resp.status_code == status.HTTP_200_OK + masked = get_resp.json()["data"]["attributes"]["credentials"] + assert all(c == "*" for c in masked["access_key_id"]) + assert all(c == "*" for c in masked["secret_access_key"]) + + # Fields filter should return decrypted credentials + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + assert creds["access_key_id"] == new_credentials["access_key_id"] + assert creds["secret_access_key"] == new_credentials["secret_access_key"] + assert creds["region"] == new_credentials["region"] + + def test_bedrock_partial_credential_update(self, authenticated_client): + """Test partial update of Bedrock credentials (e.g., only region)""" + # Create provider with full credentials + initial_credentials = { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + } + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "bedrock", + "credentials": initial_credentials, + }, + } + } + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + # Update only the region field + partial_update = { + "region": "eu-west-1", + } + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "credentials": partial_update, + }, + } + } + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + + # Verify credentials with fields filter - region should be updated, keys preserved + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + + # Original keys should be preserved + assert creds["access_key_id"] == initial_credentials["access_key_id"] + assert creds["secret_access_key"] == initial_credentials["secret_access_key"] + # Region should be updated + assert creds["region"] == "eu-west-1" + + @pytest.mark.parametrize( + "attributes", + [ + pytest.param( + { + "provider_type": "openai_compatible", + "credentials": {"api_key": "compat-key"}, + }, + id="missing", + ), + pytest.param( + { + "provider_type": "openai_compatible", + "credentials": {"api_key": "compat-key"}, + "base_url": "", + }, + id="empty", + ), + ], + ) + def test_openai_compatible_missing_base_url(self, authenticated_client, attributes): + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": attributes, + } + } + + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + error_detail = str(resp.json()).lower() + assert "base_url" in error_detail + + def test_openai_compatible_invalid_credentials(self, authenticated_client): + payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai_compatible", + "base_url": "https://compat.example/v1", + "credentials": {"api_key": ""}, + }, + } + } + + resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert resp.status_code == status.HTTP_400_BAD_REQUEST + errors = resp.json().get("errors", []) + assert any( + error.get("source", {}).get("pointer") + == "/data/attributes/credentials/api_key" + for error in errors + ) + assert any( + "may not be blank" in error.get("detail", "").lower() for error in errors + ) + + def test_openai_compatible_patch_credentials_and_fields(self, authenticated_client): + create_payload = { + "data": { + "type": "lighthouse-providers", + "attributes": { + "provider_type": "openai_compatible", + "base_url": "https://compat.example/v1", + "credentials": {"api_key": "compat-key-123"}, + }, + } + } + + create_resp = authenticated_client.post( + reverse("lighthouse-providers-list"), + data=create_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert create_resp.status_code == status.HTTP_201_CREATED + provider_id = create_resp.json()["data"]["id"] + + updated_base_url = "https://compat.example/v2" + updated_api_key = "compat-key-456" + patch_payload = { + "data": { + "type": "lighthouse-providers", + "id": provider_id, + "attributes": { + "base_url": updated_base_url, + "credentials": {"api_key": updated_api_key}, + }, + } + } + + patch_resp = authenticated_client.patch( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}), + data=patch_payload, + content_type=API_JSON_CONTENT_TYPE, + ) + assert patch_resp.status_code == status.HTTP_200_OK + updated_attrs = patch_resp.json()["data"]["attributes"] + assert updated_attrs["base_url"] == updated_base_url + assert updated_attrs["credentials"]["api_key"] == "*" * len(updated_api_key) + + get_resp = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + ) + assert get_resp.status_code == status.HTTP_200_OK + masked = get_resp.json()["data"]["attributes"]["credentials"]["api_key"] + assert masked == "*" * len(updated_api_key) + + get_full = authenticated_client.get( + reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}) + + "?fields[lighthouse-providers]=credentials" + ) + assert get_full.status_code == status.HTTP_200_OK + creds = get_full.json()["data"]["attributes"]["credentials"] + assert creds["api_key"] == updated_api_key diff --git a/api/src/backend/api/utils.py b/api/src/backend/api/utils.py index 97325d9240..958f24b3f0 100644 --- a/api/src/backend/api/utils.py +++ b/api/src/backend/api/utils.py @@ -18,8 +18,11 @@ from prowler.providers.azure.azure_provider import AzureProvider from prowler.providers.common.models import Connection from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.github.github_provider import GithubProvider +from prowler.providers.iac.iac_provider import IacProvider from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider from prowler.providers.m365.m365_provider import M365Provider +from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider +from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider class CustomOAuth2Client(OAuth2Client): @@ -65,8 +68,11 @@ def return_prowler_provider( | AzureProvider | GcpProvider | GithubProvider + | IacProvider | KubernetesProvider | M365Provider + | MongodbatlasProvider + | OraclecloudProvider ]: """Return the Prowler provider class based on the given provider type. @@ -74,7 +80,7 @@ def return_prowler_provider( provider (Provider): The provider object containing the provider type and associated secrets. Returns: - AwsProvider | AzureProvider | GcpProvider | GithubProvider | KubernetesProvider | M365Provider: The corresponding provider class. + AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: The corresponding provider class. Raises: ValueError: If the provider type specified in `provider.provider` is not supported. @@ -92,6 +98,12 @@ def return_prowler_provider( prowler_provider = M365Provider case Provider.ProviderChoices.GITHUB.value: prowler_provider = GithubProvider + case Provider.ProviderChoices.MONGODBATLAS.value: + prowler_provider = MongodbatlasProvider + case Provider.ProviderChoices.IAC.value: + prowler_provider = IacProvider + case Provider.ProviderChoices.ORACLECLOUD.value: + prowler_provider = OraclecloudProvider case _: raise ValueError(f"Provider type {provider.provider} not supported") return prowler_provider @@ -128,6 +140,21 @@ def get_prowler_provider_kwargs( **prowler_provider_kwargs, "organizations": [provider.uid], } + elif provider.provider == Provider.ProviderChoices.IAC.value: + # For IaC provider, uid contains the repository URL + # Extract the access token if present in the secret + prowler_provider_kwargs = { + "scan_repository_url": provider.uid, + } + if "access_token" in provider.secret.secret: + prowler_provider_kwargs["oauth_app_token"] = provider.secret.secret[ + "access_token" + ] + elif provider.provider == Provider.ProviderChoices.MONGODBATLAS.value: + prowler_provider_kwargs = { + **prowler_provider_kwargs, + "atlas_organization_id": provider.uid, + } if mutelist_processor: mutelist_content = mutelist_processor.configuration.get("Mutelist", {}) @@ -145,8 +172,11 @@ def initialize_prowler_provider( | AzureProvider | GcpProvider | GithubProvider + | IacProvider | KubernetesProvider | M365Provider + | MongodbatlasProvider + | OraclecloudProvider ): """Initialize a Prowler provider instance based on the given provider type. @@ -155,8 +185,8 @@ def initialize_prowler_provider( mutelist_processor (Processor): The mutelist processor object containing the mutelist configuration. Returns: - AwsProvider | AzureProvider | GcpProvider | GithubProvider | KubernetesProvider | M365Provider: An instance of the corresponding provider class - (`AwsProvider`, `AzureProvider`, `GcpProvider`, `GithubProvider`, `KubernetesProvider` or `M365Provider`) initialized with the + AwsProvider | AzureProvider | GcpProvider | GithubProvider | IacProvider | KubernetesProvider | M365Provider | OraclecloudProvider | MongodbatlasProvider: An instance of the corresponding provider class + (`AwsProvider`, `AzureProvider`, `GcpProvider`, `GithubProvider`, `IacProvider`, `KubernetesProvider`, `M365Provider`, `OraclecloudProvider` or `MongodbatlasProvider`) initialized with the provider's secrets. """ prowler_provider = return_prowler_provider(provider) @@ -180,9 +210,23 @@ def prowler_provider_connection_test(provider: Provider) -> Connection: except Provider.secret.RelatedObjectDoesNotExist as secret_error: return Connection(is_connected=False, error=secret_error) - return prowler_provider.test_connection( - **prowler_provider_kwargs, provider_id=provider.uid, raise_on_exception=False - ) + # For IaC provider, construct the kwargs properly for test_connection + if provider.provider == Provider.ProviderChoices.IAC.value: + # Don't pass repository_url from secret, use scan_repository_url with the UID + iac_test_kwargs = { + "scan_repository_url": provider.uid, + "raise_on_exception": False, + } + # Add access_token if present in the secret + if "access_token" in prowler_provider_kwargs: + iac_test_kwargs["access_token"] = prowler_provider_kwargs["access_token"] + return prowler_provider.test_connection(**iac_test_kwargs) + else: + return prowler_provider.test_connection( + **prowler_provider_kwargs, + provider_id=provider.uid, + raise_on_exception=False, + ) def prowler_integration_connection_test(integration: Integration) -> Connection: diff --git a/api/src/backend/api/v1/mixins.py b/api/src/backend/api/v1/mixins.py index 1cef76a84c..e1a5d3470f 100644 --- a/api/src/backend/api/v1/mixins.py +++ b/api/src/backend/api/v1/mixins.py @@ -12,6 +12,24 @@ from api.models import StateChoices, Task from api.v1.serializers import TaskSerializer +class DisablePaginationMixin: + disable_pagination_query_param = "page[disable]" + disable_pagination_truthy_values = {"true"} + + def should_disable_pagination(self) -> bool: + if not hasattr(self, "request"): + return False + value = self.request.query_params.get(self.disable_pagination_query_param) + if value is None: + return False + return str(value).lower() in self.disable_pagination_truthy_values + + def paginate_queryset(self, queryset): + if self.should_disable_pagination(): + return None + return super().paginate_queryset(queryset) + + class PaginateByPkMixin: """ Mixin to paginate on a list of PKs (cheaper than heavy JOINs), diff --git a/api/src/backend/api/v1/serializer_utils/lighthouse.py b/api/src/backend/api/v1/serializer_utils/lighthouse.py index f7ecef32cc..e29c679ce6 100644 --- a/api/src/backend/api/v1/serializer_utils/lighthouse.py +++ b/api/src/backend/api/v1/serializer_utils/lighthouse.py @@ -1,5 +1,6 @@ import re +from drf_spectacular.utils import extend_schema_field from rest_framework_json_api import serializers @@ -11,3 +12,198 @@ class OpenAICredentialsSerializer(serializers.Serializer): if not re.match(pattern, value or ""): raise serializers.ValidationError("Invalid OpenAI API key format.") return value + + def to_internal_value(self, data): + """Check for unknown fields before DRF filters them out.""" + if not isinstance(data, dict): + raise serializers.ValidationError( + {"non_field_errors": ["Credentials must be an object"]} + ) + + allowed_fields = set(self.fields.keys()) + provided_fields = set(data.keys()) + extra_fields = provided_fields - allowed_fields + + if extra_fields: + raise serializers.ValidationError( + { + "non_field_errors": [ + f"Unknown fields in credentials: {', '.join(sorted(extra_fields))}" + ] + } + ) + + return super().to_internal_value(data) + + +class BedrockCredentialsSerializer(serializers.Serializer): + """ + Serializer for AWS Bedrock credentials validation. + + Validates long-term AWS credentials (AKIA) and region format. + """ + + access_key_id = serializers.CharField() + secret_access_key = serializers.CharField() + region = serializers.CharField() + + def validate_access_key_id(self, value: str) -> str: + """Validate AWS access key ID format (AKIA for long-term credentials).""" + pattern = r"^AKIA[0-9A-Z]{16}$" + if not re.match(pattern, value or ""): + raise serializers.ValidationError( + "Invalid AWS access key ID format. Must be AKIA followed by 16 alphanumeric characters." + ) + return value + + def validate_secret_access_key(self, value: str) -> str: + """Validate AWS secret access key format (40 base64 characters).""" + pattern = r"^[A-Za-z0-9/+=]{40}$" + if not re.match(pattern, value or ""): + raise serializers.ValidationError( + "Invalid AWS secret access key format. Must be 40 base64 characters." + ) + return value + + def validate_region(self, value: str) -> str: + """Validate AWS region format.""" + pattern = r"^[a-z]{2}-[a-z]+-\d+$" + if not re.match(pattern, value or ""): + raise serializers.ValidationError( + "Invalid AWS region format. Expected format like 'us-east-1' or 'eu-west-2'." + ) + return value + + def to_internal_value(self, data): + """Check for unknown fields before DRF filters them out.""" + if not isinstance(data, dict): + raise serializers.ValidationError( + {"non_field_errors": ["Credentials must be an object"]} + ) + + allowed_fields = set(self.fields.keys()) + provided_fields = set(data.keys()) + extra_fields = provided_fields - allowed_fields + + if extra_fields: + raise serializers.ValidationError( + { + "non_field_errors": [ + f"Unknown fields in credentials: {', '.join(sorted(extra_fields))}" + ] + } + ) + + return super().to_internal_value(data) + + +class BedrockCredentialsUpdateSerializer(BedrockCredentialsSerializer): + """ + Serializer for AWS Bedrock credentials during UPDATE operations. + + Inherits all validation logic from BedrockCredentialsSerializer but makes + all fields optional to support partial updates. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Make all fields optional for updates + for field in self.fields.values(): + field.required = False + + +class OpenAICompatibleCredentialsSerializer(serializers.Serializer): + """ + Minimal serializer for OpenAI-compatible credentials. + + Many OpenAI-compatible providers do not use the same key format as OpenAI. + We only require a non-empty API key string. Additional fields can be added later + without breaking existing configurations. + """ + + api_key = serializers.CharField() + + def validate_api_key(self, value: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise serializers.ValidationError("API key is required.") + return value.strip() + + def to_internal_value(self, data): + """Check for unknown fields before DRF filters them out.""" + if not isinstance(data, dict): + raise serializers.ValidationError( + {"non_field_errors": ["Credentials must be an object"]} + ) + + allowed_fields = set(self.fields.keys()) + provided_fields = set(data.keys()) + extra_fields = provided_fields - allowed_fields + + if extra_fields: + raise serializers.ValidationError( + { + "non_field_errors": [ + f"Unknown fields in credentials: {', '.join(sorted(extra_fields))}" + ] + } + ) + + return super().to_internal_value(data) + + +@extend_schema_field( + { + "oneOf": [ + { + "type": "object", + "title": "OpenAI Credentials", + "properties": { + "api_key": { + "type": "string", + "description": "OpenAI API key. Must start with 'sk-' followed by alphanumeric characters, " + "hyphens, or underscores.", + "pattern": "^sk-[\\w-]+$", + } + }, + "required": ["api_key"], + }, + { + "type": "object", + "title": "AWS Bedrock Credentials", + "properties": { + "access_key_id": { + "type": "string", + "description": "AWS access key ID.", + "pattern": "^AKIA[0-9A-Z]{16}$", + }, + "secret_access_key": { + "type": "string", + "description": "AWS secret access key.", + "pattern": "^[A-Za-z0-9/+=]{40}$", + }, + "region": { + "type": "string", + "description": "AWS region identifier where Bedrock is available. Examples: us-east-1, " + "us-west-2, eu-west-1, ap-northeast-1.", + "pattern": "^[a-z]{2}-[a-z]+-\\d+$", + }, + }, + "required": ["access_key_id", "secret_access_key", "region"], + }, + { + "type": "object", + "title": "OpenAI Compatible Credentials", + "properties": { + "api_key": { + "type": "string", + "description": "API key for OpenAI-compatible provider. The format varies by provider. " + "Note: The 'base_url' field (separate from credentials) is required when using this provider type.", + } + }, + "required": ["api_key"], + }, + ] + } +) +class LighthouseCredentialsField(serializers.JSONField): + pass diff --git a/api/src/backend/api/v1/serializer_utils/providers.py b/api/src/backend/api/v1/serializer_utils/providers.py index 6dfaca794c..4ec772e02f 100644 --- a/api/src/backend/api/v1/serializer_utils/providers.py +++ b/api/src/backend/api/v1/serializer_utils/providers.py @@ -239,6 +239,71 @@ from rest_framework_json_api import serializers }, "required": ["github_app_id", "github_app_key"], }, + { + "type": "object", + "title": "IaC Repository Credentials", + "properties": { + "repository_url": { + "type": "string", + "description": "Repository URL to scan for IaC files.", + }, + "access_token": { + "type": "string", + "description": "Optional access token for private repositories.", + }, + }, + "required": ["repository_url"], + }, + { + "type": "object", + "title": "Oracle Cloud Infrastructure (OCI) API Key Credentials", + "properties": { + "user": { + "type": "string", + "description": "The OCID of the user to authenticate with.", + }, + "fingerprint": { + "type": "string", + "description": "The fingerprint of the API signing key.", + }, + "key_file": { + "type": "string", + "description": "The path to the private key file for API signing. Either key_file or key_content must be provided.", + }, + "key_content": { + "type": "string", + "description": "The content of the private key for API signing (base64 encoded). Either key_file or key_content must be provided.", + }, + "tenancy": { + "type": "string", + "description": "The OCID of the tenancy.", + }, + "region": { + "type": "string", + "description": "The OCI region identifier (e.g., us-ashburn-1, us-phoenix-1).", + }, + "pass_phrase": { + "type": "string", + "description": "The passphrase for the private key, if encrypted.", + }, + }, + "required": ["user", "fingerprint", "tenancy", "region"], + }, + { + "type": "object", + "title": "MongoDB Atlas API Key", + "properties": { + "atlas_public_key": { + "type": "string", + "description": "MongoDB Atlas API public key.", + }, + "atlas_private_key": { + "type": "string", + "description": "MongoDB Atlas API private key.", + }, + }, + "required": ["atlas_public_key", "atlas_private_key"], + }, ] } ) diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 511b1f89e4..b2d95a2609 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -31,6 +31,7 @@ from api.models import ( LighthouseProviderModels, LighthouseTenantConfiguration, Membership, + MuteRule, Processor, Provider, ProviderGroup, @@ -46,6 +47,7 @@ from api.models import ( StatusChoices, Task, TenantAPIKey, + ThreatScoreSnapshot, User, UserRoleRelationship, ) @@ -59,7 +61,13 @@ from api.v1.serializer_utils.integrations import ( S3ConfigSerializer, SecurityHubConfigSerializer, ) -from api.v1.serializer_utils.lighthouse import OpenAICredentialsSerializer +from api.v1.serializer_utils.lighthouse import ( + BedrockCredentialsSerializer, + BedrockCredentialsUpdateSerializer, + LighthouseCredentialsField, + OpenAICompatibleCredentialsSerializer, + OpenAICredentialsSerializer, +) from api.v1.serializer_utils.processors import ProcessorConfigField from api.v1.serializer_utils.providers import ProviderSecretField from prowler.lib.mutelist.mutelist import Mutelist @@ -1159,11 +1167,17 @@ class ResourceSerializer(RLSSerializer): "findings", "failed_findings_count", "url", + "metadata", + "details", + "partition", ] extra_kwargs = { "id": {"read_only": True}, "inserted_at": {"read_only": True}, "updated_at": {"read_only": True}, + "metadata": {"read_only": True}, + "details": {"read_only": True}, + "partition": {"read_only": True}, } included_serializers = { @@ -1220,11 +1234,15 @@ class ResourceIncludeSerializer(RLSSerializer): "service", "type_", "tags", + "details", + "partition", ] extra_kwargs = { "id": {"read_only": True}, "inserted_at": {"read_only": True}, "updated_at": {"read_only": True}, + "details": {"read_only": True}, + "partition": {"read_only": True}, } @extend_schema_field( @@ -1355,10 +1373,16 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer): serializer = GCPProviderSecret(data=secret) elif provider_type == Provider.ProviderChoices.GITHUB.value: serializer = GithubProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.IAC.value: + serializer = IacProviderSecret(data=secret) elif provider_type == Provider.ProviderChoices.KUBERNETES.value: serializer = KubernetesProviderSecret(data=secret) elif provider_type == Provider.ProviderChoices.M365.value: serializer = M365ProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.ORACLECLOUD.value: + serializer = OracleCloudProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.MONGODBATLAS.value: + serializer = MongoDBAtlasProviderSecret(data=secret) else: raise serializers.ValidationError( {"provider": f"Provider type not supported {provider_type}"} @@ -1455,6 +1479,14 @@ class GCPServiceAccountProviderSecret(serializers.Serializer): resource_name = "provider-secrets" +class MongoDBAtlasProviderSecret(serializers.Serializer): + atlas_public_key = serializers.CharField() + atlas_private_key = serializers.CharField() + + class Meta: + resource_name = "provider-secrets" + + class KubernetesProviderSecret(serializers.Serializer): kubeconfig_content = serializers.CharField() @@ -1472,6 +1504,27 @@ class GithubProviderSecret(serializers.Serializer): resource_name = "provider-secrets" +class IacProviderSecret(serializers.Serializer): + repository_url = serializers.CharField() + access_token = serializers.CharField(required=False) + + class Meta: + resource_name = "provider-secrets" + + +class OracleCloudProviderSecret(serializers.Serializer): + user = serializers.CharField() + fingerprint = serializers.CharField() + key_file = serializers.CharField(required=False) + key_content = serializers.CharField(required=False) + tenancy = serializers.CharField() + region = serializers.CharField() + pass_phrase = serializers.CharField(required=False) + + class Meta: + resource_name = "provider-secrets" + + class AWSRoleAssumptionProviderSecret(serializers.Serializer): role_arn = serializers.CharField() external_id = serializers.CharField() @@ -2105,6 +2158,17 @@ class OverviewProviderSerializer(serializers.Serializer): } +class OverviewProviderCountSerializer(serializers.Serializer): + id = serializers.CharField(source="provider") + count = serializers.IntegerField() + + class JSONAPIMeta: + resource_name = "providers-count-overview" + + def get_root_meta(self, _resource, _many): + return {"version": "v1"} + + class OverviewFindingSerializer(serializers.Serializer): id = serializers.CharField(default="n/a") new = serializers.IntegerField() @@ -3049,7 +3113,12 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize Accepts credentials as JSON; stored encrypted via credentials_decoded. """ - credentials = serializers.JSONField(write_only=True, required=True) + credentials = LighthouseCredentialsField(write_only=True, required=True) + base_url = serializers.URLField( + required=False, + allow_null=True, + help_text="Base URL for the LLM provider API. Required for 'openai_compatible' provider type.", + ) class Meta: model = LighthouseProviderConfiguration @@ -3061,7 +3130,10 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize ] extra_kwargs = { "is_active": {"required": False}, - "base_url": {"required": False, "allow_null": True}, + "provider_type": { + "help_text": "LLM provider type. Determines which credential format to use. " + "See 'credentials' field documentation for provider-specific requirements." + }, } def create(self, validated_data): @@ -3084,6 +3156,7 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize def validate(self, attrs): provider_type = attrs.get("provider_type") credentials = attrs.get("credentials") or {} + base_url = attrs.get("base_url") if provider_type == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI: try: @@ -3096,6 +3169,35 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize e.detail[f"credentials/{key}"] = value del e.detail[key] raise e + elif ( + provider_type == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + try: + BedrockCredentialsSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e + elif ( + provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE + ): + if not base_url: + raise ValidationError({"base_url": "Base URL is required."}) + try: + OpenAICompatibleCredentialsSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e return super().validate(attrs) @@ -3105,7 +3207,12 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): Update serializer for LighthouseProviderConfiguration. """ - credentials = serializers.JSONField(write_only=True, required=False) + credentials = LighthouseCredentialsField(write_only=True, required=False) + base_url = serializers.URLField( + required=False, + allow_null=True, + help_text="Base URL for the LLM provider API. Required for 'openai_compatible' provider type.", + ) class Meta: model = LighthouseProviderConfiguration @@ -3119,7 +3226,6 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): extra_kwargs = { "id": {"read_only": True}, "provider_type": {"read_only": True}, - "base_url": {"required": False, "allow_null": True}, "is_active": {"required": False}, } @@ -3130,7 +3236,11 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): setattr(instance, attr, value) if credentials is not None: - instance.credentials_decoded = credentials + # Merge partial credentials with existing ones + # New values overwrite existing ones, but unspecified fields are preserved + existing_credentials = instance.credentials_decoded or {} + merged_credentials = {**existing_credentials, **credentials} + instance.credentials_decoded = merged_credentials instance.save() return instance @@ -3138,6 +3248,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): def validate(self, attrs): provider_type = getattr(self.instance, "provider_type", None) credentials = attrs.get("credentials", None) + base_url = attrs.get("base_url", None) if ( credentials is not None @@ -3154,6 +3265,40 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer): e.detail[f"credentials/{key}"] = value del e.detail[key] raise e + elif ( + credentials is not None + and provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + try: + BedrockCredentialsUpdateSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e + elif ( + credentials is not None + and provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE + ): + if base_url is None: + pass + elif not base_url: + raise ValidationError({"base_url": "Base URL cannot be empty."}) + try: + OpenAICompatibleCredentialsSerializer(data=credentials).is_valid( + raise_exception=True + ) + except ValidationError as e: + details = e.detail.copy() + for key, value in details.items(): + e.detail[f"credentials/{key}"] = value + del e.detail[key] + raise e return super().validate(attrs) @@ -3171,7 +3316,7 @@ class LighthouseTenantConfigSerializer(RLSSerializer): def get_url(self, obj): request = self.context.get("request") - return reverse("lighthouse-config", request=request) + return reverse("lighthouse-configurations", request=request) class Meta: model = LighthouseTenantConfiguration @@ -3319,3 +3464,237 @@ class LighthouseProviderModelsUpdateSerializer(BaseWriteSerializer): extra_kwargs = { "id": {"read_only": True}, } + + +# Mute Rules + + +class MuteRuleSerializer(RLSSerializer): + """ + Serializer for reading MuteRule instances. + """ + + finding_uids = serializers.ListField( + child=serializers.CharField(), + read_only=True, + help_text="List of finding UIDs that are muted by this rule", + ) + + class Meta: + model = MuteRule + fields = [ + "id", + "inserted_at", + "updated_at", + "name", + "reason", + "enabled", + "created_by", + "finding_uids", + ] + + included_serializers = { + "created_by": "api.v1.serializers.UserIncludeSerializer", + } + + +class MuteRuleCreateSerializer(RLSSerializer, BaseWriteSerializer): + """ + Serializer for creating new MuteRule instances. + + Accepts finding_ids in the request, converts them to UIDs, and stores in finding_uids. + """ + + finding_ids = serializers.ListField( + child=serializers.UUIDField(), + write_only=True, + required=True, + help_text="List of Finding IDs to mute (will be converted to UIDs)", + ) + finding_uids = serializers.ListField( + child=serializers.CharField(), + read_only=True, + help_text="List of finding UIDs that are muted by this rule", + ) + + class Meta: + model = MuteRule + fields = [ + "id", + "inserted_at", + "updated_at", + "name", + "reason", + "enabled", + "created_by", + "finding_ids", + "finding_uids", + ] + extra_kwargs = { + "id": {"read_only": True}, + "inserted_at": {"read_only": True}, + "updated_at": {"read_only": True}, + "enabled": {"read_only": True}, + "created_by": {"read_only": True}, + } + + def validate_name(self, value): + """Validate that the name is unique within the tenant.""" + tenant_id = self.context.get("tenant_id") + if MuteRule.objects.filter(tenant_id=tenant_id, name=value).exists(): + raise ValidationError("A mute rule with this name already exists.") + return value + + def validate_finding_ids(self, value): + """Validate that all finding IDs exist and belong to the tenant.""" + if not value: + raise ValidationError("At least one finding_id must be provided.") + + tenant_id = self.context.get("tenant_id") + + # Check that all findings exist and belong to this tenant + findings = Finding.all_objects.filter(tenant_id=tenant_id, id__in=value) + found_ids = set(findings.values_list("id", flat=True)) + provided_ids = set(value) + + missing_ids = provided_ids - found_ids + if missing_ids: + raise ValidationError( + f"The following finding IDs do not exist or do not belong to your tenant: {missing_ids}" + ) + + return value + + def validate(self, data): + """Validate the entire mute rule, including overlap detection.""" + data = super().validate(data) + + tenant_id = self.context.get("tenant_id") + finding_ids = data.get("finding_ids", []) + + if not finding_ids: + return data + + # Convert finding IDs to UIDs (deduplicate in case multiple findings have same UID) + findings = Finding.all_objects.filter(id__in=finding_ids, tenant_id=tenant_id) + finding_uids = list(set(findings.values_list("uid", flat=True))) + + # Check for overlaps with existing enabled rules + existing_rules = MuteRule.objects.filter(tenant_id=tenant_id, enabled=True) + + for rule in existing_rules: + overlap = set(finding_uids) & set(rule.finding_uids) + if overlap: + raise ConflictException( + detail=f"The following finding UIDs are already muted by rule '{rule.name}': {overlap}" + ) + + # Store finding_uids in validated_data for create + data["finding_uids"] = finding_uids + + return data + + def create(self, validated_data): + """Create a new mute rule and set created_by.""" + # Remove finding_ids from validated_data (we've already converted to finding_uids) + validated_data.pop("finding_ids", None) + + # Set created_by to the current user + request = self.context.get("request") + if request and hasattr(request, "user"): + validated_data["created_by"] = request.user + + return super().create(validated_data) + + +class MuteRuleUpdateSerializer(BaseWriteSerializer): + """ + Serializer for updating MuteRule instances. + """ + + class Meta: + model = MuteRule + fields = [ + "id", + "name", + "reason", + "enabled", + ] + extra_kwargs = { + "id": {"read_only": True}, + "name": {"required": False}, + "reason": {"required": False}, + "enabled": {"required": False}, + } + + def validate_name(self, value): + """Validate that the name is unique within the tenant, excluding current instance.""" + tenant_id = self.context.get("tenant_id") + if ( + MuteRule.objects.filter(tenant_id=tenant_id, name=value) + .exclude(id=self.instance.id) + .exists() + ): + raise ValidationError("A mute rule with this name already exists.") + return value + + +# ThreatScore Snapshots + + +class ThreatScoreSnapshotSerializer(RLSSerializer): + """ + Serializer for ThreatScore snapshots. + Read-only serializer for retrieving historical ThreatScore metrics. + """ + + id = serializers.SerializerMethodField() + + class Meta: + model = ThreatScoreSnapshot + fields = [ + "id", + "inserted_at", + "scan", + "provider", + "compliance_id", + "overall_score", + "score_delta", + "section_scores", + "critical_requirements", + "total_requirements", + "passed_requirements", + "failed_requirements", + "manual_requirements", + "total_findings", + "passed_findings", + "failed_findings", + ] + extra_kwargs = { + "id": {"read_only": True}, + "inserted_at": {"read_only": True}, + "scan": {"read_only": True}, + "provider": {"read_only": True}, + "compliance_id": {"read_only": True}, + "overall_score": {"read_only": True}, + "score_delta": {"read_only": True}, + "section_scores": {"read_only": True}, + "critical_requirements": {"read_only": True}, + "total_requirements": {"read_only": True}, + "passed_requirements": {"read_only": True}, + "failed_requirements": {"read_only": True}, + "manual_requirements": {"read_only": True}, + "total_findings": {"read_only": True}, + "passed_findings": {"read_only": True}, + "failed_findings": {"read_only": True}, + } + + included_serializers = { + "scan": "api.v1.serializers.ScanIncludeSerializer", + "provider": "api.v1.serializers.ProviderIncludeSerializer", + } + + def get_id(self, obj): + if getattr(obj, "_aggregated", False): + return "n/a" + return str(obj.id) diff --git a/api/src/backend/api/v1/urls.py b/api/src/backend/api/v1/urls.py index 87f55556be..d879d1476b 100644 --- a/api/src/backend/api/v1/urls.py +++ b/api/src/backend/api/v1/urls.py @@ -21,6 +21,7 @@ from api.v1.views import ( LighthouseProviderModelsViewSet, LighthouseTenantConfigViewSet, MembershipViewSet, + MuteRuleViewSet, OverviewViewSet, ProcessorViewSet, ProviderGroupProvidersRelationshipView, @@ -80,6 +81,7 @@ router.register( LighthouseProviderModelsViewSet, basename="lighthouse-models", ) +router.register(r"mute-rules", MuteRuleViewSet, basename="mute-rule") tenants_router = routers.NestedSimpleRouter(router, r"tenants", lookup="tenant") tenants_router.register( @@ -150,13 +152,12 @@ urlpatterns = [ ), name="provider_group-providers-relationship", ), - # Lighthouse tenant config as singleton endpoint path( "lighthouse/configuration", LighthouseTenantConfigViewSet.as_view( {"get": "list", "patch": "partial_update"} ), - name="lighthouse-config", + name="lighthouse-configurations", ), # API endpoint to start SAML SSO flow path( diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 2e8a78207e..700550e4e4 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -3,7 +3,10 @@ import glob import json import logging import os +from collections import defaultdict +from copy import deepcopy from datetime import datetime, timedelta, timezone +from decimal import ROUND_HALF_UP, Decimal, InvalidOperation from urllib.parse import urljoin import sentry_sdk @@ -24,9 +27,23 @@ from django.conf import settings as django_settings from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.search import SearchQuery from django.db import transaction -from django.db.models import Count, F, Prefetch, Q, Subquery, Sum +from django.db.models import ( + Case, + Count, + DecimalField, + ExpressionWrapper, + F, + IntegerField, + Max, + Prefetch, + Q, + Subquery, + Sum, + Value, + When, +) from django.db.models.functions import Coalesce -from django.http import HttpResponse +from django.http import HttpResponse, QueryDict from django.shortcuts import redirect from django.urls import reverse from django.utils.dateparse import parse_date @@ -58,6 +75,7 @@ from rest_framework_simplejwt.exceptions import InvalidToken, TokenError from tasks.beat import schedule_provider_scan from tasks.jobs.export import get_s3_client from tasks.tasks import ( + backfill_compliance_summaries_task, backfill_scan_resource_summaries_task, check_integration_connection_task, check_lighthouse_connection_task, @@ -66,6 +84,7 @@ from tasks.tasks import ( delete_provider_task, delete_tenant_task, jira_integration_task, + mute_historical_findings_task, perform_scan_task, refresh_lighthouse_provider_models_task, ) @@ -90,6 +109,7 @@ from api.filters import ( LighthouseProviderConfigFilter, LighthouseProviderModelsFilter, MembershipFilter, + MuteRuleFilter, ProcessorFilter, ProviderFilter, ProviderGroupFilter, @@ -99,13 +119,14 @@ from api.filters import ( ScanFilter, ScanSummaryFilter, ScanSummarySeverityFilter, - ServiceOverviewFilter, TaskFilter, TenantApiKeyFilter, TenantFilter, + ThreatScoreSnapshotFilter, UserFilter, ) from api.models import ( + ComplianceOverviewSummary, ComplianceRequirementOverview, Finding, Integration, @@ -115,6 +136,7 @@ from api.models import ( LighthouseProviderModels, LighthouseTenantConfiguration, Membership, + MuteRule, Processor, Provider, ProviderGroup, @@ -135,6 +157,7 @@ from api.models import ( StateChoices, Task, TenantAPIKey, + ThreatScoreSnapshot, User, UserRoleRelationship, ) @@ -147,7 +170,7 @@ from api.utils import ( validate_invitation, ) from api.uuid_utils import datetime_to_uuid7, uuid7_start -from api.v1.mixins import PaginateByPkMixin, TaskManagementMixin +from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin from api.v1.serializers import ( ComplianceOverviewAttributesSerializer, ComplianceOverviewDetailSerializer, @@ -175,7 +198,11 @@ from api.v1.serializers import ( LighthouseTenantConfigSerializer, LighthouseTenantConfigUpdateSerializer, MembershipSerializer, + MuteRuleCreateSerializer, + MuteRuleSerializer, + MuteRuleUpdateSerializer, OverviewFindingSerializer, + OverviewProviderCountSerializer, OverviewProviderSerializer, OverviewServiceSerializer, OverviewSeveritySerializer, @@ -211,6 +238,7 @@ from api.v1.serializers import ( TenantApiKeySerializer, TenantApiKeyUpdateSerializer, TenantSerializer, + ThreatScoreSnapshotSerializer, TokenRefreshSerializer, TokenSerializer, TokenSocialLoginSerializer, @@ -413,6 +441,12 @@ class SchemaView(SpectacularAPIView): "description": "Endpoints for API keys management. These can be used as an alternative to JWT " "authorization.", }, + { + "name": "Mute Rules", + "description": "Endpoints for simple mute rules management. These can be used as an alternative to the" + " Mutelist Processor if you need to mute specific findings across your tenant with a " + "specific reason.", + }, ] return super().get(request, *args, **kwargs) @@ -1431,7 +1465,7 @@ class ProviderGroupProvidersRelationshipView(RelationshipView, BaseRLSViewSet): ) @method_decorator(CACHE_DECORATOR, name="list") @method_decorator(CACHE_DECORATOR, name="retrieve") -class ProviderViewSet(BaseRLSViewSet): +class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet): queryset = Provider.objects.all() serializer_class = ProviderSerializer http_method_names = ["get", "post", "patch", "delete"] @@ -3200,15 +3234,50 @@ class RoleProviderGroupRelationshipView(RelationshipView, BaseRLSViewSet): @extend_schema_view( list=extend_schema( tags=["Compliance Overview"], - summary="List compliance overviews for a scan", - description="Retrieve an overview of all the compliance in a given scan.", + summary="List compliance overviews", + description=( + "Retrieve an overview of all compliance frameworks. " + "If scan_id is provided, returns compliance data for that specific scan. " + "If scan_id is omitted, returns compliance data aggregated from the latest completed scan of each provider." + ), parameters=[ OpenApiParameter( name="filter[scan_id]", - required=True, + required=False, type=OpenApiTypes.UUID, location=OpenApiParameter.QUERY, - description="Related scan ID.", + description=( + "Optional scan ID. If provided, returns compliance for that scan. " + "If omitted, returns compliance for the latest completed scan per provider." + ), + ), + OpenApiParameter( + name="filter[provider_id]", + required=False, + type=OpenApiTypes.UUID, + location=OpenApiParameter.QUERY, + description="Filter by specific provider ID.", + ), + OpenApiParameter( + name="filter[provider_id__in]", + required=False, + type={"type": "array", "items": {"type": "string", "format": "uuid"}}, + location=OpenApiParameter.QUERY, + description="Filter by multiple provider IDs (comma-separated).", + ), + OpenApiParameter( + name="filter[provider_type]", + required=False, + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Filter by provider type (e.g., aws, azure, gcp).", + ), + OpenApiParameter( + name="filter[provider_type__in]", + required=False, + type={"type": "array", "items": {"type": "string"}}, + location=OpenApiParameter.QUERY, + description="Filter by multiple provider types (comma-separated).", ), ], responses={ @@ -3365,33 +3434,45 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): def retrieve(self, request, *args, **kwargs): raise MethodNotAllowed(method="GET") - def list(self, request, *args, **kwargs): - scan_id = request.query_params.get("filter[scan_id]") - if not scan_id: - raise ValidationError( - [ - { - "detail": "This query parameter is required.", - "status": 400, - "source": {"pointer": "filter[scan_id]"}, - "code": "required", - } - ] - ) - try: - if task := self.get_task_response_if_running( - task_name="scan-compliance-overviews", - task_kwargs={"tenant_id": self.request.tenant_id, "scan_id": scan_id}, - raise_on_not_found=False, - ): - return task - except TaskFailedException: - return Response( - {"detail": "Task failed to generate compliance overview data."}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) - queryset = self.filter_queryset(self.filter_queryset(self.get_queryset())) + def _compliance_summaries_queryset(self, scan_id): + """Return pre-aggregated summaries constrained by RBAC visibility.""" + role = get_role(self.request.user) + unlimited_visibility = getattr( + role, Permissions.UNLIMITED_VISIBILITY.value, False + ) + summaries = ComplianceOverviewSummary.objects.filter( + tenant_id=self.request.tenant_id, + scan_id=scan_id, + ) + if not unlimited_visibility: + providers = Provider.all_objects.filter( + provider_groups__in=role.provider_groups.all() + ).distinct() + summaries = summaries.filter(scan__provider__in=providers) + + return summaries + + def _get_compliance_template(self, *, provider=None, scan_id=None): + """Return the compliance template for the given provider or scan.""" + if provider is None and scan_id is not None: + scan = Scan.all_objects.select_related("provider").get(pk=scan_id) + provider = scan.provider + + if not provider: + return {} + + return PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE.get(provider.provider, {}) + + def _aggregate_compliance_overview(self, queryset, template_metadata=None): + """ + Aggregate requirement rows into compliance overview dictionaries. + + Args: + queryset: ComplianceRequirementOverview queryset already filtered. + template_metadata: Optional dict mapping compliance_id -> metadata. + """ + template_metadata = template_metadata or {} requirement_status_subquery = queryset.values( "compliance_id", "requirement_id" ).annotate( @@ -3401,13 +3482,15 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): ) compliance_data = {} - framework_info = {} - - for item in queryset.values("compliance_id", "framework", "version").distinct(): - framework_info[item["compliance_id"]] = { + fallback_metadata = { + item["compliance_id"]: { "framework": item["framework"], "version": item["version"], } + for item in queryset.values( + "compliance_id", "framework", "version" + ).distinct() + } for item in requirement_status_subquery: compliance_id = item["compliance_id"] @@ -3419,32 +3502,36 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): else: req_status = "MANUAL" - if compliance_id not in compliance_data: - compliance_data[compliance_id] = { + compliance_status = compliance_data.setdefault( + compliance_id, + { "total_requirements": 0, "requirements_passed": 0, "requirements_failed": 0, "requirements_manual": 0, - } + }, + ) - compliance_data[compliance_id]["total_requirements"] += 1 + compliance_status["total_requirements"] += 1 if req_status == "PASS": - compliance_data[compliance_id]["requirements_passed"] += 1 + compliance_status["requirements_passed"] += 1 elif req_status == "FAIL": - compliance_data[compliance_id]["requirements_failed"] += 1 + compliance_status["requirements_failed"] += 1 else: - compliance_data[compliance_id]["requirements_manual"] += 1 + compliance_status["requirements_manual"] += 1 response_data = [] for compliance_id, data in compliance_data.items(): - framework = framework_info.get(compliance_id, {}) + template = template_metadata.get(compliance_id, {}) + fallback = fallback_metadata.get(compliance_id, {}) response_data.append( { "id": compliance_id, "compliance_id": compliance_id, - "framework": framework.get("framework", ""), - "version": framework.get("version", ""), + "framework": template.get("framework") + or fallback.get("framework", ""), + "version": template.get("version") or fallback.get("version", ""), "requirements_passed": data["requirements_passed"], "requirements_failed": data["requirements_failed"], "requirements_manual": data["requirements_manual"], @@ -3453,7 +3540,151 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): ) serializer = self.get_serializer(response_data, many=True) - return Response(serializer.data) + return serializer.data + + def _task_response_if_running(self, scan_id): + """Check for an in-progress task only when no compliance data exists.""" + try: + return self.get_task_response_if_running( + task_name="scan-compliance-overviews", + task_kwargs={"tenant_id": self.request.tenant_id, "scan_id": scan_id}, + raise_on_not_found=False, + ) + except TaskFailedException: + return Response( + {"detail": "Task failed to generate compliance overview data."}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + def _list_with_region_filter(self, scan_id, region_filter): + """ + Fall back to detailed ComplianceRequirementOverview query when region filter is applied. + This uses the original aggregation logic across filtered regions. + """ + regions = region_filter.split(",") if "," in region_filter else [region_filter] + queryset = self.filter_queryset(self.get_queryset()).filter( + scan_id=scan_id, + region__in=regions, + ) + + data = self._aggregate_compliance_overview(queryset) + if data: + return Response(data) + + task_response = self._task_response_if_running(scan_id) + if task_response: + return task_response + + return Response(data) + + def _list_without_region_aggregation(self, scan_id): + """ + Fall back aggregation when compliance summaries don't exist yet. + Aggregates ComplianceRequirementOverview data across ALL regions. + """ + queryset = self.filter_queryset(self.get_queryset()).filter(scan_id=scan_id) + compliance_template = self._get_compliance_template(scan_id=scan_id) + data = self._aggregate_compliance_overview( + queryset, template_metadata=compliance_template + ) + if data: + return Response(data) + + task_response = self._task_response_if_running(scan_id) + if task_response: + return task_response + + return Response(data) + + def list(self, request, *args, **kwargs): + scan_id = request.query_params.get("filter[scan_id]") + tenant_id = self.request.tenant_id + + if scan_id: + # Specific scan requested - use optimized summaries with region support + region_filter = request.query_params.get( + "filter[region]" + ) or request.query_params.get("filter[region__in]") + + if region_filter: + # Fall back to detailed query with region filtering + return self._list_with_region_filter(scan_id, region_filter) + + summaries = list(self._compliance_summaries_queryset(scan_id)) + if not summaries: + # Trigger async backfill for next time + backfill_compliance_summaries_task.delay( + tenant_id=self.request.tenant_id, scan_id=scan_id + ) + # Use fallback aggregation for this request + return self._list_without_region_aggregation(scan_id) + + # Get compliance template for provider to enrich with framework/version + compliance_template = self._get_compliance_template(scan_id=scan_id) + + # Convert to response format with framework/version enrichment + response_data = [] + for summary in summaries: + compliance_metadata = compliance_template.get(summary.compliance_id, {}) + response_data.append( + { + "id": summary.compliance_id, + "compliance_id": summary.compliance_id, + "framework": compliance_metadata.get("framework", ""), + "version": compliance_metadata.get("version", ""), + "requirements_passed": summary.requirements_passed, + "requirements_failed": summary.requirements_failed, + "requirements_manual": summary.requirements_manual, + "total_requirements": summary.total_requirements, + } + ) + + serializer = self.get_serializer(response_data, many=True) + return Response(serializer.data) + else: + # No scan_id provided - use latest scans per provider + # First, check if provider filters are present + provider_id = request.query_params.get("filter[provider_id]") + provider_id__in = request.query_params.get("filter[provider_id__in]") + provider_type = request.query_params.get("filter[provider_type]") + provider_type__in = request.query_params.get("filter[provider_type__in]") + + scan_filters = {"tenant_id": tenant_id, "state": StateChoices.COMPLETED} + + # Apply provider ID filters + if provider_id: + scan_filters["provider_id"] = provider_id + elif provider_id__in: + # Convert comma-separated string to list + provider_ids = [pid.strip() for pid in provider_id__in.split(",")] + scan_filters["provider_id__in"] = provider_ids + + # Apply provider type filters + if provider_type: + scan_filters["provider__provider"] = provider_type + elif provider_type__in: + # Convert comma-separated string to list + provider_types = [pt.strip() for pt in provider_type__in.split(",")] + scan_filters["provider__provider__in"] = provider_types + + latest_scan_ids = ( + Scan.all_objects.filter(**scan_filters) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + + base_queryset = self.get_queryset() + queryset = self.filter_queryset( + base_queryset.filter(scan_id__in=latest_scan_ids) + ) + + # Aggregate compliance data across latest scans + compliance_template = self._get_compliance_template() + data = self._aggregate_compliance_overview( + queryset, template_metadata=compliance_template + ) + return Response(data) @action(detail=False, methods=["get"], url_name="metadata") def metadata(self, request): @@ -3469,18 +3700,6 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): } ] ) - try: - if task := self.get_task_response_if_running( - task_name="scan-compliance-overviews", - task_kwargs={"tenant_id": self.request.tenant_id, "scan_id": scan_id}, - raise_on_not_found=False, - ): - return task - except TaskFailedException: - return Response( - {"detail": "Task failed to generate compliance overview data."}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) regions = list( self.get_queryset() .filter(scan_id=scan_id) @@ -3490,6 +3709,15 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): ) result = {"regions": regions} + if regions: + serializer = self.get_serializer(data=result) + serializer.is_valid(raise_exception=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + task_response = self._task_response_if_running(scan_id) + if task_response: + return task_response + serializer = self.get_serializer(data=result) serializer.is_valid(raise_exception=True) return Response(serializer.data, status=status.HTTP_200_OK) @@ -3522,18 +3750,6 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): } ] ) - try: - if task := self.get_task_response_if_running( - task_name="scan-compliance-overviews", - task_kwargs={"tenant_id": self.request.tenant_id, "scan_id": scan_id}, - raise_on_not_found=False, - ): - return task - except TaskFailedException: - return Response( - {"detail": "Task failed to generate compliance overview data."}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) filtered_queryset = self.filter_queryset(self.get_queryset()) all_requirements = filtered_queryset.values( @@ -3593,6 +3809,13 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): requirements_summary, many=True ) + if requirements_summary: + return Response(serializer.data, status=status.HTTP_200_OK) + + task_response = self._task_response_if_running(scan_id) + if task_response: + return task_response + return Response(serializer.data, status=status.HTTP_200_OK) @action(detail=False, methods=["get"], url_name="attributes") @@ -3611,15 +3834,6 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): ) provider_type = None - try: - sample_requirement = ( - self.get_queryset().filter(compliance_id=compliance_id).first() - ) - - if sample_requirement: - provider_type = sample_requirement.scan.provider.provider - except Exception: - pass # If we couldn't determine from database, try each provider type if not provider_type: @@ -3691,6 +3905,13 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): "each provider are considered in the aggregation to ensure accurate and up-to-date insights." ), ), + providers_count=extend_schema( + summary="Get provider counts grouped by type", + description=( + "Retrieve the number of providers grouped by provider type. " + "This endpoint counts every provider in the tenant, including those without completed scans." + ), + ), findings=extend_schema( summary="Get aggregated findings data", description=( @@ -3715,8 +3936,7 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): summary="Get findings data by service", description=( "Retrieve an aggregated summary of findings grouped by service. The response includes the total count " - "of findings for each service, as long as there are at least one finding for that service. At least " - "one of the `inserted_at` filters must be provided." + "of findings for each service, as long as there are at least one finding for that service." ), filters=True, ), @@ -3742,12 +3962,16 @@ class OverviewViewSet(BaseRLSViewSet): def get_serializer_class(self): if self.action == "providers": return OverviewProviderSerializer + elif self.action == "providers_count": + return OverviewProviderCountSerializer elif self.action == "findings": return OverviewFindingSerializer elif self.action == "findings_severity": return OverviewSeveritySerializer elif self.action == "services": return OverviewServiceSerializer + elif self.action == "threatscore": + return ThreatScoreSnapshotSerializer return super().get_serializer_class() def get_filterset_class(self): @@ -3758,7 +3982,7 @@ class OverviewViewSet(BaseRLSViewSet): elif self.action == "findings_severity": return ScanSummarySeverityFilter elif self.action == "services": - return ServiceOverviewFilter + return ScanSummaryFilter return None @extend_schema(exclude=True) @@ -3790,10 +4014,7 @@ class OverviewViewSet(BaseRLSViewSet): findings_aggregated = ( queryset.filter(scan_id__in=latest_scan_ids) - .values( - "scan__provider_id", - provider=F("scan__provider__provider"), - ) + .values(provider=F("scan__provider__provider")) .annotate( findings_passed=Coalesce(Sum("_pass"), 0), findings_failed=Coalesce(Sum("fail"), 0), @@ -3802,13 +4023,16 @@ class OverviewViewSet(BaseRLSViewSet): ) ) - resources_aggregated = ( - Resource.all_objects.filter(tenant_id=tenant_id) - .values("provider_id") - .annotate(total_resources=Count("id")) - ) + resources_queryset = Resource.all_objects.filter(tenant_id=tenant_id) + if hasattr(self, "allowed_providers"): + resources_queryset = resources_queryset.filter( + provider__in=self.allowed_providers + ) + resources_aggregated = resources_queryset.values( + provider_type=F("provider__provider") + ).annotate(total_resources=Count("id")) resource_map = { - row["provider_id"]: row["total_resources"] for row in resources_aggregated + row["provider_type"]: row["total_resources"] for row in resources_aggregated } overview = [] @@ -3816,7 +4040,7 @@ class OverviewViewSet(BaseRLSViewSet): overview.append( { "provider": row["provider"], - "total_resources": resource_map.get(row["scan__provider_id"], 0), + "total_resources": resource_map.get(row["provider"], 0), "total_findings": row["total_findings"], "findings_passed": row["findings_passed"], "findings_failed": row["findings_failed"], @@ -3829,6 +4053,36 @@ class OverviewViewSet(BaseRLSViewSet): status=status.HTTP_200_OK, ) + @action( + detail=False, + methods=["get"], + url_path="providers/count", + url_name="providers-count", + ) + def providers_count(self, request): + tenant_id = self.request.tenant_id + providers_qs = Provider.objects.filter(tenant_id=tenant_id) + + if hasattr(self, "allowed_providers"): + allowed_ids = list(self.allowed_providers.values_list("id", flat=True)) + if not allowed_ids: + overview = [] + return Response( + self.get_serializer(overview, many=True).data, + status=status.HTTP_200_OK, + ) + providers_qs = providers_qs.filter(id__in=allowed_ids) + + overview = ( + providers_qs.values("provider") + .annotate(count=Count("id")) + .order_by("provider") + ) + return Response( + self.get_serializer(overview, many=True).data, + status=status.HTTP_200_OK, + ) + @action(detail=False, methods=["get"], url_name="findings") def findings(self, request): tenant_id = self.request.tenant_id @@ -3959,6 +4213,332 @@ class OverviewViewSet(BaseRLSViewSet): return Response(serializer.data, status=status.HTTP_200_OK) + @extend_schema( + summary="Get ThreatScore snapshots", + description=( + "Retrieve ThreatScore metrics. By default, returns the latest snapshot for each provider. " + "Use snapshot_id to retrieve a specific historical snapshot." + ), + tags=["Overviews"], + parameters=[ + OpenApiParameter( + name="snapshot_id", + type=OpenApiTypes.UUID, + location=OpenApiParameter.QUERY, + description="Retrieve a specific snapshot by ID. If not provided, returns latest snapshots.", + ), + OpenApiParameter( + name="provider_id", + type=OpenApiTypes.UUID, + location=OpenApiParameter.QUERY, + description="Filter by specific provider ID", + ), + OpenApiParameter( + name="provider_id__in", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Filter by multiple provider IDs (comma-separated UUIDs)", + ), + OpenApiParameter( + name="provider_type", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Filter by provider type (aws, azure, gcp, etc.)", + ), + OpenApiParameter( + name="provider_type__in", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Filter by multiple provider types (comma-separated)", + ), + ], + ) + @action(detail=False, methods=["get"], url_name="threatscore") + def threatscore(self, request): + """ + Get ThreatScore snapshots. + + Default behavior: Returns the latest snapshot for each provider. + With snapshot_id: Returns the specific snapshot requested. + """ + tenant_id = self.request.tenant_id + snapshot_id = request.query_params.get("snapshot_id") + + # Base queryset with RLS + base_queryset = ThreatScoreSnapshot.objects.filter(tenant_id=tenant_id) + + # Apply RBAC filtering + if hasattr(self, "allowed_providers"): + base_queryset = base_queryset.filter(provider__in=self.allowed_providers) + + # Case 1: Specific snapshot requested + if snapshot_id: + try: + snapshot = base_queryset.get(id=snapshot_id) + serializer = ThreatScoreSnapshotSerializer( + snapshot, context={"request": request} + ) + return Response(serializer.data, status=status.HTTP_200_OK) + except ThreatScoreSnapshot.DoesNotExist: + raise NotFound(detail="ThreatScore snapshot not found") + + # Case 2: Latest snapshot per provider (default) + # Apply filters manually: this @action is outside the standard list endpoint flow, + # so DRF's filter backends don't execute and we must flatten JSON:API params ourselves. + normalized_params = QueryDict(mutable=True) + for param_key, values in request.query_params.lists(): + normalized_key = param_key + if param_key.startswith("filter[") and param_key.endswith("]"): + normalized_key = param_key[7:-1] + if normalized_key == "snapshot_id": + continue + normalized_params.setlist(normalized_key, values) + + filterset = ThreatScoreSnapshotFilter(normalized_params, queryset=base_queryset) + filtered_queryset = filterset.qs + + # Get distinct provider IDs from filtered queryset + # Pick the latest snapshot per provider using Postgres DISTINCT ON pattern. + # This avoids issuing one query per provider (N+1) when the filtered dataset is large. + latest_snapshot_ids = list( + filtered_queryset.order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + latest_snapshot_map = { + snapshot.id: snapshot + for snapshot in filtered_queryset.filter(id__in=latest_snapshot_ids) + } + latest_snapshots = [ + latest_snapshot_map[snapshot_id] + for snapshot_id in latest_snapshot_ids + if snapshot_id in latest_snapshot_map + ] + + if len(latest_snapshots) <= 1: + serializer = ThreatScoreSnapshotSerializer( + latest_snapshots, many=True, context={"request": request} + ) + return Response(serializer.data, status=status.HTTP_200_OK) + + snapshot_ids = [ + snapshot.id for snapshot in latest_snapshots if snapshot and snapshot.id + ] + aggregated_snapshot = self._build_threatscore_overview_snapshot( + snapshot_ids, tenant_id + ) + serializer = ThreatScoreSnapshotSerializer( + [aggregated_snapshot], many=True, context={"request": request} + ) + return Response(serializer.data, status=status.HTTP_200_OK) + + def _build_threatscore_overview_snapshot(self, snapshot_ids, tenant_id): + """ + Aggregate the latest snapshots into a single overview snapshot for the tenant. + """ + if not snapshot_ids: + raise ValueError( + "Snapshot id list cannot be empty when aggregating threatscore overview" + ) + + base_queryset = ThreatScoreSnapshot.objects.filter( + tenant_id=tenant_id, id__in=snapshot_ids + ) + + annotated_queryset = ( + base_queryset.annotate( + active_requirements=ExpressionWrapper( + F("total_requirements") - F("manual_requirements"), + output_field=IntegerField(), + ) + ) + .annotate( + weight=Case( + When(total_findings__gt=0, then=F("total_findings")), + When( + active_requirements__gt=0, + then=F("active_requirements"), + ), + default=Value(1, output_field=IntegerField()), + output_field=IntegerField(), + ) + ) + .order_by() + ) + + aggregated_metrics = annotated_queryset.aggregate( + total_requirements=Sum("total_requirements"), + passed_requirements=Sum("passed_requirements"), + failed_requirements=Sum("failed_requirements"), + manual_requirements=Sum("manual_requirements"), + total_findings=Sum("total_findings"), + passed_findings=Sum("passed_findings"), + failed_findings=Sum("failed_findings"), + weighted_overall_sum=Sum( + ExpressionWrapper( + F("overall_score") * F("weight"), + output_field=DecimalField(max_digits=14, decimal_places=4), + ) + ), + overall_weight=Sum("weight"), + unweighted_overall_sum=Sum("overall_score"), + weighted_delta_sum=Sum( + Case( + When( + score_delta__isnull=False, + then=ExpressionWrapper( + F("score_delta") * F("weight"), + output_field=DecimalField(max_digits=14, decimal_places=4), + ), + ), + default=Value( + Decimal("0"), + output_field=DecimalField(max_digits=14, decimal_places=4), + ), + output_field=DecimalField(max_digits=14, decimal_places=4), + ) + ), + delta_weight=Sum( + Case( + When(score_delta__isnull=False, then=F("weight")), + default=Value(0, output_field=IntegerField()), + output_field=IntegerField(), + ) + ), + provider_count=Count("id"), + latest_inserted_at=Max("inserted_at"), + ) + + total_requirements = aggregated_metrics["total_requirements"] or 0 + passed_requirements = aggregated_metrics["passed_requirements"] or 0 + failed_requirements = aggregated_metrics["failed_requirements"] or 0 + manual_requirements = aggregated_metrics["manual_requirements"] or 0 + total_findings = aggregated_metrics["total_findings"] or 0 + passed_findings = aggregated_metrics["passed_findings"] or 0 + failed_findings = aggregated_metrics["failed_findings"] or 0 + + weighted_overall_sum = aggregated_metrics["weighted_overall_sum"] + if weighted_overall_sum is None: + weighted_overall_sum = Decimal("0") + unweighted_overall_sum = aggregated_metrics["unweighted_overall_sum"] + if unweighted_overall_sum is None: + unweighted_overall_sum = Decimal("0") + + overall_weight = aggregated_metrics["overall_weight"] or 0 + provider_count = aggregated_metrics["provider_count"] or 0 + + weighted_delta_sum = aggregated_metrics["weighted_delta_sum"] + if weighted_delta_sum is None: + weighted_delta_sum = Decimal("0") + delta_weight = aggregated_metrics["delta_weight"] or 0 + + if overall_weight > 0: + overall_score = (weighted_overall_sum / Decimal(overall_weight)).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + elif provider_count > 0: + overall_score = (unweighted_overall_sum / Decimal(provider_count)).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + else: + overall_score = Decimal("0.00") + + if delta_weight > 0: + score_delta = (weighted_delta_sum / Decimal(delta_weight)).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + else: + score_delta = None + + section_weighted_sums = defaultdict(lambda: Decimal("0")) + section_weights = defaultdict(lambda: Decimal("0")) + + combined_critical_requirements = {} + + snapshots_with_weight = list(annotated_queryset) + + for snapshot in snapshots_with_weight: + weight_value = getattr(snapshot, "weight", None) + try: + weight_decimal = Decimal(weight_value) + except (InvalidOperation, TypeError): + weight_decimal = Decimal("1") + if weight_decimal <= 0: + weight_decimal = Decimal("1") + + section_scores = snapshot.section_scores or {} + for section, score in section_scores.items(): + try: + score_decimal = Decimal(str(score)) + except (InvalidOperation, TypeError): + continue + section_weighted_sums[section] += score_decimal * weight_decimal + section_weights[section] += weight_decimal + + for requirement in snapshot.critical_requirements or []: + key = requirement.get("requirement_id") or requirement.get("title") + if not key: + continue + existing = combined_critical_requirements.get(key) + + def requirement_sort_key(item): + return ( + item.get("risk_level") or 0, + item.get("weight") or 0, + ) + + if existing is None or requirement_sort_key( + requirement + ) > requirement_sort_key(existing): + combined_critical_requirements[key] = deepcopy(requirement) + + aggregated_section_scores = {} + for section, total in section_weighted_sums.items(): + weight_total = section_weights[section] + if weight_total > 0: + aggregated_section_scores[section] = str( + (total / weight_total).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + ) + + aggregated_section_scores = dict(sorted(aggregated_section_scores.items())) + + aggregated_critical_requirements = sorted( + combined_critical_requirements.values(), + key=lambda item: ( + item.get("risk_level") or 0, + item.get("weight") or 0, + ), + reverse=True, + ) + + aggregated_snapshot = ThreatScoreSnapshot( + tenant_id=tenant_id, + scan=None, + provider=None, + compliance_id="prowler_threatscore_overview", + overall_score=overall_score, + score_delta=score_delta, + section_scores=aggregated_section_scores, + critical_requirements=aggregated_critical_requirements, + total_requirements=total_requirements, + passed_requirements=passed_requirements, + failed_requirements=failed_requirements, + manual_requirements=manual_requirements, + total_findings=total_findings, + passed_findings=passed_findings, + failed_findings=failed_findings, + ) + + latest_inserted_at = aggregated_metrics["latest_inserted_at"] + if latest_inserted_at is not None: + aggregated_snapshot.inserted_at = latest_inserted_at + + aggregated_snapshot._aggregated = True + + return aggregated_snapshot + @extend_schema(tags=["Schedule"]) @extend_schema_view( @@ -4271,28 +4851,30 @@ class LighthouseConfigViewSet(BaseRLSViewSet): @extend_schema_view( list=extend_schema( tags=["Lighthouse AI"], - summary="List all LLM provider configs", + summary="List all LLM provider configurations", description="Retrieve all LLM provider configurations for the current tenant", ), retrieve=extend_schema( tags=["Lighthouse AI"], - summary="Retrieve LLM provider config", + summary="Retrieve LLM provider configuration", description="Get details for a specific provider configuration in the current tenant.", ), create=extend_schema( tags=["Lighthouse AI"], - summary="Create LLM provider config", - description="Create a per-tenant configuration for an LLM provider. Only one configuration per provider type is allowed per tenant.", + summary="Create LLM provider configuration", + description="Create a per-tenant configuration for an LLM provider. Only one configuration per provider type " + "is allowed per tenant.", ), partial_update=extend_schema( tags=["Lighthouse AI"], - summary="Update LLM provider config", + summary="Update LLM provider configuration", description="Partially update a provider configuration (e.g., base_url, is_active).", ), destroy=extend_schema( tags=["Lighthouse AI"], - summary="Delete LLM provider config", - description="Delete a provider configuration. Any tenant defaults that reference this provider are cleared during deletion.", + summary="Delete LLM provider configuration", + description="Delete a provider configuration. Any tenant defaults that reference this provider are cleared " + "during deletion.", ), ) class LighthouseProviderConfigViewSet(BaseRLSViewSet): @@ -4357,16 +4939,6 @@ class LighthouseProviderConfigViewSet(BaseRLSViewSet): @action(detail=True, methods=["post"], url_name="connection") def connection(self, request, pk=None): instance = self.get_object() - if ( - instance.provider_type - != LighthouseProviderConfiguration.LLMProviderChoices.OPENAI - ): - return Response( - data={ - "errors": [{"detail": "Only 'openai' provider supported in MVP"}] - }, - status=status.HTTP_400_BAD_REQUEST, - ) with transaction.atomic(): task = check_lighthouse_provider_connection_task.delay( @@ -4388,7 +4960,7 @@ class LighthouseProviderConfigViewSet(BaseRLSViewSet): @extend_schema( tags=["Lighthouse AI"], summary="Refresh LLM models catalog", - description="Fetch available models for this provider configuration and upsert into catalog.", + description="Fetch available models for this provider configuration and upsert into catalog. Supports OpenAI, OpenAI-compatible, and AWS Bedrock providers.", request=None, responses={202: OpenApiResponse(response=TaskSerializer)}, ) @@ -4400,16 +4972,6 @@ class LighthouseProviderConfigViewSet(BaseRLSViewSet): ) def refresh_models(self, request, pk=None): instance = self.get_object() - if ( - instance.provider_type - != LighthouseProviderConfiguration.LLMProviderChoices.OPENAI - ): - return Response( - data={ - "errors": [{"detail": "Only 'openai' provider supported in MVP"}] - }, - status=status.HTTP_400_BAD_REQUEST, - ) with transaction.atomic(): task = refresh_lighthouse_provider_models_task.delay( @@ -4665,3 +5227,95 @@ class TenantApiKeyViewSet(BaseRLSViewSet): serializer = self.get_serializer(instance) return Response(data=serializer.data, status=status.HTTP_200_OK) + + +# MuteRules +@extend_schema_view( + list=extend_schema( + tags=["Mute Rules"], + summary="List all mute rules", + description="Retrieve a list of all mute rules with filtering options.", + ), + retrieve=extend_schema( + tags=["Mute Rules"], + summary="Retrieve a mute rule", + description="Fetch detailed information about a specific mute rule by ID.", + ), + create=extend_schema( + tags=["Mute Rules"], + summary="Create a new mute rule", + description="Create a new mute rule by providing finding IDs, name, and reason. " + "The rule will immediately mute the selected findings and launch a background task " + "to mute all historical findings with matching UIDs.", + request=MuteRuleCreateSerializer, + ), + partial_update=extend_schema( + tags=["Mute Rules"], + summary="Partially update a mute rule", + description="Update certain fields of an existing mute rule (e.g., name, reason, enabled).", + request=MuteRuleUpdateSerializer, + responses={200: MuteRuleSerializer}, + ), + destroy=extend_schema( + tags=["Mute Rules"], + summary="Delete a mute rule", + description="Remove a mute rule from the system. Note: Previously muted findings remain muted.", + ), +) +class MuteRuleViewSet(BaseRLSViewSet): + queryset = MuteRule.objects.all() + serializer_class = MuteRuleSerializer + filterset_class = MuteRuleFilter + http_method_names = ["get", "post", "patch", "delete"] + search_fields = ["name", "reason"] + ordering = ["-inserted_at"] + ordering_fields = [ + "name", + "enabled", + "inserted_at", + "updated_at", + ] + required_permissions = [Permissions.MANAGE_SCANS] + + def get_queryset(self): + queryset = MuteRule.objects.filter(tenant_id=self.request.tenant_id) + return queryset.select_related("created_by") + + def get_serializer_class(self): + if self.action == "create": + return MuteRuleCreateSerializer + elif self.action == "partial_update": + return MuteRuleUpdateSerializer + return super().get_serializer_class() + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + # Create the mute rule + mute_rule = serializer.save() + + tenant_id = str(request.tenant_id) + finding_ids = request.data.get("finding_ids", []) + + # Immediately mute the selected findings + Finding.all_objects.filter( + id__in=finding_ids, tenant_id=tenant_id, muted=False + ).update( + muted=True, + muted_at=mute_rule.inserted_at, + muted_reason=mute_rule.reason, + ) + + # Launch background task for historical muting + with transaction.atomic(): + mute_historical_findings_task.apply_async( + kwargs={"tenant_id": tenant_id, "mute_rule_id": str(mute_rule.id)} + ) + + # Return the created mute rule + serializer = self.get_serializer(mute_rule) + return Response( + data=serializer.data, + status=status.HTTP_201_CREATED, + ) diff --git a/api/src/backend/config/urls.py b/api/src/backend/config/urls.py index 195329d171..c307bcc310 100644 --- a/api/src/backend/config/urls.py +++ b/api/src/backend/config/urls.py @@ -1,7 +1,5 @@ -from django.contrib import admin from django.urls import include, path urlpatterns = [ - path("admin/", admin.site.urls), path("api/v1/", include("api.v1.urls")), ] diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index 5d1acf1e88..d5fa21a787 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -23,6 +23,7 @@ from api.models import ( Invitation, LighthouseConfiguration, Membership, + MuteRule, Processor, Provider, ProviderGroup, @@ -499,8 +500,29 @@ def providers_fixture(tenants_fixture): alias="m365_testing", tenant_id=tenant.id, ) + provider7 = Provider.objects.create( + provider="oraclecloud", + uid="ocid1.tenancy.oc1..aaaaaaaa3dwoazoox4q7wrvriywpokp5grlhgnkwtyt6dmwyou7no6mdmzda", + alias="oci_testing", + tenant_id=tenant.id, + ) + provider8 = Provider.objects.create( + provider="mongodbatlas", + uid="64b1d3c0e4b03b1234567890", + alias="mongodbatlas_testing", + tenant_id=tenant.id, + ) - return provider1, provider2, provider3, provider4, provider5, provider6 + return ( + provider1, + provider2, + provider3, + provider4, + provider5, + provider6, + provider7, + provider8, + ) @pytest.fixture @@ -1419,6 +1441,34 @@ def api_keys_fixture(tenants_fixture, create_test_user): return [api_key1, api_key2, api_key3] +@pytest.fixture +def mute_rules_fixture(tenants_fixture, create_test_user, findings_fixture): + """Create test mute rules for testing.""" + tenant = tenants_fixture[0] + user = create_test_user + + # Create two mute rules: one enabled, one disabled + mute_rule1 = MuteRule.objects.create( + tenant_id=tenant.id, + name="Test Rule 1", + reason="Security exception for testing", + enabled=True, + created_by=user, + finding_uids=[findings_fixture[0].uid], + ) + + mute_rule2 = MuteRule.objects.create( + tenant_id=tenant.id, + name="Test Rule 2", + reason="Compliance exception approved", + enabled=False, + created_by=user, + finding_uids=[findings_fixture[1].uid], + ) + + return mute_rule1, mute_rule2 + + def get_authorization_header(access_token: str) -> dict: return {"Authorization": f"Bearer {access_token}"} diff --git a/api/src/backend/tasks/jobs/backfill.py b/api/src/backend/tasks/jobs/backfill.py index 6c83e6a69b..39fcd41fe1 100644 --- a/api/src/backend/tasks/jobs/backfill.py +++ b/api/src/backend/tasks/jobs/backfill.py @@ -1,5 +1,10 @@ +from collections import defaultdict + +from api.db_router import READ_REPLICA_ALIAS from api.db_utils import rls_transaction from api.models import ( + ComplianceOverviewSummary, + ComplianceRequirementOverview, Resource, ResourceFindingMapping, ResourceScanSummary, @@ -9,7 +14,7 @@ from api.models import ( def backfill_resource_scan_summaries(tenant_id: str, scan_id: str): - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): if ResourceScanSummary.objects.filter( tenant_id=tenant_id, scan_id=scan_id ).exists(): @@ -59,3 +64,114 @@ def backfill_resource_scan_summaries(tenant_id: str, scan_id: str): ) return {"status": "backfilled", "inserted": len(summaries)} + + +def backfill_compliance_summaries(tenant_id: str, scan_id: str): + """ + Backfill ComplianceOverviewSummary records for a completed scan. + + This function checks if summary records already exist for the scan. + If not, it aggregates compliance requirement data and creates the summaries. + + Args: + tenant_id: Target tenant UUID + scan_id: Scan UUID to backfill + + Returns: + dict: Status indicating whether backfill was performed + """ + with rls_transaction(tenant_id): + if ComplianceOverviewSummary.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).exists(): + return {"status": "already backfilled"} + + with rls_transaction(tenant_id): + if not Scan.objects.filter( + tenant_id=tenant_id, + id=scan_id, + state__in=(StateChoices.COMPLETED, StateChoices.FAILED), + ).exists(): + return {"status": "scan is not completed"} + + # Fetch all compliance requirement overview rows for this scan + requirement_rows = ComplianceRequirementOverview.objects.filter( + tenant_id=tenant_id, scan_id=scan_id + ).values( + "compliance_id", + "requirement_id", + "requirement_status", + ) + + if not requirement_rows: + return {"status": "no compliance data to backfill"} + + # Group by (compliance_id, requirement_id) across regions + requirement_statuses = defaultdict( + lambda: {"fail_count": 0, "pass_count": 0, "total_count": 0} + ) + + for row in requirement_rows: + compliance_id = row["compliance_id"] + requirement_id = row["requirement_id"] + requirement_status = row["requirement_status"] + + # Aggregate requirement status across regions + key = (compliance_id, requirement_id) + requirement_statuses[key]["total_count"] += 1 + + if requirement_status == "FAIL": + requirement_statuses[key]["fail_count"] += 1 + elif requirement_status == "PASS": + requirement_statuses[key]["pass_count"] += 1 + + # Determine per-requirement status and aggregate to compliance level + compliance_summaries = defaultdict( + lambda: { + "total_requirements": 0, + "requirements_passed": 0, + "requirements_failed": 0, + "requirements_manual": 0, + } + ) + + for (compliance_id, requirement_id), counts in requirement_statuses.items(): + # Apply business rule: any FAIL → requirement fails + if counts["fail_count"] > 0: + req_status = "FAIL" + elif counts["pass_count"] == counts["total_count"]: + req_status = "PASS" + else: + req_status = "MANUAL" + + # Aggregate to compliance level + compliance_summaries[compliance_id]["total_requirements"] += 1 + if req_status == "PASS": + compliance_summaries[compliance_id]["requirements_passed"] += 1 + elif req_status == "FAIL": + compliance_summaries[compliance_id]["requirements_failed"] += 1 + else: + compliance_summaries[compliance_id]["requirements_manual"] += 1 + + # Create summary objects + summary_objects = [] + for compliance_id, data in compliance_summaries.items(): + summary_objects.append( + ComplianceOverviewSummary( + tenant_id=tenant_id, + scan_id=scan_id, + compliance_id=compliance_id, + requirements_passed=data["requirements_passed"], + requirements_failed=data["requirements_failed"], + requirements_manual=data["requirements_manual"], + total_requirements=data["total_requirements"], + ) + ) + + # Bulk insert summaries + if summary_objects: + ComplianceOverviewSummary.objects.bulk_create( + summary_objects, batch_size=500, ignore_conflicts=True + ) + + return {"status": "backfilled", "inserted": len(summary_objects)} diff --git a/api/src/backend/tasks/jobs/export.py b/api/src/backend/tasks/jobs/export.py index 2b45c75094..a3aee7bfc9 100644 --- a/api/src/backend/tasks/jobs/export.py +++ b/api/src/backend/tasks/jobs/export.py @@ -21,6 +21,8 @@ from prowler.lib.outputs.compliance.aws_well_architected.aws_well_architected im AWSWellArchitected, ) from prowler.lib.outputs.compliance.c5.c5_aws import AWSC5 +from prowler.lib.outputs.compliance.c5.c5_azure import AzureC5 +from prowler.lib.outputs.compliance.c5.c5_gcp import GCPC5 from prowler.lib.outputs.compliance.ccc.ccc_aws import CCC_AWS from prowler.lib.outputs.compliance.ccc.ccc_azure import CCC_Azure from prowler.lib.outputs.compliance.ccc.ccc_gcp import CCC_GCP @@ -30,6 +32,7 @@ from prowler.lib.outputs.compliance.cis.cis_gcp import GCPCIS from prowler.lib.outputs.compliance.cis.cis_github import GithubCIS from prowler.lib.outputs.compliance.cis.cis_kubernetes import KubernetesCIS from prowler.lib.outputs.compliance.cis.cis_m365 import M365CIS +from prowler.lib.outputs.compliance.cis.cis_oraclecloud import OracleCloudCIS from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS @@ -87,6 +90,7 @@ COMPLIANCE_CLASS_MAP = { (lambda name: name.startswith("iso27001_"), AzureISO27001), (lambda name: name == "ccc_azure", CCC_Azure), (lambda name: name == "prowler_threatscore_azure", ProwlerThreatScoreAzure), + (lambda name: name == "c5_azure", AzureC5), ], "gcp": [ (lambda name: name.startswith("cis_"), GCPCIS), @@ -95,6 +99,7 @@ COMPLIANCE_CLASS_MAP = { (lambda name: name.startswith("iso27001_"), GCPISO27001), (lambda name: name == "prowler_threatscore_gcp", ProwlerThreatScoreGCP), (lambda name: name == "ccc_gcp", CCC_GCP), + (lambda name: name == "c5_gcp", GCPC5), ], "kubernetes": [ (lambda name: name.startswith("cis_"), KubernetesCIS), @@ -108,6 +113,13 @@ COMPLIANCE_CLASS_MAP = { "github": [ (lambda name: name.startswith("cis_"), GithubCIS), ], + "iac": [ + # IaC provider doesn't have specific compliance frameworks yet + # Trivy handles its own compliance checks + ], + "oraclecloud": [ + (lambda name: name.startswith("cis_"), OracleCloudCIS), + ], } diff --git a/api/src/backend/tasks/jobs/integrations.py b/api/src/backend/tasks/jobs/integrations.py index 4de4d89bb7..71d2185deb 100644 --- a/api/src/backend/tasks/jobs/integrations.py +++ b/api/src/backend/tasks/jobs/integrations.py @@ -5,7 +5,7 @@ from celery.utils.log import get_task_logger from config.django.base import DJANGO_FINDINGS_BATCH_SIZE from tasks.utils import batched -from api.db_router import READ_REPLICA_ALIAS +from api.db_router import READ_REPLICA_ALIAS, MainRouter from api.db_utils import rls_transaction from api.models import Finding, Integration, Provider from api.utils import initialize_prowler_integration, initialize_prowler_provider @@ -179,7 +179,7 @@ def get_security_hub_client_from_integration( if the connection was successful and the SecurityHub client or connection object. """ # Get the provider associated with this integration - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): provider_relationship = integration.integrationproviderrelationship_set.first() if not provider_relationship: return Connection( @@ -208,7 +208,7 @@ def get_security_hub_client_from_integration( regions_status[region] = region in connection.enabled_regions # Save regions information in the integration configuration - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=MainRouter.default_db): integration.configuration["regions"] = regions_status integration.save() @@ -223,7 +223,7 @@ def get_security_hub_client_from_integration( return True, security_hub else: # Reset regions information if connection fails - with rls_transaction(tenant_id): + with rls_transaction(tenant_id, using=MainRouter.default_db): integration.configuration["regions"] = {} integration.save() @@ -334,8 +334,11 @@ def upload_security_hub_integration( f"Security Hub connection failed for integration {integration.id}: " f"{security_hub.error}" ) - integration.connected = False - integration.save() + with rls_transaction( + tenant_id, using=MainRouter.default_db + ): + integration.connected = False + integration.save() break # Skip this integration security_hub_client = security_hub diff --git a/api/src/backend/tasks/jobs/lighthouse_providers.py b/api/src/backend/tasks/jobs/lighthouse_providers.py index 1a68a5c1e2..53f9e4dd97 100644 --- a/api/src/backend/tasks/jobs/lighthouse_providers.py +++ b/api/src/backend/tasks/jobs/lighthouse_providers.py @@ -1,6 +1,8 @@ -from typing import Dict, Set +from typing import Dict +import boto3 import openai +from botocore.exceptions import BotoCoreError, ClientError from celery.utils.log import get_task_logger from api.models import LighthouseProviderConfiguration, LighthouseProviderModels @@ -30,16 +32,71 @@ def _extract_openai_api_key( return api_key +def _extract_openai_compatible_params( + provider_cfg: LighthouseProviderConfiguration, +) -> Dict[str, str] | None: + """ + Extract base_url and api_key for OpenAI-compatible providers. + """ + creds = provider_cfg.credentials_decoded + base_url = provider_cfg.base_url + if not isinstance(creds, dict): + return None + api_key = creds.get("api_key") + if not isinstance(api_key, str) or not api_key: + return None + if not isinstance(base_url, str) or not base_url: + return None + return {"base_url": base_url, "api_key": api_key} + + +def _extract_bedrock_credentials( + provider_cfg: LighthouseProviderConfiguration, +) -> Dict[str, str] | None: + """ + Safely extract AWS Bedrock credentials from a provider configuration. + + Args: + provider_cfg (LighthouseProviderConfiguration): The provider configuration instance + containing the credentials. + + Returns: + Dict[str, str] | None: Dictionary with 'access_key_id', 'secret_access_key', and + 'region' if present and valid, otherwise None. + """ + creds = provider_cfg.credentials_decoded + if not isinstance(creds, dict): + return None + + access_key_id = creds.get("access_key_id") + secret_access_key = creds.get("secret_access_key") + region = creds.get("region") + + # Validate all required fields are present and are strings + if ( + not isinstance(access_key_id, str) + or not access_key_id + or not isinstance(secret_access_key, str) + or not secret_access_key + or not isinstance(region, str) + or not region + ): + return None + + return { + "access_key_id": access_key_id, + "secret_access_key": secret_access_key, + "region": region, + } + + def check_lighthouse_provider_connection(provider_config_id: str) -> Dict: """ Validate a Lighthouse provider configuration by calling the provider API and toggle its active state accordingly. - Currently supports the OpenAI provider by invoking `models.list` to verify that - the provided credentials are valid. - Args: - provider_config_id (str): The primary key of the `LighthouseProviderConfiguration` + provider_config_id: The primary key of the `LighthouseProviderConfiguration` to validate. Returns: @@ -55,42 +112,238 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> Dict: """ provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id) - # TODO: Add support for other providers - if ( - provider_cfg.provider_type - != LighthouseProviderConfiguration.LLMProviderChoices.OPENAI - ): - return {"connected": False, "error": "Unsupported provider type"} - - api_key = _extract_openai_api_key(provider_cfg) - if not api_key: - provider_cfg.is_active = False - provider_cfg.save() - return {"connected": False, "error": "API key is invalid or missing"} - try: - client = openai.OpenAI(api_key=api_key) - _ = client.models.list() + if ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI + ): + api_key = _extract_openai_api_key(provider_cfg) + if not api_key: + provider_cfg.is_active = False + provider_cfg.save() + return {"connected": False, "error": "API key is invalid or missing"} + + # Test connection by listing models + client = openai.OpenAI(api_key=api_key) + _ = client.models.list() + + elif ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + bedrock_creds = _extract_bedrock_credentials(provider_cfg) + if not bedrock_creds: + provider_cfg.is_active = False + provider_cfg.save() + return { + "connected": False, + "error": "AWS credentials are invalid or missing", + } + + # Test connection by listing foundation models + bedrock_client = boto3.client( + "bedrock", + aws_access_key_id=bedrock_creds["access_key_id"], + aws_secret_access_key=bedrock_creds["secret_access_key"], + region_name=bedrock_creds["region"], + ) + _ = bedrock_client.list_foundation_models() + + elif ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE + ): + params = _extract_openai_compatible_params(provider_cfg) + if not params: + provider_cfg.is_active = False + provider_cfg.save() + return { + "connected": False, + "error": "Base URL or API key is invalid or missing", + } + + # Test connection using OpenAI SDK with custom base_url + # Note: base_url should include version (e.g., https://openrouter.ai/api/v1) + client = openai.OpenAI( + api_key=params["api_key"], + base_url=params["base_url"], + ) + _ = client.models.list() + + else: + return {"connected": False, "error": "Unsupported provider type"} + + # Connection successful provider_cfg.is_active = True provider_cfg.save() return {"connected": True, "error": None} + except Exception as e: - logger.warning("OpenAI connection check failed: %s", str(e)) + logger.warning( + "%s connection check failed: %s", provider_cfg.provider_type, str(e) + ) provider_cfg.is_active = False provider_cfg.save() return {"connected": False, "error": str(e)} +def _fetch_openai_models(api_key: str) -> Dict[str, str]: + """ + Fetch available models from OpenAI API. + + Args: + api_key: OpenAI API key for authentication. + + Returns: + Dict mapping model_id to model_name. For OpenAI, both are the same + as the API doesn't provide separate display names. + + Raises: + Exception: If the API call fails. + """ + client = openai.OpenAI(api_key=api_key) + models = client.models.list() + # OpenAI uses model.id for both ID and display name + return {m.id: m.id for m in getattr(models, "data", [])} + + +def _fetch_openai_compatible_models(base_url: str, api_key: str) -> Dict[str, str]: + """ + Fetch available models from an OpenAI-compatible API using the OpenAI SDK. + + Returns a mapping of model_id -> model_name. Prefers the 'name' attribute + if available (e.g., from OpenRouter), otherwise falls back to 'id'. + + Note: base_url should include version (e.g., https://openrouter.ai/api/v1) + """ + client = openai.OpenAI(api_key=api_key, base_url=base_url) + models = client.models.list() + + available_models: Dict[str, str] = {} + for model in models.data: + model_id = model.id + # Prefer provider-supplied human-friendly name when available + name = getattr(model, "name", None) + if name: + available_models[model_id] = name + else: + available_models[model_id] = model_id + + return available_models + + +def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]: + """ + Fetch available models from AWS Bedrock with entitlement verification. + + This function: + 1. Lists foundation models with TEXT modality support + 2. Lists inference profiles with TEXT modality support + 3. Verifies user has entitlement access to each model + + Args: + bedrock_creds: Dictionary with 'access_key_id', 'secret_access_key', and 'region'. + + Returns: + Dict mapping model_id to model_name for all accessible models. + + Raises: + BotoCoreError, ClientError: If AWS API calls fail. + """ + bedrock_client = boto3.client( + "bedrock", + aws_access_key_id=bedrock_creds["access_key_id"], + aws_secret_access_key=bedrock_creds["secret_access_key"], + region_name=bedrock_creds["region"], + ) + + models_to_check: Dict[str, str] = {} + + # Step 1: Get foundation models with TEXT modality + foundation_response = bedrock_client.list_foundation_models() + model_summaries = foundation_response.get("modelSummaries", []) + + for model in model_summaries: + # Check if model supports TEXT input and output modality + input_modalities = model.get("inputModalities", []) + output_modalities = model.get("outputModalities", []) + + if "TEXT" not in input_modalities or "TEXT" not in output_modalities: + continue + + model_id = model.get("modelId") + if not model_id: + continue + + inference_types = model.get("inferenceTypesSupported", []) + + # Only include models with ON_DEMAND inference support + if "ON_DEMAND" in inference_types: + models_to_check[model_id] = model["modelName"] + + # Step 2: Get inference profiles + try: + inference_profiles_response = bedrock_client.list_inference_profiles() + inference_profiles = inference_profiles_response.get( + "inferenceProfileSummaries", [] + ) + + for profile in inference_profiles: + # Check if profile supports TEXT modality + input_modalities = profile.get("inputModalities", []) + output_modalities = profile.get("outputModalities", []) + + if "TEXT" not in input_modalities or "TEXT" not in output_modalities: + continue + + profile_id = profile.get("inferenceProfileId") + if profile_id: + models_to_check[profile_id] = profile["inferenceProfileName"] + + except (BotoCoreError, ClientError) as e: + logger.info( + "Could not fetch inference profiles in %s: %s", + bedrock_creds["region"], + str(e), + ) + + # Step 3: Verify entitlement availability for each model + available_models: Dict[str, str] = {} + + for model_id, model_name in models_to_check.items(): + try: + availability = bedrock_client.get_foundation_model_availability( + modelId=model_id + ) + + entitlement = availability.get("entitlementAvailability") + + # Only include models user has access to + if entitlement == "AVAILABLE": + available_models[model_id] = model_name + else: + logger.debug( + "Skipping model %s - entitlement status: %s", model_id, entitlement + ) + + except (BotoCoreError, ClientError) as e: + logger.debug( + "Could not check availability for model %s: %s", model_id, str(e) + ) + continue + + return available_models + + def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict: """ Refresh the catalog of models for a Lighthouse provider configuration. - For the OpenAI provider, this fetches the current list of models, upserts entries - into `LighthouseProviderModels`, and deletes stale entries no longer returned by - the provider. + Fetches the current list of models from the provider, upserts entries into + `LighthouseProviderModels`, and deletes stale entries no longer returned. Args: - provider_config_id (str): The primary key of the `LighthouseProviderConfiguration` + provider_config_id: The primary key of the `LighthouseProviderConfiguration` whose models should be refreshed. Returns: @@ -104,45 +357,81 @@ def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict: LighthouseProviderConfiguration.DoesNotExist: If no configuration exists with the given ID. """ provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id) + fetched_models: Dict[str, str] = {} - if ( - provider_cfg.provider_type - != LighthouseProviderConfiguration.LLMProviderChoices.OPENAI - ): - return { - "created": 0, - "updated": 0, - "deleted": 0, - "error": "Unsupported provider type", - } - - api_key = _extract_openai_api_key(provider_cfg) - if not api_key: - return { - "created": 0, - "updated": 0, - "deleted": 0, - "error": "API key is invalid or missing", - } - + # Fetch models from the appropriate provider try: - client = openai.OpenAI(api_key=api_key) - models = client.models.list() - fetched_ids: Set[str] = {m.id for m in getattr(models, "data", [])} - except Exception as e: # noqa: BLE001 - logger.warning("OpenAI models refresh failed: %s", str(e)) + if ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI + ): + api_key = _extract_openai_api_key(provider_cfg) + if not api_key: + return { + "created": 0, + "updated": 0, + "deleted": 0, + "error": "API key is invalid or missing", + } + fetched_models = _fetch_openai_models(api_key) + + elif ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + bedrock_creds = _extract_bedrock_credentials(provider_cfg) + if not bedrock_creds: + return { + "created": 0, + "updated": 0, + "deleted": 0, + "error": "AWS credentials are invalid or missing", + } + fetched_models = _fetch_bedrock_models(bedrock_creds) + + elif ( + provider_cfg.provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE + ): + params = _extract_openai_compatible_params(provider_cfg) + if not params: + return { + "created": 0, + "updated": 0, + "deleted": 0, + "error": "Base URL or API key is invalid or missing", + } + fetched_models = _fetch_openai_compatible_models( + params["base_url"], params["api_key"] + ) + + else: + return { + "created": 0, + "updated": 0, + "deleted": 0, + "error": "Unsupported provider type", + } + + except Exception as e: + logger.warning( + "Unexpected error refreshing %s models: %s", + provider_cfg.provider_type, + str(e), + ) return {"created": 0, "updated": 0, "deleted": 0, "error": str(e)} + # Upsert models into the catalog created = 0 updated = 0 - for model_id in fetched_ids: + for model_id, model_name in fetched_models.items(): obj, was_created = LighthouseProviderModels.objects.update_or_create( tenant_id=provider_cfg.tenant_id, provider_configuration=provider_cfg, model_id=model_id, defaults={ - "model_name": model_id, # OpenAI doesn't return a separate display name + "model_name": model_name, "default_parameters": {}, }, ) @@ -156,7 +445,7 @@ def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict: LighthouseProviderModels.objects.filter( tenant_id=provider_cfg.tenant_id, provider_configuration=provider_cfg ) - .exclude(model_id__in=fetched_ids) + .exclude(model_id__in=fetched_models.keys()) .delete() ) diff --git a/api/src/backend/tasks/jobs/muting.py b/api/src/backend/tasks/jobs/muting.py new file mode 100644 index 0000000000..6ef4d127f5 --- /dev/null +++ b/api/src/backend/tasks/jobs/muting.py @@ -0,0 +1,64 @@ +from celery.utils.log import get_task_logger +from config.django.base import DJANGO_FINDINGS_BATCH_SIZE +from tasks.utils import batched + +from api.db_utils import rls_transaction +from api.models import Finding, MuteRule + +logger = get_task_logger(__name__) + + +def mute_historical_findings(tenant_id: str, mute_rule_id: str): + """ + Mute historical findings that match the given mute rule. + + This function processes findings in batches, updating their muted status + and adding the mute reason. + + Args: + tenant_id (str): The tenant ID for RLS context + mute_rule_id (str): The ID of the mute rule to apply + + Returns: + dict: Summary of the muting operation with findings_muted count + """ + findings_muted_count = 0 + + # Get the list of UIDs to mute and the reason + with rls_transaction(tenant_id): + mute_rule = MuteRule.objects.get(id=mute_rule_id, tenant_id=tenant_id) + finding_uids = mute_rule.finding_uids + mute_reason = mute_rule.reason + muted_at = mute_rule.inserted_at + + # Query findings that match the UIDs and are not already muted + with rls_transaction(tenant_id): + findings_to_mute = Finding.objects.filter( + tenant_id=tenant_id, uid__in=finding_uids, muted=False + ) + total_findings = findings_to_mute.count() + + logger.info( + f"Processing {total_findings} findings for mute rule {mute_rule_id}" + ) + + if total_findings > 0: + for batch, is_last in batched( + findings_to_mute.iterator(), DJANGO_FINDINGS_BATCH_SIZE + ): + batch_ids = [f.id for f in batch] + updated_count = Finding.all_objects.filter( + id__in=batch_ids, tenant_id=tenant_id + ).update( + muted=True, + muted_at=muted_at, + muted_reason=mute_reason, + ) + findings_muted_count += updated_count + + logger.info(f"Muted {findings_muted_count} findings for rule {mute_rule_id}") + + return { + "findings_muted": findings_muted_count, + "rule_id": mute_rule_id, + } diff --git a/api/src/backend/tasks/jobs/report.py b/api/src/backend/tasks/jobs/report.py index 641fa5757a..4136cd3748 100644 --- a/api/src/backend/tasks/jobs/report.py +++ b/api/src/backend/tasks/jobs/report.py @@ -7,7 +7,6 @@ from shutil import rmtree import matplotlib.pyplot as plt from celery.utils.log import get_task_logger from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY -from django.db.models import Count, Q from reportlab.lib import colors from reportlab.lib.enums import TA_CENTER from reportlab.lib.pagesizes import letter @@ -26,11 +25,22 @@ from reportlab.platypus import ( TableStyle, ) from tasks.jobs.export import _generate_output_directory, _upload_to_s3 +from tasks.jobs.threatscore import compute_threatscore_metrics +from tasks.jobs.threatscore_utils import ( + _aggregate_requirement_statistics_from_database, + _calculate_requirements_data_from_statistics, +) from tasks.utils import batched from api.db_router import READ_REPLICA_ALIAS from api.db_utils import rls_transaction -from api.models import Finding, Provider, ScanSummary, StatusChoices +from api.models import ( + Finding, + Provider, + ScanSummary, + StatusChoices, + ThreatScoreSnapshot, +) from api.utils import initialize_prowler_provider from prowler.lib.check.compliance_models import Compliance from prowler.lib.outputs.finding import Finding as FindingOutput @@ -434,56 +444,6 @@ def _add_pdf_footer(canvas_obj: canvas.Canvas, doc: SimpleDocTemplate) -> None: canvas_obj.drawString(width - text_width - 30, 20, powered_text) -def _aggregate_requirement_statistics_from_database( - tenant_id: str, scan_id: str -) -> dict[str, dict[str, int]]: - """ - Aggregate finding statistics by check_id using database aggregation. - - This function uses Django ORM aggregation to calculate pass/fail statistics - entirely in the database, avoiding the need to load findings into memory. - - Args: - tenant_id (str): The tenant ID for Row-Level Security context. - scan_id (str): The ID of the scan to retrieve findings for. - - Returns: - dict[str, dict[str, int]]: Dictionary mapping check_id to statistics: - - 'passed' (int): Number of passed findings for this check - - 'total' (int): Total number of findings for this check - - Example: - { - 'aws_iam_user_mfa_enabled': {'passed': 10, 'total': 15}, - 'aws_s3_bucket_public_access': {'passed': 0, 'total': 5} - } - """ - requirement_statistics_by_check_id = {} - - with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): - # Use database aggregation to calculate stats without loading findings into memory - aggregated_statistics_queryset = ( - Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id) - .values("check_id") - .annotate( - total_findings=Count("id"), - passed_findings=Count("id", filter=Q(status=StatusChoices.PASS)), - ) - ) - - for aggregated_stat in aggregated_statistics_queryset: - check_id = aggregated_stat["check_id"] - requirement_statistics_by_check_id[check_id] = { - "passed": aggregated_stat["passed_findings"], - "total": aggregated_stat["total_findings"], - } - - logger.info( - f"Aggregated statistics for {len(requirement_statistics_by_check_id)} unique checks" - ) - return requirement_statistics_by_check_id - - def _load_findings_for_requirement_checks( tenant_id: str, scan_id: str, check_ids: list[str], prowler_provider ) -> dict[str, list[FindingOutput]]: @@ -544,84 +504,6 @@ def _load_findings_for_requirement_checks( return dict(findings_by_check_id) -def _calculate_requirements_data_from_statistics( - compliance_obj, requirement_statistics_by_check_id: dict[str, dict[str, int]] -) -> tuple[dict[str, dict], list[dict]]: - """ - Calculate requirement status and statistics using pre-aggregated database statistics. - - This function uses O(n) lookups with pre-aggregated statistics from the database, - avoiding the need to iterate over all findings for each requirement. - - Args: - compliance_obj: The compliance framework object containing requirements. - requirement_statistics_by_check_id (dict[str, dict[str, int]]): Pre-aggregated statistics - mapping check_id to {'passed': int, 'total': int} counts. - - Returns: - tuple[dict[str, dict], list[dict]]: A tuple containing: - - attributes_by_requirement_id: Dictionary mapping requirement IDs to their attributes. - - requirements_list: List of requirement dictionaries with status and statistics. - """ - attributes_by_requirement_id = {} - requirements_list = [] - - compliance_framework = getattr(compliance_obj, "Framework", "N/A") - compliance_version = getattr(compliance_obj, "Version", "N/A") - - for requirement in compliance_obj.Requirements: - requirement_id = requirement.Id - requirement_description = getattr(requirement, "Description", "") - requirement_checks = getattr(requirement, "Checks", []) - requirement_attributes = getattr(requirement, "Attributes", []) - - # Store requirement metadata for later use - attributes_by_requirement_id[requirement_id] = { - "attributes": { - "req_attributes": requirement_attributes, - "checks": requirement_checks, - }, - "description": requirement_description, - } - - # Calculate aggregated passed and total findings for this requirement - total_passed_findings = 0 - total_findings_count = 0 - - for check_id in requirement_checks: - if check_id in requirement_statistics_by_check_id: - check_statistics = requirement_statistics_by_check_id[check_id] - total_findings_count += check_statistics["total"] - total_passed_findings += check_statistics["passed"] - - # Determine overall requirement status based on findings - if total_findings_count > 0: - if total_passed_findings == total_findings_count: - requirement_status = StatusChoices.PASS - else: - # Partial pass or complete fail both count as FAIL - requirement_status = StatusChoices.FAIL - else: - # No findings means manual review required - requirement_status = StatusChoices.MANUAL - - requirements_list.append( - { - "id": requirement_id, - "attributes": { - "framework": compliance_framework, - "version": compliance_version, - "status": requirement_status, - "description": requirement_description, - "passed_findings": total_passed_findings, - "total_findings": total_findings_count, - }, - } - ) - - return attributes_by_requirement_id, requirements_list - - def generate_threatscore_report( tenant_id: str, scan_id: str, @@ -1262,8 +1144,9 @@ def generate_threatscore_report_job( 2. Checks provider type compatibility 3. Generates the output directory 4. Calls generate_threatscore_report to create the PDF - 5. Uploads the PDF to S3 - 6. Cleans up temporary files + 5. Computes and stores ThreatScore metrics snapshot + 6. Uploads the PDF to S3 + 7. Cleans up temporary files Args: tenant_id (str): The tenant ID for Row-Level Security context. @@ -1317,6 +1200,66 @@ def generate_threatscore_report_job( min_risk_level=4, ) + # Compute and store ThreatScore metrics snapshot + logger.info(f"Computing ThreatScore metrics for scan {scan_id}") + try: + metrics = compute_threatscore_metrics( + tenant_id=tenant_id, + scan_id=scan_id, + provider_id=provider_id, + compliance_id=compliance_id, + min_risk_level=4, + ) + + # Create snapshot in database + with rls_transaction(tenant_id): + # Get previous snapshot for the same provider to calculate delta + previous_snapshot = ( + ThreatScoreSnapshot.objects.filter( + tenant_id=tenant_id, + provider_id=provider_id, + compliance_id=compliance_id, + ) + .order_by("-inserted_at") + .first() + ) + + # Calculate score delta (improvement) + score_delta = None + if previous_snapshot: + score_delta = metrics["overall_score"] - float( + previous_snapshot.overall_score + ) + + snapshot = ThreatScoreSnapshot.objects.create( + tenant_id=tenant_id, + scan_id=scan_id, + provider_id=provider_id, + compliance_id=compliance_id, + overall_score=metrics["overall_score"], + score_delta=score_delta, + section_scores=metrics["section_scores"], + critical_requirements=metrics["critical_requirements"], + total_requirements=metrics["total_requirements"], + passed_requirements=metrics["passed_requirements"], + failed_requirements=metrics["failed_requirements"], + manual_requirements=metrics["manual_requirements"], + total_findings=metrics["total_findings"], + passed_findings=metrics["passed_findings"], + failed_findings=metrics["failed_findings"], + ) + + delta_msg = ( + f" (delta: {score_delta:+.2f}%)" if score_delta is not None else "" + ) + logger.info( + f"ThreatScore snapshot created with ID {snapshot.id} " + f"(score: {snapshot.overall_score}%{delta_msg})" + ) + except Exception as e: + # Log error but don't fail the job if snapshot creation fails + logger.error(f"Error creating ThreatScore snapshot: {e}") + upload_uri = _upload_to_s3( tenant_id, scan_id, diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 1486cb13ae..c4cf413301 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -1,23 +1,21 @@ import csv import io import json +import re import time import uuid from collections import defaultdict -from copy import deepcopy from datetime import datetime, timezone from typing import Any from celery.utils.log import get_task_logger +from config.env import env from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS from django.db import IntegrityError, OperationalError from django.db.models import Case, Count, IntegerField, Prefetch, Sum, When from tasks.utils import CustomEncoder -from api.compliance import ( - PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE, - generate_scan_compliance, -) +from api.compliance import PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE from api.db_router import READ_REPLICA_ALIAS, MainRouter from api.db_utils import ( POSTGRES_TENANT_VAR, @@ -28,11 +26,14 @@ from api.db_utils import ( ) from api.exceptions import ProviderConnectionError from api.models import ( + ComplianceOverviewSummary, ComplianceRequirementOverview, Finding, + MuteRule, Processor, Provider, Resource, + ResourceFindingMapping, ResourceScanSummary, ResourceTag, Scan, @@ -68,6 +69,10 @@ COMPLIANCE_REQUIREMENT_COPY_COLUMNS = ( "total_findings", "scan_id", ) +# Controls how many findings we process per micro-batch before flushing to DB writes +FINDINGS_MICRO_BATCH_SIZE = env.int("DJANGO_FINDINGS_MICRO_BATCH_SIZE", default=3000) +# Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres +SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=500) def _create_finding_delta( @@ -200,25 +205,43 @@ def _copy_compliance_requirement_rows( def _persist_compliance_requirement_rows( - tenant_id: str, rows: list[dict[str, Any]] + tenant_id: str, rows: list[dict[str, Any]], batch_size: int = 10000 ) -> None: - """Persist compliance requirement rows using COPY with ORM fallback. + """Persist compliance requirement rows using batched COPY with ORM fallback. + + Splits large row sets into batches to reduce lock duration and improve concurrency. Args: tenant_id: Target tenant UUID. rows: Precomputed row dictionaries that reflect the compliance overview state for a scan. + batch_size: Number of rows per COPY batch (default: 10000). """ if not rows: return + total_rows = len(rows) + total_batches = (total_rows + batch_size - 1) // batch_size + try: - _copy_compliance_requirement_rows(tenant_id, rows) + # Process rows in batches to reduce lock duration + for batch_num in range(total_batches): + start_idx = batch_num * batch_size + end_idx = min(start_idx + batch_size, total_rows) + batch = rows[start_idx:end_idx] + + _copy_compliance_requirement_rows(tenant_id, batch) + + logger.info( + f"Compliance COPY batch {batch_num + 1}/{total_batches}: " + f"inserted {len(batch)} rows ({start_idx + len(batch)}/{total_rows} total)" + ) except Exception as error: logger.exception( "COPY bulk insert for compliance requirements failed; falling back to ORM bulk_create", exc_info=error, ) + # Fallback: use ORM bulk_create for all remaining rows fallback_objects = [ ComplianceRequirementOverview( id=row["id"], @@ -246,12 +269,367 @@ def _persist_compliance_requirement_rows( ) +def _create_compliance_summaries( + tenant_id: str, scan_id: str, requirement_statuses: dict +) -> None: + """ + Create pre-aggregated compliance summaries from pre-computed requirement statuses. + + This computes the overall compliance scores across all regions for fast + lookup in the main compliance overview endpoint. + + Args: + tenant_id: Target tenant UUID + scan_id: Scan UUID + requirement_statuses: Pre-computed dict of {(compliance_id, requirement_id): {fail_count, pass_count, total_count}} + """ + # Determine per-requirement status and aggregate to compliance level + compliance_summaries = defaultdict( + lambda: { + "total_requirements": 0, + "requirements_passed": 0, + "requirements_failed": 0, + "requirements_manual": 0, + } + ) + + for (compliance_id, requirement_id), counts in requirement_statuses.items(): + # Apply business rule: any FAIL → requirement fails + if counts["fail_count"] > 0: + req_status = "FAIL" + elif counts["pass_count"] == counts["total_count"]: + req_status = "PASS" + else: + req_status = "MANUAL" + + # Aggregate to compliance level + compliance_summaries[compliance_id]["total_requirements"] += 1 + if req_status == "PASS": + compliance_summaries[compliance_id]["requirements_passed"] += 1 + elif req_status == "FAIL": + compliance_summaries[compliance_id]["requirements_failed"] += 1 + else: + compliance_summaries[compliance_id]["requirements_manual"] += 1 + + # Create summary objects + summary_objects = [] + for compliance_id, data in compliance_summaries.items(): + summary_objects.append( + ComplianceOverviewSummary( + tenant_id=tenant_id, + scan_id=scan_id, + compliance_id=compliance_id, + requirements_passed=data["requirements_passed"], + requirements_failed=data["requirements_failed"], + requirements_manual=data["requirements_manual"], + total_requirements=data["total_requirements"], + ) + ) + + # Bulk insert summaries + if summary_objects: + with rls_transaction(tenant_id): + ComplianceOverviewSummary.objects.bulk_create( + summary_objects, batch_size=500 + ) + + def _normalized_compliance_key(framework: str | None, version: str | None) -> str: """Return normalized identifier used to group compliance totals.""" - normalized_framework = (framework or "").lower().replace("-", "").replace("_", "") - normalized_version = (version or "").lower().replace("-", "").replace("_", "") - return f"{normalized_framework}{normalized_version}" + def _normalize(value: str | None) -> str: + if not value: + return "" + return re.sub(r"[^a-z0-9]", "", value.lower()) + + return f"{_normalize(framework)}{_normalize(version)}" + + +def _process_finding_micro_batch( + tenant_id: str, + findings_batch: list[ProwlerFinding], + scan_instance: Scan, + provider_instance: Provider, + resource_cache: dict, + tag_cache: dict, + last_status_cache: dict, + resource_failed_findings_cache: dict, + unique_resources: set, + scan_resource_cache: set, + mute_rules_cache: dict, +) -> None: + """ + Process a micro-batch of findings and persist them using bulk operations. + + Each batch reuses caches (resources, tags, last statuses, mute rules) to avoid + redundant queries, updates denormalized resource data, and writes findings plus + resource mappings in as few transactions as possible. + + Args: + tenant_id: Tenant owning the scan. + findings_batch: Findings yielded by the Prowler scanner for this slice. + scan_instance: Scan ORM instance being updated. + provider_instance: Provider tied to the scan. + resource_cache: In-memory cache of provider resources indexed by UID. + tag_cache: Cache of `ResourceTag` instances keyed by (key, value). + last_status_cache: Cache of prior finding statuses keyed by finding UID. + resource_failed_findings_cache: Mutable counter of failed findings per resource. + unique_resources: Set tracking (uid, region) pairs seen in the scan. + scan_resource_cache: Set of tuples used to create `ResourceScanSummary` rows. + mute_rules_cache: Map of finding UID -> mute reason gathered before the scan. + """ + # Accumulate objects for bulk operations + findings_to_create = [] + mappings_to_create = [] + dirty_resources = {} + resource_denormalized_data = [] # (finding_instance, resource_instance) pairs + skipped_findings_count = 0 # Track findings skipped due to UID length + + # Prefetch last statuses for all findings in this batch + # TEMPORARY WORKAROUND: Filter out UIDs > 300 chars to avoid query errors + finding_uids = [ + f.uid for f in findings_batch if f is not None and len(f.uid) <= 300 + ] + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + last_statuses = { + item["uid"]: (item["status"], item["first_seen_at"]) + for item in Finding.all_objects.filter( + tenant_id=tenant_id, uid__in=finding_uids + ) + .values("uid", "status", "first_seen_at") + .order_by("uid", "-inserted_at") + .distinct("uid") + } + # Update cache + for uid, data in last_statuses.items(): + if uid not in last_status_cache: + last_status_cache[uid] = data + + # Process each finding in the batch + for finding in findings_batch: + if finding is None: + logger.error(f"None finding detected on scan {scan_instance.id}.") + continue + + # Process resource with deadlock retry + for attempt in range(CELERY_DEADLOCK_ATTEMPTS): + try: + with rls_transaction(tenant_id): + resource_uid = finding.resource_uid + if resource_uid not in resource_cache: + resource_instance, _ = Resource.objects.get_or_create( + tenant_id=tenant_id, + provider=provider_instance, + uid=resource_uid, + defaults={ + "region": finding.region, + "service": finding.service_name, + "type": finding.resource_type, + "name": finding.resource_name, + }, + ) + resource_cache[resource_uid] = resource_instance + resource_failed_findings_cache[resource_uid] = 0 + else: + resource_instance = resource_cache[resource_uid] + break + except (OperationalError, IntegrityError) as db_err: + if attempt < CELERY_DEADLOCK_ATTEMPTS - 1: + logger.warning( + f"{'Deadlock error' if isinstance(db_err, OperationalError) else 'Integrity error'} " + f"detected when processing resource {resource_uid} on scan {scan_instance.id}. Retrying..." + ) + time.sleep(0.1 * (2**attempt)) + continue + else: + raise db_err + + # Track resource field changes (defer save) + updated = False + if finding.region and resource_instance.region != finding.region: + resource_instance.region = finding.region + updated = True + if resource_instance.service != finding.service_name: + resource_instance.service = finding.service_name + updated = True + if resource_instance.type != finding.resource_type: + resource_instance.type = finding.resource_type + updated = True + if resource_instance.metadata != finding.resource_metadata: + resource_instance.metadata = json.dumps( + finding.resource_metadata, cls=CustomEncoder + ) + updated = True + if resource_instance.details != finding.resource_details: + resource_instance.details = finding.resource_details + updated = True + if resource_instance.partition != finding.partition: + resource_instance.partition = finding.partition + updated = True + + if updated: + dirty_resources[resource_uid] = resource_instance + + # Process tags + tags = [] + with rls_transaction(tenant_id): + for key, value in finding.resource_tags.items(): + tag_key = (key, value) + if tag_key not in tag_cache: + tag_instance, _ = ResourceTag.objects.get_or_create( + tenant_id=tenant_id, key=key, value=value + ) + tag_cache[tag_key] = tag_instance + else: + tag_instance = tag_cache[tag_key] + tags.append(tag_instance) + resource_instance.upsert_or_delete_tags(tags=tags) + + unique_resources.add((resource_instance.uid, resource_instance.region)) + + # Prepare finding data + finding_uid = finding.uid + + # TEMPORARY WORKAROUND: Skip findings with UID > 300 chars + # TODO: Remove this after implementing text field migration for finding.uid + if len(finding_uid) > 300: + skipped_findings_count += 1 + logger.warning( + f"Skipping finding with UID exceeding 300 characters. " + f"Length: {len(finding_uid)}, " + f"Check: {finding.check_id}, " + f"Resource: {finding.resource_name}, " + f"UID: {finding_uid}" + ) + continue + + last_status, last_first_seen_at = last_status_cache.get( + finding_uid, (None, None) + ) + + status = FindingStatus[finding.status] + delta = _create_finding_delta(last_status, status) + + if not last_first_seen_at: + last_first_seen_at = datetime.now(tz=timezone.utc) + + # Determine if finding should be muted and why + # Priority: mutelist processor (highest) > manual mute rules + is_muted = False + muted_reason = None + + # Check mutelist processor first (highest priority) + if finding.muted: + is_muted = True + muted_reason = "Muted by mutelist" + # If not muted by mutelist, check manual mute rules + elif finding_uid in mute_rules_cache: + is_muted = True + muted_reason = mute_rules_cache[finding_uid] + + # Increment failed_findings_count cache if needed + if status == FindingStatus.FAIL and not is_muted: + resource_failed_findings_cache[resource_uid] += 1 + + # Create finding object (don't save yet) + finding_instance = Finding( + tenant_id=tenant_id, + uid=finding_uid, + delta=delta, + check_metadata=finding.get_metadata(), + status=status, + status_extended=finding.status_extended, + severity=finding.severity, + impact=finding.severity, + raw_result=finding.raw, + check_id=finding.check_id, + scan=scan_instance, + first_seen_at=last_first_seen_at, + muted=is_muted, + muted_at=datetime.now(tz=timezone.utc) if is_muted else None, + muted_reason=muted_reason, + compliance=finding.compliance, + ) + findings_to_create.append(finding_instance) + resource_denormalized_data.append((finding_instance, resource_instance)) + + # Track for scan summary + scan_resource_cache.add( + ( + str(resource_instance.id), + resource_instance.service, + resource_instance.region, + resource_instance.type, + ) + ) + + # Bulk operations within single transaction + with rls_transaction(tenant_id): + # Bulk create findings + if findings_to_create: + Finding.objects.bulk_create( + findings_to_create, batch_size=SCAN_DB_BATCH_SIZE + ) + + # Bulk create resource-finding mappings + for finding_instance, resource_instance in resource_denormalized_data: + mappings_to_create.append( + ResourceFindingMapping( + tenant_id=tenant_id, + resource=resource_instance, + finding=finding_instance, + ) + ) + + if mappings_to_create: + ResourceFindingMapping.objects.bulk_create( + mappings_to_create, + batch_size=SCAN_DB_BATCH_SIZE, + ignore_conflicts=True, + ) + + # Update finding denormalized arrays + findings_to_update = [] + for finding_instance, resource_instance in resource_denormalized_data: + if not finding_instance.resource_regions: + finding_instance.resource_regions = [] + if not finding_instance.resource_services: + finding_instance.resource_services = [] + if not finding_instance.resource_types: + finding_instance.resource_types = [] + + if resource_instance.region not in finding_instance.resource_regions: + finding_instance.resource_regions.append(resource_instance.region) + if resource_instance.service not in finding_instance.resource_services: + finding_instance.resource_services.append(resource_instance.service) + if resource_instance.type not in finding_instance.resource_types: + finding_instance.resource_types.append(resource_instance.type) + + findings_to_update.append(finding_instance) + + if findings_to_update: + Finding.objects.bulk_update( + findings_to_update, + ["resource_regions", "resource_services", "resource_types"], + batch_size=SCAN_DB_BATCH_SIZE, + ) + + # Bulk update dirty resources + if dirty_resources: + update_objects_in_batches( + tenant_id=tenant_id, + model=Resource, + objects=list(dirty_resources.values()), + fields=["metadata", "details", "partition", "region", "service", "type"], + batch_size=1000, + ) + + # Log skipped findings summary + if skipped_findings_count > 0: + logger.warning( + f"Scan {scan_instance.id}: Skipped {skipped_findings_count} finding(s) " + f"due to UID length exceeding 300 characters in this micro-batch." + ) def perform_prowler_scan( @@ -261,20 +639,25 @@ def perform_prowler_scan( checks_to_execute: list[str] | None = None, ): """ - Perform a scan using Prowler and store the findings and resources in the database. + Run a Prowler scan and persist all generated resources, findings, and summaries. + + The scan stream is processed in micro-batches to keep memory bounded while still + benefiting from bulk writes. When the scan completes we also derive + `ResourceScanSummary` rows and return the serialized `Scan` payload used by the + API layer. Args: - tenant_id (str): The ID of the tenant for which the scan is performed. - scan_id (str): The ID of the scan instance. - provider_id (str): The ID of the provider to scan. - checks_to_execute (list[str], optional): A list of specific checks to execute. Defaults to None. + tenant_id: Tenant that owns the scan. + scan_id: UUID of the `Scan` row being executed. + provider_id: Provider to authenticate against. + checks_to_execute: Optional subset of check IDs to run. Returns: - dict: Serialized data of the completed scan instance. + Serialized `ScanTaskSerializer` data for the updated scan. Raises: - ValueError: If the provider cannot be connected. - + ProviderConnectionError: If the provider cannot be validated before scanning. + Exception: Any downstream persistence/processing error (re-raised after cleanup). """ exception = None unique_resources = set() @@ -301,6 +684,21 @@ def perform_prowler_scan( logger.error(f"Error processing mutelist rules: {e}") mutelist_processor = None + # Load enabled mute rules for this tenant + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + try: + active_mute_rules = MuteRule.objects.filter( + tenant_id=tenant_id, enabled=True + ).values_list("finding_uids", "reason") + + mute_rules_cache = {} + for finding_uids, reason in active_mute_rules: + for uid in finding_uids: + mute_rules_cache[uid] = reason + except Exception as e: + logger.error(f"Error loading mute rules: {e}") + mute_rules_cache = {} + try: with rls_transaction(tenant_id): try: @@ -333,159 +731,26 @@ def perform_prowler_scan( resource_failed_findings_cache = defaultdict(int) for progress, findings in prowler_scan.scan(): - for finding in findings: - if finding is None: - logger.error(f"None finding detected on scan {scan_id}.") - continue - for attempt in range(CELERY_DEADLOCK_ATTEMPTS): - try: - with rls_transaction(tenant_id): - # Process resource - resource_uid = finding.resource_uid - if resource_uid not in resource_cache: - # Get or create the resource - resource_instance, _ = Resource.objects.get_or_create( - tenant_id=tenant_id, - provider=provider_instance, - uid=resource_uid, - defaults={ - "region": finding.region, - "service": finding.service_name, - "type": finding.resource_type, - "name": finding.resource_name, - }, - ) - resource_cache[resource_uid] = resource_instance + # Process findings in micro-batches + findings_list = list(findings) + total_findings = len(findings_list) - # Initialize all processed resources in the cache - resource_failed_findings_cache[resource_uid] = 0 - else: - resource_instance = resource_cache[resource_uid] + # Chunk findings into micro-batches + for i in range(0, total_findings, FINDINGS_MICRO_BATCH_SIZE): + micro_batch = findings_list[i : i + FINDINGS_MICRO_BATCH_SIZE] - # Update resource fields if necessary - updated_fields = [] - if ( - finding.region - and resource_instance.region != finding.region - ): - resource_instance.region = finding.region - updated_fields.append("region") - if resource_instance.service != finding.service_name: - resource_instance.service = finding.service_name - updated_fields.append("service") - if resource_instance.type != finding.resource_type: - resource_instance.type = finding.resource_type - updated_fields.append("type") - if resource_instance.metadata != finding.resource_metadata: - resource_instance.metadata = json.dumps( - finding.resource_metadata, cls=CustomEncoder - ) - updated_fields.append("metadata") - if resource_instance.details != finding.resource_details: - resource_instance.details = finding.resource_details - updated_fields.append("details") - if resource_instance.partition != finding.partition: - resource_instance.partition = finding.partition - updated_fields.append("partition") - if updated_fields: - with rls_transaction(tenant_id): - resource_instance.save(update_fields=updated_fields) - except (OperationalError, IntegrityError) as db_err: - if attempt < CELERY_DEADLOCK_ATTEMPTS - 1: - logger.warning( - f"{'Deadlock error' if isinstance(db_err, OperationalError) else 'Integrity error'} " - f"detected when processing resource {resource_uid} on scan {scan_id}. Retrying..." - ) - time.sleep(0.1 * (2**attempt)) - continue - else: - raise db_err - - # Update tags - tags = [] - with rls_transaction(tenant_id): - for key, value in finding.resource_tags.items(): - tag_key = (key, value) - if tag_key not in tag_cache: - tag_instance, _ = ResourceTag.objects.get_or_create( - tenant_id=tenant_id, key=key, value=value - ) - tag_cache[tag_key] = tag_instance - else: - tag_instance = tag_cache[tag_key] - tags.append(tag_instance) - resource_instance.upsert_or_delete_tags(tags=tags) - - unique_resources.add((resource_instance.uid, resource_instance.region)) - - # Process finding - with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): - finding_uid = finding.uid - last_first_seen_at = None - if finding_uid not in last_status_cache: - most_recent_finding = ( - Finding.all_objects.filter( - tenant_id=tenant_id, uid=finding_uid - ) - .order_by("-inserted_at") - .values("status", "first_seen_at") - .first() - ) - last_status = None - if most_recent_finding: - last_status = most_recent_finding["status"] - last_first_seen_at = most_recent_finding["first_seen_at"] - last_status_cache[finding_uid] = last_status, last_first_seen_at - else: - last_status, last_first_seen_at = last_status_cache[finding_uid] - - status = FindingStatus[finding.status] - delta = _create_finding_delta(last_status, status) - # For the findings prior to the change, when a first finding is found with delta!="new" it will be - # assigned a current date as first_seen_at and the successive findings with the same UID will - # always get the date of the previous finding. - # For new findings, when a finding (delta="new") is found for the first time, the first_seen_at - # attribute will be assigned the current date, the following findings will get that date. - if not last_first_seen_at: - last_first_seen_at = datetime.now(tz=timezone.utc) - - # If the finding is muted at this time the reason must be the configured Mutelist - muted_reason = "Muted by mutelist" if finding.muted else None - - # Increment failed_findings_count cache if the finding status is FAIL and not muted - if status == FindingStatus.FAIL and not finding.muted: - resource_uid = finding.resource_uid - resource_failed_findings_cache[resource_uid] += 1 - - with rls_transaction(tenant_id): - # Create the finding - finding_instance = Finding.objects.create( - tenant_id=tenant_id, - uid=finding_uid, - delta=delta, - check_metadata=finding.get_metadata(), - status=status, - status_extended=finding.status_extended, - severity=finding.severity, - impact=finding.severity, - raw_result=finding.raw, - check_id=finding.check_id, - scan=scan_instance, - first_seen_at=last_first_seen_at, - muted=finding.muted, - muted_reason=muted_reason, - compliance=finding.compliance, - ) - finding_instance.add_resources([resource_instance]) - - # Update scan resource summaries - scan_resource_cache.add( - ( - str(resource_instance.id), - resource_instance.service, - resource_instance.region, - resource_instance.type, - ) + _process_finding_micro_batch( + tenant_id=tenant_id, + findings_batch=micro_batch, + scan_instance=scan_instance, + provider_instance=provider_instance, + resource_cache=resource_cache, + tag_cache=tag_cache, + last_status_cache=last_status_cache, + resource_failed_findings_cache=resource_failed_findings_cache, + unique_resources=unique_resources, + scan_resource_cache=scan_resource_cache, + mute_rules_cache=mute_rules_cache, ) # Update scan progress @@ -558,34 +823,15 @@ def perform_prowler_scan( def aggregate_findings(tenant_id: str, scan_id: str): """ - Aggregates findings for a given scan and stores the results in the ScanSummary table. + Aggregate findings for a scan and populate `ScanSummary` rows. - This function retrieves all findings associated with a given `scan_id` and calculates various - metrics such as counts of failed, passed, and muted findings, as well as their deltas (new, - changed, unchanged). The results are grouped by `check_id`, `service`, `severity`, and `region`. - These aggregated metrics are then stored in the `ScanSummary` table. - - Additionally, it updates the failed_findings_count field for each resource based on the most - recent findings for each finding.uid. + We group findings by check/service/severity/region and compute pass/fail/muted + totals plus delta counts (new/changed/unchanged). The summary dataset feeds the + overview API and dashboards, so it is recomputed every time a scan finishes. Args: - tenant_id (str): The ID of the tenant to which the scan belongs. - scan_id (str): The ID of the scan for which findings need to be aggregated. - - Aggregated Metrics: - - fail: Total number of failed findings. - - _pass: Total number of passed findings. - - muted: Total number of muted findings. - - total: Total number of findings. - - new: Total number of new findings. - - changed: Total number of changed findings. - - unchanged: Total number of unchanged findings. - - fail_new: Failed findings with a delta of 'new'. - - fail_changed: Failed findings with a delta of 'changed'. - - pass_new: Passed findings with a delta of 'new'. - - pass_changed: Passed findings with a delta of 'changed'. - - muted_new: Muted findings with a delta of 'new'. - - muted_changed: Muted findings with a delta of 'changed'. + tenant_id: Tenant that owns the scan. + scan_id: Scan UUID whose findings should be aggregated. """ with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): findings = Finding.objects.filter(tenant_id=tenant_id, scan_id=scan_id) @@ -711,50 +957,34 @@ def aggregate_findings(tenant_id: str, scan_id: str): ScanSummary.objects.bulk_create(scan_aggregations, batch_size=3000) -def create_compliance_requirements(tenant_id: str, scan_id: str): +def _aggregate_findings_by_region( + tenant_id: str, scan_id: str, modeled_threatscore_compliance_id: str +) -> tuple[dict, dict]: """ - Create detailed compliance requirement overview records for a scan. + Aggregate findings by region using optimized ORM queries. - This function processes the compliance data collected during a scan and creates - individual records for each compliance requirement in each region. These detailed - records provide a granular view of compliance status. + Replaces nested Python loops with efficient queries and aggregation. Args: - tenant_id (str): The ID of the tenant for which to create records. - scan_id (str): The ID of the scan for which to create records. + tenant_id: Tenant UUID + scan_id: Scan UUID + modeled_threatscore_compliance_id: ID for ThreatScore compliance framework Returns: - dict: A dictionary containing the number of requirements created and the regions processed. - - Raises: - ValidationError: If tenant_id is not a valid UUID. + tuple: (check_status_by_region, findings_count_by_compliance) + - check_status_by_region: {region: {check_id: status}} + - findings_count_by_compliance: {region: {normalized_id: {requirement_id: {total, pass}}}} """ - try: - with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): - scan_instance = Scan.objects.get(pk=scan_id) - provider_instance = scan_instance.provider - prowler_provider = return_prowler_provider(provider_instance) + check_status_by_region = {} + findings_count_by_compliance = {} - compliance_template = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE[ - provider_instance.provider - ] - modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" - threatscore_requirements_by_check: dict[str, set[str]] = {} - threatscore_framework = compliance_template.get( - modeled_threatscore_compliance_id - ) - if threatscore_framework: - for requirement_id, requirement in threatscore_framework[ - "requirements" - ].items(): - for check_id in requirement["checks"]: - threatscore_requirements_by_check.setdefault(check_id, set()).add( - requirement_id - ) - - # Get check status data by region from findings + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + # Fetch findings with resources in a single efficient query + # Use select_related for finding fields and prefetch_related for many-to-many resources findings = ( - Finding.all_objects.filter(scan_id=scan_id, muted=False) + Finding.all_objects.filter( + tenant_id=tenant_id, scan_id=scan_id, muted=False + ) .only("id", "check_id", "status", "compliance") .prefetch_related( Prefetch( @@ -763,117 +993,198 @@ def create_compliance_requirements(tenant_id: str, scan_id: str): to_attr="small_resources", ) ) - .iterator(chunk_size=1000) ) - findings_count_by_compliance = {} - check_status_by_region = {} + # Process findings in a single pass (more efficient than original nested loops) + normalized_id = re.sub( + r"[^a-z0-9]", "", modeled_threatscore_compliance_id.lower() + ) + + for finding in findings: + for resource in finding.small_resources: + region = resource.region + + # Aggregate check status by region + current_status = check_status_by_region.setdefault(region, {}) + # Priority: FAIL > any other status + if current_status.get(finding.check_id) != "FAIL": + current_status[finding.check_id] = finding.status + + # Aggregate ThreatScore compliance counts + if modeled_threatscore_compliance_id in (finding.compliance or {}): + compliance_key = findings_count_by_compliance.setdefault( + region, {} + ).setdefault(normalized_id, {}) + + for requirement_id in finding.compliance[ + modeled_threatscore_compliance_id + ]: + requirement_stats = compliance_key.setdefault( + requirement_id, {"total": 0, "pass": 0} + ) + requirement_stats["total"] += 1 + if finding.status == "PASS": + requirement_stats["pass"] += 1 + + return check_status_by_region, findings_count_by_compliance + + +def create_compliance_requirements(tenant_id: str, scan_id: str): + """ + Materialize per-requirement compliance rows (and summaries) for a scan. + + Using the provider’s compliance template plus the scan’s findings, we compute a + row per (region, compliance, requirement) and write it to + `ComplianceRequirementOverview` via COPY. The same pass tally requirement + statuses so we can persist `ComplianceOverviewSummary` records for the fast + overview endpoint. + + Args: + tenant_id: Tenant running the scan. + scan_id: Scan identifier whose findings should be translated into compliance data. + + Returns: + dict: Counts/metadata about the generated rows (e.g., frameworks touched, regions processed). + """ + try: with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): - for finding in findings: - for resource in finding.small_resources: - region = resource.region - current_status = check_status_by_region.setdefault(region, {}) - if current_status.get(finding.check_id) != "FAIL": - current_status[finding.check_id] = finding.status - if modeled_threatscore_compliance_id in finding.compliance: - for requirement_id in finding.compliance[ - modeled_threatscore_compliance_id - ]: - compliance_key = findings_count_by_compliance.setdefault( - region, {} - ).setdefault( - modeled_threatscore_compliance_id.lower().replace( - "-", "" - ), - {}, - ) - if requirement_id not in compliance_key: - compliance_key[requirement_id] = { - "total": 0, - "pass": 0, - } + scan_instance = Scan.objects.get(pk=scan_id) + provider_instance = scan_instance.provider + return_prowler_provider(provider_instance) - compliance_key[requirement_id]["total"] += 1 - if finding.status == "PASS": - compliance_key[requirement_id]["pass"] += 1 + compliance_template = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE[ + provider_instance.provider + ] + modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" - try: - # Try to get regions from provider - regions = prowler_provider.get_regions() - except (AttributeError, Exception): - # If not available, use regions from findings - regions = set(check_status_by_region.keys()) - - # Create compliance data by region - compliance_overview_by_region = { - region: deepcopy(compliance_template) for region in regions - } - - # Apply check statuses to compliance data - for region, check_status in check_status_by_region.items(): - compliance_data = compliance_overview_by_region.setdefault( - region, deepcopy(compliance_template) - ) - for check_name, status in check_status.items(): - generate_scan_compliance( - compliance_data, - provider_instance.provider, - check_name, - status, - ) - - # Prepare compliance requirement rows - compliance_requirement_rows: list[dict[str, Any]] = [] - utc_datetime_now = datetime.now(tz=timezone.utc) - for region, compliance_data in compliance_overview_by_region.items(): - for compliance_id, compliance in compliance_data.items(): - modeled_compliance_id = _normalized_compliance_key( - compliance["framework"], compliance["version"] - ) - # Create an overview record for each requirement within each compliance framework - for requirement_id, requirement in compliance["requirements"].items(): - checks_status = requirement["checks_status"] - compliance_requirement_rows.append( - { - "id": uuid.uuid4(), - "tenant_id": tenant_id, - "inserted_at": utc_datetime_now, - "compliance_id": compliance_id, - "framework": compliance["framework"], - "version": compliance["version"] or "", - "description": requirement.get("description") or "", - "region": region, - "requirement_id": requirement_id, - "requirement_status": requirement["status"], - "passed_checks": checks_status["pass"], - "failed_checks": checks_status["fail"], - "total_checks": checks_status["total"], - "scan_id": scan_instance.id, - "passed_findings": findings_count_by_compliance.get( - region, {} - ) - .get(modeled_compliance_id, {}) - .get(requirement_id, {}) - .get("pass", 0), - "total_findings": findings_count_by_compliance.get( - region, {} - ) - .get(modeled_compliance_id, {}) - .get(requirement_id, {}) - .get("total", 0), - } + requirement_lookup: dict[str, list[tuple[str, str]]] = {} + for compliance_id, compliance in compliance_template.items(): + for requirement_id, requirement in compliance["requirements"].items(): + for check_id in requirement["checks"].keys(): + requirement_lookup.setdefault(check_id, []).append( + (compliance_id, requirement_id) ) - # Bulk create requirement records using PostgreSQL COPY - _persist_compliance_requirement_rows(tenant_id, compliance_requirement_rows) + compliance_requirement_rows: list[dict[str, Any]] = [] + regions = [] + requirement_statuses = defaultdict( + lambda: {"fail_count": 0, "pass_count": 0, "total_count": 0} + ) + + # Skip if provider has no compliance frameworks + if compliance_template: + # Aggregate findings by region using SQL for optimal performance + check_status_by_region, findings_count_by_compliance = ( + _aggregate_findings_by_region( + tenant_id, scan_id, modeled_threatscore_compliance_id + ) + ) + + # Only process regions that have findings (optimization: reduces row count) + regions = list(check_status_by_region.keys()) + + region_requirement_stats: dict[ + str, dict[str, dict[str, dict[str, int]]] + ] = defaultdict(lambda: defaultdict(dict)) + for region, check_status in check_status_by_region.items(): + for check_name, status in check_status.items(): + targets = requirement_lookup.get(check_name) + if not targets: + continue + status_lower = (status or "").lower() + if status_lower not in {"pass", "fail"}: + continue + for compliance_id, requirement_id in targets: + compliance_stats = region_requirement_stats[region].setdefault( + compliance_id, {} + ) + requirement_stats = compliance_stats.setdefault( + requirement_id, {"passed_checks": 0, "failed_checks": 0} + ) + if status_lower == "pass": + requirement_stats["passed_checks"] += 1 + else: + requirement_stats["failed_checks"] += 1 + + # Prepare compliance requirement rows and compute summaries in single pass + utc_datetime_now = datetime.now(tz=timezone.utc) + + # Pre-compute shared strings (optimization: reduces string conversions) + tenant_id_str = str(tenant_id) + scan_id_str = str(scan_instance.id) + + for region in regions: + region_stats = region_requirement_stats.get(region, {}) + for compliance_id, compliance in compliance_template.items(): + modeled_compliance_id = _normalized_compliance_key( + compliance["framework"], compliance["version"] + ) + compliance_stats = region_stats.get(compliance_id, {}) + # Create an overview record for each requirement within each compliance framework + for requirement_id, requirement in compliance[ + "requirements" + ].items(): + stats = compliance_stats.get(requirement_id) + passed_checks = stats["passed_checks"] if stats else 0 + failed_checks = stats["failed_checks"] if stats else 0 + total_checks = len(requirement["checks"]) + if total_checks == 0: + requirement_status = "MANUAL" + elif failed_checks > 0: + requirement_status = "FAIL" + else: + requirement_status = "PASS" + + compliance_requirement_rows.append( + { + "id": uuid.uuid4(), + "tenant_id": tenant_id_str, + "inserted_at": utc_datetime_now, + "compliance_id": compliance_id, + "framework": compliance["framework"], + "version": compliance["version"] or "", + "description": requirement.get("description") or "", + "region": region, + "requirement_id": requirement_id, + "requirement_status": requirement_status, + "passed_checks": passed_checks, + "failed_checks": failed_checks, + "total_checks": total_checks, + "scan_id": scan_id_str, + "passed_findings": findings_count_by_compliance.get( + region, {} + ) + .get(modeled_compliance_id, {}) + .get(requirement_id, {}) + .get("pass", 0), + "total_findings": findings_count_by_compliance.get( + region, {} + ) + .get(modeled_compliance_id, {}) + .get(requirement_id, {}) + .get("total", 0), + } + ) + + # Update summary tracking (single-pass optimization) + key = (compliance_id, requirement_id) + requirement_statuses[key]["total_count"] += 1 + if requirement_status == "FAIL": + requirement_statuses[key]["fail_count"] += 1 + elif requirement_status == "PASS": + requirement_statuses[key]["pass_count"] += 1 + + # Bulk create requirement records using PostgreSQL COPY + _persist_compliance_requirement_rows(tenant_id, compliance_requirement_rows) + + # Create pre-aggregated summaries for fast compliance overview lookups + _create_compliance_summaries(tenant_id, scan_id, requirement_statuses) return { "requirements_created": len(compliance_requirement_rows), "regions_processed": list(regions), "compliance_frameworks": ( - list(compliance_overview_by_region.get(list(regions)[0], {}).keys()) - if regions - else [] + list(compliance_template.keys()) if regions else [] ), } diff --git a/api/src/backend/tasks/jobs/threatscore.py b/api/src/backend/tasks/jobs/threatscore.py new file mode 100644 index 0000000000..414f2d20f2 --- /dev/null +++ b/api/src/backend/tasks/jobs/threatscore.py @@ -0,0 +1,214 @@ +from celery.utils.log import get_task_logger +from tasks.jobs.threatscore_utils import ( + _aggregate_requirement_statistics_from_database, + _calculate_requirements_data_from_statistics, +) + +from api.db_router import READ_REPLICA_ALIAS +from api.db_utils import rls_transaction +from api.models import Provider, StatusChoices +from prowler.lib.check.compliance_models import Compliance + +logger = get_task_logger(__name__) + + +def compute_threatscore_metrics( + tenant_id: str, + scan_id: str, + provider_id: str, + compliance_id: str, + min_risk_level: int = 4, +) -> dict: + """ + Compute ThreatScore metrics for a given scan. + + This function calculates all the metrics needed for a ThreatScore snapshot: + - Overall ThreatScore percentage + - Section-by-section scores + - Critical failed requirements (risk >= min_risk_level) + - Summary statistics (requirements and findings counts) + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): The ID of the scan to analyze. + provider_id (str): The ID of the provider used in the scan. + compliance_id (str): Compliance framework ID (e.g., "prowler_threatscore_aws"). + min_risk_level (int): Minimum risk level for critical requirements. Defaults to 4. + + Returns: + dict: A dictionary containing: + - overall_score (float): Overall ThreatScore percentage (0-100) + - section_scores (dict): Section name -> score percentage mapping + - critical_requirements (list): List of critical failed requirement dicts + - total_requirements (int): Total number of requirements + - passed_requirements (int): Number of PASS requirements + - failed_requirements (int): Number of FAIL requirements + - manual_requirements (int): Number of MANUAL requirements + - total_findings (int): Total findings count + - passed_findings (int): Passed findings count + - failed_findings (int): Failed findings count + + Example: + >>> metrics = compute_threatscore_metrics( + ... tenant_id="tenant-123", + ... scan_id="scan-456", + ... provider_id="provider-789", + ... compliance_id="prowler_threatscore_aws" + ... ) + >>> print(f"Overall ThreatScore: {metrics['overall_score']:.2f}%") + """ + # Get provider and compliance information + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + provider_obj = Provider.objects.get(id=provider_id) + provider_type = provider_obj.provider + + frameworks_bulk = Compliance.get_bulk(provider_type) + compliance_obj = frameworks_bulk[compliance_id] + + # Aggregate requirement statistics from database + requirement_statistics_by_check_id = ( + _aggregate_requirement_statistics_from_database(tenant_id, scan_id) + ) + + # Calculate requirements data using aggregated statistics + attributes_by_requirement_id, requirements_list = ( + _calculate_requirements_data_from_statistics( + compliance_obj, requirement_statistics_by_check_id + ) + ) + + # Initialize metrics + overall_numerator = 0 + overall_denominator = 0 + overall_has_findings = False + + sections_data = {} + + total_requirements = len(requirements_list) + passed_requirements = 0 + failed_requirements = 0 + manual_requirements = 0 + total_findings = 0 + passed_findings = 0 + failed_findings = 0 + + critical_requirements_list = [] + + # Process each requirement + for requirement in requirements_list: + requirement_id = requirement["id"] + requirement_status = requirement["attributes"]["status"] + requirement_attributes = attributes_by_requirement_id.get(requirement_id, {}) + + # Count requirements by status + if requirement_status == StatusChoices.PASS: + passed_requirements += 1 + elif requirement_status == StatusChoices.FAIL: + failed_requirements += 1 + elif requirement_status == StatusChoices.MANUAL: + manual_requirements += 1 + + # Get findings data + req_passed_findings = requirement["attributes"].get("passed_findings", 0) + req_total_findings = requirement["attributes"].get("total_findings", 0) + + # Accumulate findings counts + total_findings += req_total_findings + passed_findings += req_passed_findings + failed_findings += req_total_findings - req_passed_findings + + # Skip requirements with no findings + if req_total_findings == 0: + continue + + overall_has_findings = True + + # Get requirement metadata + metadata = requirement_attributes.get("attributes", {}).get( + "req_attributes", [] + ) + if not metadata or len(metadata) == 0: + continue + + m = metadata[0] + risk_level = getattr(m, "LevelOfRisk", 0) + weight = getattr(m, "Weight", 0) + section = getattr(m, "Section", "Unknown") + + # Calculate ThreatScore components using formula from UI + rate_i = req_passed_findings / req_total_findings + rfac_i = 1 + 0.25 * risk_level + + # Update overall score + overall_numerator += rate_i * req_total_findings * weight * rfac_i + overall_denominator += req_total_findings * weight * rfac_i + + # Update section scores + if section not in sections_data: + sections_data[section] = { + "numerator": 0, + "denominator": 0, + "has_findings": False, + } + + sections_data[section]["has_findings"] = True + sections_data[section]["numerator"] += ( + rate_i * req_total_findings * weight * rfac_i + ) + sections_data[section]["denominator"] += req_total_findings * weight * rfac_i + + # Identify critical failed requirements + if requirement_status == StatusChoices.FAIL and risk_level >= min_risk_level: + critical_requirements_list.append( + { + "requirement_id": requirement_id, + "title": getattr(m, "Title", "N/A"), + "section": section, + "subsection": getattr(m, "SubSection", "N/A"), + "risk_level": risk_level, + "weight": weight, + "passed_findings": req_passed_findings, + "total_findings": req_total_findings, + "description": getattr(m, "AttributeDescription", "N/A"), + } + ) + + # Calculate overall ThreatScore + if not overall_has_findings: + overall_score = 100.0 + elif overall_denominator > 0: + overall_score = (overall_numerator / overall_denominator) * 100 + else: + overall_score = 0.0 + + # Calculate section scores + section_scores = {} + for section, data in sections_data.items(): + if data["has_findings"] and data["denominator"] > 0: + section_scores[section] = (data["numerator"] / data["denominator"]) * 100 + else: + section_scores[section] = 100.0 + + # Sort critical requirements by risk level (desc) and weight (desc) + critical_requirements_list.sort( + key=lambda x: (x["risk_level"], x["weight"]), reverse=True + ) + + logger.info( + f"ThreatScore computed: {overall_score:.2f}% " + f"({passed_requirements}/{total_requirements} requirements passed, " + f"{len(critical_requirements_list)} critical failures)" + ) + + return { + "overall_score": round(overall_score, 2), + "section_scores": {k: round(v, 2) for k, v in section_scores.items()}, + "critical_requirements": critical_requirements_list, + "total_requirements": total_requirements, + "passed_requirements": passed_requirements, + "failed_requirements": failed_requirements, + "manual_requirements": manual_requirements, + "total_findings": total_findings, + "passed_findings": passed_findings, + "failed_findings": failed_findings, + } diff --git a/api/src/backend/tasks/jobs/threatscore_utils.py b/api/src/backend/tasks/jobs/threatscore_utils.py new file mode 100644 index 0000000000..bc2cb009a2 --- /dev/null +++ b/api/src/backend/tasks/jobs/threatscore_utils.py @@ -0,0 +1,127 @@ +from celery.utils.log import get_task_logger +from django.db.models import Count, Q + +from api.db_router import READ_REPLICA_ALIAS +from api.db_utils import rls_transaction +from api.models import Finding, StatusChoices + +logger = get_task_logger(__name__) + + +def _aggregate_requirement_statistics_from_database( + tenant_id: str, scan_id: str +) -> dict[str, dict[str, int]]: + """ + Aggregate finding statistics by check_id using database aggregation. + + This function uses Django ORM aggregation to calculate pass/fail statistics + entirely in the database, avoiding the need to load findings into memory. + + Args: + tenant_id (str): The tenant ID for Row-Level Security context. + scan_id (str): The ID of the scan to retrieve findings for. + + Returns: + dict[str, dict[str, int]]: Dictionary mapping check_id to statistics: + - 'passed' (int): Number of passed findings for this check + - 'total' (int): Total number of findings for this check + + Example: + { + 'aws_iam_user_mfa_enabled': {'passed': 10, 'total': 15}, + 'aws_s3_bucket_public_access': {'passed': 0, 'total': 5} + } + """ + requirement_statistics_by_check_id = {} + + with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS): + aggregated_statistics_queryset = ( + Finding.all_objects.filter(tenant_id=tenant_id, scan_id=scan_id) + .values("check_id") + .annotate( + total_findings=Count("id"), + passed_findings=Count("id", filter=Q(status=StatusChoices.PASS)), + ) + ) + + for aggregated_stat in aggregated_statistics_queryset: + check_id = aggregated_stat["check_id"] + requirement_statistics_by_check_id[check_id] = { + "passed": aggregated_stat["passed_findings"], + "total": aggregated_stat["total_findings"], + } + + logger.info( + f"Aggregated statistics for {len(requirement_statistics_by_check_id)} unique checks" + ) + return requirement_statistics_by_check_id + + +def _calculate_requirements_data_from_statistics( + compliance_obj, requirement_statistics_by_check_id: dict[str, dict[str, int]] +) -> tuple[dict[str, dict], list[dict]]: + """ + Calculate requirement status and statistics using pre-aggregated database statistics. + + Args: + compliance_obj: The compliance framework object containing requirements. + requirement_statistics_by_check_id (dict[str, dict[str, int]]): Pre-aggregated statistics + mapping check_id to {'passed': int, 'total': int} counts. + + Returns: + tuple[dict[str, dict], list[dict]]: A tuple containing: + - attributes_by_requirement_id: Dictionary mapping requirement IDs to their attributes. + - requirements_list: List of requirement dictionaries with status and statistics. + """ + attributes_by_requirement_id = {} + requirements_list = [] + + compliance_framework = getattr(compliance_obj, "Framework", "N/A") + compliance_version = getattr(compliance_obj, "Version", "N/A") + + for requirement in compliance_obj.Requirements: + requirement_id = requirement.Id + requirement_description = getattr(requirement, "Description", "") + requirement_checks = getattr(requirement, "Checks", []) + requirement_attributes = getattr(requirement, "Attributes", []) + + attributes_by_requirement_id[requirement_id] = { + "attributes": { + "req_attributes": requirement_attributes, + "checks": requirement_checks, + }, + "description": requirement_description, + } + + total_passed_findings = 0 + total_findings_count = 0 + + for check_id in requirement_checks: + if check_id in requirement_statistics_by_check_id: + check_statistics = requirement_statistics_by_check_id[check_id] + total_findings_count += check_statistics["total"] + total_passed_findings += check_statistics["passed"] + + if total_findings_count > 0: + if total_passed_findings == total_findings_count: + requirement_status = StatusChoices.PASS + else: + requirement_status = StatusChoices.FAIL + else: + requirement_status = StatusChoices.MANUAL + + requirements_list.append( + { + "id": requirement_id, + "attributes": { + "framework": compliance_framework, + "version": compliance_version, + "status": requirement_status, + "description": requirement_description, + "passed_findings": total_passed_findings, + "total_findings": total_findings_count, + }, + } + ) + + return attributes_by_requirement_id, requirements_list diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index 97b320b71d..9524a1b985 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -8,7 +8,10 @@ from celery.utils.log import get_task_logger from config.celery import RLSTask from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY from django_celery_beat.models import PeriodicTask -from tasks.jobs.backfill import backfill_resource_scan_summaries +from tasks.jobs.backfill import ( + backfill_compliance_summaries, + backfill_resource_scan_summaries, +) from tasks.jobs.connection import ( check_integration_connection, check_lighthouse_connection, @@ -31,6 +34,7 @@ from tasks.jobs.lighthouse_providers import ( check_lighthouse_provider_connection, refresh_lighthouse_provider_models, ) +from tasks.jobs.muting import mute_historical_findings from tasks.jobs.report import generate_threatscore_report_job from tasks.jobs.scan import ( aggregate_findings, @@ -493,6 +497,21 @@ def backfill_scan_resource_summaries_task(tenant_id: str, scan_id: str): return backfill_resource_scan_summaries(tenant_id=tenant_id, scan_id=scan_id) +@shared_task(name="backfill-compliance-summaries", queue="backfill") +def backfill_compliance_summaries_task(tenant_id: str, scan_id: str): + """ + Tries to backfill compliance overview summaries for a completed scan. + + This task aggregates compliance requirement data across regions + to create pre-computed summary records for fast compliance overview queries. + + Args: + tenant_id (str): The tenant identifier. + scan_id (str): The scan identifier. + """ + return backfill_compliance_summaries(tenant_id=tenant_id, scan_id=scan_id) + + @shared_task(base=RLSTask, name="scan-compliance-overviews", queue="compliance") def create_compliance_requirements_task(tenant_id: str, scan_id: str): """ @@ -681,3 +700,25 @@ def generate_threatscore_report_task(tenant_id: str, scan_id: str, provider_id: return generate_threatscore_report_job( tenant_id=tenant_id, scan_id=scan_id, provider_id=provider_id ) + + +@shared_task(name="findings-mute-historical") +def mute_historical_findings_task(tenant_id: str, mute_rule_id: str): + """ + Background task to mute all historical findings matching a mute rule. + + This task processes findings in batches to avoid memory issues with large datasets. + It updates the Finding.muted, Finding.muted_at, and Finding.muted_reason fields + for all findings whose UID is in the mute rule's finding_uids list. + + Args: + tenant_id (str): The tenant ID for RLS context. + mute_rule_id (str): The primary key of the MuteRule to apply. + + Returns: + dict: A dictionary containing: + - 'findings_muted' (int): Total number of findings muted. + - 'rule_id' (str): The mute rule ID. + - 'status' (str): Final status ('completed'). + """ + return mute_historical_findings(tenant_id, mute_rule_id) diff --git a/api/src/backend/tasks/tests/test_backfill.py b/api/src/backend/tasks/tests/test_backfill.py index b436f13151..13fa481b2e 100644 --- a/api/src/backend/tasks/tests/test_backfill.py +++ b/api/src/backend/tasks/tests/test_backfill.py @@ -1,43 +1,53 @@ from uuid import uuid4 import pytest -from tasks.jobs.backfill import backfill_resource_scan_summaries +from tasks.jobs.backfill import ( + backfill_compliance_summaries, + backfill_resource_scan_summaries, +) -from api.models import ResourceScanSummary, Scan, StateChoices +from api.models import ( + ComplianceOverviewSummary, + ResourceScanSummary, + Scan, + StateChoices, +) + + +@pytest.fixture(scope="function") +def resource_scan_summary_data(scans_fixture): + scan = scans_fixture[0] + return ResourceScanSummary.objects.create( + tenant_id=scan.tenant_id, + scan_id=scan.id, + resource_id=str(uuid4()), + service="aws", + region="us-east-1", + resource_type="instance", + ) + + +@pytest.fixture(scope="function") +def get_not_completed_scans(providers_fixture): + provider_id = providers_fixture[0].id + tenant_id = providers_fixture[0].tenant_id + scan_1 = Scan.objects.create( + tenant_id=tenant_id, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.EXECUTING, + provider_id=provider_id, + ) + scan_2 = Scan.objects.create( + tenant_id=tenant_id, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.AVAILABLE, + provider_id=provider_id, + ) + return scan_1, scan_2 @pytest.mark.django_db class TestBackfillResourceScanSummaries: - @pytest.fixture(scope="function") - def resource_scan_summary_data(self, scans_fixture): - scan = scans_fixture[0] - return ResourceScanSummary.objects.create( - tenant_id=scan.tenant_id, - scan_id=scan.id, - resource_id=str(uuid4()), - service="aws", - region="us-east-1", - resource_type="instance", - ) - - @pytest.fixture(scope="function") - def get_not_completed_scans(self, providers_fixture): - provider_id = providers_fixture[0].id - tenant_id = providers_fixture[0].tenant_id - scan_1 = Scan.objects.create( - tenant_id=tenant_id, - trigger=Scan.TriggerChoices.MANUAL, - state=StateChoices.EXECUTING, - provider_id=provider_id, - ) - scan_2 = Scan.objects.create( - tenant_id=tenant_id, - trigger=Scan.TriggerChoices.MANUAL, - state=StateChoices.AVAILABLE, - provider_id=provider_id, - ) - return scan_1, scan_2 - def test_already_backfilled(self, resource_scan_summary_data): tenant_id = resource_scan_summary_data.tenant_id scan_id = resource_scan_summary_data.scan_id @@ -77,3 +87,88 @@ class TestBackfillResourceScanSummaries: assert summary.service == resource.service assert summary.region == resource.region assert summary.resource_type == resource.type + + def test_no_resources_to_backfill(self, scans_fixture): + scan = scans_fixture[1] # Failed scan with no findings/resources + tenant_id = str(scan.tenant_id) + scan_id = str(scan.id) + + result = backfill_resource_scan_summaries(tenant_id, scan_id) + + assert result == {"status": "no resources to backfill"} + + +@pytest.mark.django_db +class TestBackfillComplianceSummaries: + def test_already_backfilled(self, scans_fixture): + scan = scans_fixture[0] + tenant_id = str(scan.tenant_id) + ComplianceOverviewSummary.objects.create( + tenant_id=scan.tenant_id, + scan=scan, + compliance_id="aws_account_security_onboarding_aws", + requirements_passed=1, + requirements_failed=0, + requirements_manual=0, + total_requirements=1, + ) + + result = backfill_compliance_summaries(tenant_id, str(scan.id)) + + assert result == {"status": "already backfilled"} + + def test_not_completed_scan(self, get_not_completed_scans): + for scan in get_not_completed_scans: + result = backfill_compliance_summaries(str(scan.tenant_id), str(scan.id)) + assert result == {"status": "scan is not completed"} + + def test_no_compliance_data(self, scans_fixture): + scan = scans_fixture[1] # Failed scan with no compliance rows + + result = backfill_compliance_summaries(str(scan.tenant_id), str(scan.id)) + + assert result == {"status": "no compliance data to backfill"} + + def test_backfill_creates_compliance_summaries( + self, tenants_fixture, scans_fixture, compliance_requirements_overviews_fixture + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + result = backfill_compliance_summaries(str(tenant.id), str(scan.id)) + + expected = { + "aws_account_security_onboarding_aws": { + "requirements_passed": 1, + "requirements_failed": 1, + "requirements_manual": 1, + "total_requirements": 3, + }, + "cis_1.4_aws": { + "requirements_passed": 0, + "requirements_failed": 1, + "requirements_manual": 0, + "total_requirements": 1, + }, + "mitre_attack_aws": { + "requirements_passed": 0, + "requirements_failed": 1, + "requirements_manual": 0, + "total_requirements": 1, + }, + } + + assert result == {"status": "backfilled", "inserted": len(expected)} + + summaries = ComplianceOverviewSummary.objects.filter( + tenant_id=str(tenant.id), scan_id=str(scan.id) + ) + assert summaries.count() == len(expected) + + for summary in summaries: + assert summary.compliance_id in expected + expected_counts = expected[summary.compliance_id] + assert summary.requirements_passed == expected_counts["requirements_passed"] + assert summary.requirements_failed == expected_counts["requirements_failed"] + assert summary.requirements_manual == expected_counts["requirements_manual"] + assert summary.total_requirements == expected_counts["total_requirements"] diff --git a/api/src/backend/tasks/tests/test_integrations.py b/api/src/backend/tasks/tests/test_integrations.py index b8e06c9e7b..75f07677a0 100644 --- a/api/src/backend/tasks/tests/test_integrations.py +++ b/api/src/backend/tasks/tests/test_integrations.py @@ -9,6 +9,7 @@ from tasks.jobs.integrations import ( upload_security_hub_integration, ) +from api.db_router import READ_REPLICA_ALIAS, MainRouter from api.models import Integration from api.utils import prowler_integration_connection_test from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection @@ -880,7 +881,8 @@ class TestSecurityHubIntegrationUploads: # Verify RLS transaction was used correctly # Should be called twice: once for getting provider info, once for resetting regions assert mock_rls.call_count == 2 - mock_rls.assert_any_call(tenant_id) + mock_rls.assert_any_call(tenant_id, using=READ_REPLICA_ALIAS) + mock_rls.assert_any_call(tenant_id, using=MainRouter.default_db) # Verify test_connection was called with integration credentials (not provider's) mock_test_connection.assert_called_once_with( diff --git a/api/src/backend/tasks/tests/test_muting.py b/api/src/backend/tasks/tests/test_muting.py new file mode 100644 index 0000000000..d8ae310f2e --- /dev/null +++ b/api/src/backend/tasks/tests/test_muting.py @@ -0,0 +1,532 @@ +from datetime import datetime, timezone +from uuid import uuid4 + +import pytest +from django.core.exceptions import ObjectDoesNotExist +from tasks.jobs.muting import mute_historical_findings + +from api.models import Finding, MuteRule +from prowler.lib.check.models import Severity +from prowler.lib.outputs.finding import Status + + +@pytest.mark.django_db +class TestMuteHistoricalFindings: + """ + Test suite for the mute_historical_findings function. + + This class tests the batch processing of findings to update their muted status + based on MuteRule criteria. + """ + + @pytest.fixture(scope="function") + def test_user(self, create_test_user): + """Create a test user for mute rule creation.""" + return create_test_user + + @pytest.fixture(scope="function") + def mute_rule_with_findings(self, tenants_fixture, findings_fixture, test_user): + """ + Create a mute rule that targets the first finding in the fixture. + """ + tenant = tenants_fixture[0] + finding = findings_fixture[0] + + mute_rule = MuteRule.objects.create( + tenant_id=tenant.id, + name="Test Mute Rule", + reason="Testing mute functionality", + enabled=True, + created_by=test_user, + finding_uids=[finding.uid], + ) + + return mute_rule + + @pytest.fixture(scope="function") + def mute_rule_multiple_findings(self, scans_fixture, test_user): + """ + Create multiple unmuted findings and a mute rule targeting all of them. + """ + scan = scans_fixture[0] + tenant_id = scan.tenant_id + + # Create 5 unmuted findings + finding_uids = [] + for i in range(5): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"test_finding_uid_mute_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Test status {i}", + impact=Severity.high, + severity=Severity.high, + raw_result={ + "status": Status.FAIL, + "impact": Severity.high, + "severity": Severity.high, + }, + check_id=f"test_check_id_{i}", + check_metadata={ + "CheckId": f"test_check_id_{i}", + "Description": f"Test description {i}", + }, + muted=False, + ) + finding_uids.append(finding.uid) + + # Create mute rule targeting all findings + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Multiple Findings Mute Rule", + reason="Testing batch muting", + enabled=True, + created_by=test_user, + finding_uids=finding_uids, + ) + + return mute_rule, finding_uids + + @pytest.fixture(scope="function") + def mute_rule_already_muted(self, findings_fixture, test_user): + """ + Create a mute rule that targets an already-muted finding. + """ + tenant_id = findings_fixture[1].tenant_id + already_muted_finding = findings_fixture[1] + + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Already Muted Rule", + reason="Testing already muted findings", + enabled=True, + created_by=test_user, + finding_uids=[already_muted_finding.uid], + ) + + return mute_rule + + @pytest.fixture(scope="function") + def mute_rule_mixed_findings(self, scans_fixture, test_user): + """ + Create a mute rule with a mix of muted and unmuted findings. + """ + scan = scans_fixture[0] + tenant_id = scan.tenant_id + + # Create 3 unmuted findings + unmuted_uids = [] + for i in range(3): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"unmuted_finding_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Unmuted status {i}", + impact=Severity.medium, + severity=Severity.medium, + raw_result={ + "status": Status.FAIL, + "impact": Severity.medium, + "severity": Severity.medium, + }, + check_id=f"unmuted_check_{i}", + check_metadata={ + "CheckId": f"unmuted_check_{i}", + "Description": f"Unmuted description {i}", + }, + muted=False, + ) + unmuted_uids.append(finding.uid) + + # Create 2 already muted findings + muted_uids = [] + for i in range(2): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"muted_finding_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Muted status {i}", + impact=Severity.low, + severity=Severity.low, + raw_result={ + "status": Status.FAIL, + "impact": Severity.low, + "severity": Severity.low, + }, + check_id=f"muted_check_{i}", + check_metadata={ + "CheckId": f"muted_check_{i}", + "Description": f"Muted description {i}", + }, + muted=True, + muted_at=datetime.now(timezone.utc), + muted_reason="Already muted", + ) + muted_uids.append(finding.uid) + + # Create mute rule targeting all findings + all_uids = unmuted_uids + muted_uids + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Mixed Findings Rule", + reason="Testing mixed muted/unmuted findings", + enabled=True, + created_by=test_user, + finding_uids=all_uids, + ) + + return mute_rule, unmuted_uids, muted_uids + + @pytest.fixture(scope="function") + def mute_rule_batch_test(self, scans_fixture, test_user): + """ + Create enough findings to test batch processing (>1000 for default batch size). + """ + scan = scans_fixture[0] + tenant_id = scan.tenant_id + + # Create 1500 findings to exceed default batch size of 1000 + finding_uids = [] + for i in range(1500): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"batch_test_finding_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Batch test status {i}", + impact=Severity.critical, + severity=Severity.critical, + raw_result={ + "status": Status.FAIL, + "impact": Severity.critical, + "severity": Severity.critical, + }, + check_id=f"batch_test_check_{i}", + check_metadata={ + "CheckId": f"batch_test_check_{i}", + "Description": f"Batch test description {i}", + }, + muted=False, + ) + finding_uids.append(finding.uid) + + # Create mute rule targeting all findings + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Batch Processing Rule", + reason="Testing batch processing functionality", + enabled=True, + created_by=test_user, + finding_uids=finding_uids, + ) + + return mute_rule, finding_uids + + def test_mute_historical_findings_single_finding( + self, mute_rule_with_findings, findings_fixture + ): + """ + Test muting a single historical finding. + """ + mute_rule = mute_rule_with_findings + tenant_id = str(mute_rule.tenant_id) + finding = findings_fixture[0] + + # Ensure the finding is not muted before execution + finding.refresh_from_db() + assert finding.muted is False + assert finding.muted_at is None + assert finding.muted_reason is None + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify return value + assert result["findings_muted"] == 1 + assert result["rule_id"] == str(mute_rule.id) + + # Verify the finding was muted + finding.refresh_from_db() + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + def test_mute_historical_findings_multiple_findings( + self, mute_rule_multiple_findings + ): + """ + Test muting multiple historical findings. + """ + mute_rule, finding_uids = mute_rule_multiple_findings + tenant_id = str(mute_rule.tenant_id) + + # Verify all findings are unmuted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids) + assert findings.count() == 5 + for finding in findings: + assert finding.muted is False + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify return value + assert result["findings_muted"] == 5 + assert result["rule_id"] == str(mute_rule.id) + + # Verify all findings were muted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids) + for finding in findings: + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + def test_mute_historical_findings_already_muted( + self, mute_rule_already_muted, findings_fixture + ): + """ + Test that already-muted findings are not counted or updated. + """ + mute_rule = mute_rule_already_muted + tenant_id = str(mute_rule.tenant_id) + finding = findings_fixture[1] + + # Verify the finding is already muted + finding.refresh_from_db() + assert finding.muted is True + original_muted_at = finding.muted_at + original_muted_reason = finding.muted_reason + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify no findings were muted + assert result["findings_muted"] == 0 + assert result["rule_id"] == str(mute_rule.id) + + # Verify the finding's mute status did not change + finding.refresh_from_db() + assert finding.muted is True + assert finding.muted_at == original_muted_at + assert finding.muted_reason == original_muted_reason + + def test_mute_historical_findings_mixed_status(self, mute_rule_mixed_findings): + """ + Test muting when some findings are already muted and others are not. + """ + mute_rule, unmuted_uids, muted_uids = mute_rule_mixed_findings + tenant_id = str(mute_rule.tenant_id) + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify only unmuted findings were counted + assert result["findings_muted"] == 3 + assert result["rule_id"] == str(mute_rule.id) + + # Verify unmuted findings are now muted + unmuted_findings = Finding.objects.filter( + tenant_id=tenant_id, uid__in=unmuted_uids + ) + for finding in unmuted_findings: + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + # Verify already-muted findings remained unchanged + already_muted_findings = Finding.objects.filter( + tenant_id=tenant_id, uid__in=muted_uids + ) + for finding in already_muted_findings: + assert finding.muted is True + assert finding.muted_reason == "Already muted" + + def test_mute_historical_findings_nonexistent_rule(self, tenants_fixture): + """ + Test that a nonexistent mute rule raises ObjectDoesNotExist. + """ + tenant_id = str(tenants_fixture[0].id) + nonexistent_rule_id = str(uuid4()) + + with pytest.raises(ObjectDoesNotExist): + mute_historical_findings(tenant_id, nonexistent_rule_id) + + def test_mute_historical_findings_no_matching_findings( + self, tenants_fixture, test_user + ): + """ + Test muting when no findings match the rule's UIDs. + """ + tenant_id = str(tenants_fixture[0].id) + + # Create a mute rule with non-existent finding UIDs + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test No Match Rule", + reason="Testing no matching findings", + enabled=True, + created_by=test_user, + finding_uids=[ + "nonexistent_uid_1", + "nonexistent_uid_2", + "nonexistent_uid_3", + ], + ) + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify no findings were muted + assert result["findings_muted"] == 0 + assert result["rule_id"] == str(mute_rule.id) + + def test_mute_historical_findings_batch_processing(self, mute_rule_batch_test): + """ + Test that large numbers of findings are processed in batches correctly. + """ + mute_rule, finding_uids = mute_rule_batch_test + tenant_id = str(mute_rule.tenant_id) + + # Verify all findings exist and are unmuted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids) + assert findings.count() == 1500 + for finding in findings: + assert finding.muted is False + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify return value + assert result["findings_muted"] == 1500 + assert result["rule_id"] == str(mute_rule.id) + + # Verify all findings were muted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=finding_uids) + for finding in findings: + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + def test_mute_historical_findings_preserves_muted_at_timestamp( + self, mute_rule_with_findings, findings_fixture + ): + """ + Test that muted_at is set to the rule's inserted_at, not the current time. + """ + mute_rule = mute_rule_with_findings + tenant_id = str(mute_rule.tenant_id) + finding = findings_fixture[0] + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify the finding was muted + assert result["findings_muted"] == 1 + + # Verify muted_at matches the rule's inserted_at timestamp + finding.refresh_from_db() + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_at is not None + + def test_mute_historical_findings_partial_match(self, scans_fixture, test_user): + """ + Test muting when only some of the rule's UIDs exist as findings. + """ + scan = scans_fixture[0] + tenant_id = str(scan.tenant_id) + + # Create 3 findings + existing_uids = [] + for i in range(3): + finding = Finding.objects.create( + tenant_id=tenant_id, + uid=f"partial_match_finding_{i}", + scan=scan, + status=Status.FAIL, + status_extended=f"Partial match status {i}", + impact=Severity.high, + severity=Severity.high, + raw_result={ + "status": Status.FAIL, + "impact": Severity.high, + "severity": Severity.high, + }, + check_id=f"partial_match_check_{i}", + check_metadata={ + "CheckId": f"partial_match_check_{i}", + "Description": f"Partial match description {i}", + }, + muted=False, + ) + existing_uids.append(finding.uid) + + # Create a mute rule with both existing and non-existing UIDs + all_uids = existing_uids + [ + "nonexistent_uid_1", + "nonexistent_uid_2", + ] + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Partial Match Rule", + reason="Testing partial matching", + enabled=True, + created_by=test_user, + finding_uids=all_uids, + ) + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify only existing findings were muted + assert result["findings_muted"] == 3 + assert result["rule_id"] == str(mute_rule.id) + + # Verify the existing findings were muted + findings = Finding.objects.filter(tenant_id=tenant_id, uid__in=existing_uids) + assert findings.count() == 3 + for finding in findings: + assert finding.muted is True + assert finding.muted_at == mute_rule.inserted_at + assert finding.muted_reason == mute_rule.reason + + def test_mute_historical_findings_empty_uids(self, tenants_fixture, test_user): + """ + Test muting when the rule has an empty finding_uids array. + """ + tenant_id = str(tenants_fixture[0].id) + + # Create a mute rule with empty finding_uids + mute_rule = MuteRule.objects.create( + tenant_id=tenant_id, + name="Test Empty UIDs Rule", + reason="Testing empty UIDs", + enabled=True, + created_by=test_user, + finding_uids=[], + ) + + # Execute the muting function + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify no findings were muted + assert result["findings_muted"] == 0 + assert result["rule_id"] == str(mute_rule.id) + + def test_mute_historical_findings_return_format(self, mute_rule_with_findings): + """ + Test that the return value has the correct format and fields. + """ + mute_rule = mute_rule_with_findings + tenant_id = str(mute_rule.tenant_id) + + result = mute_historical_findings(tenant_id, str(mute_rule.id)) + + # Verify return value structure + assert isinstance(result, dict) + assert "findings_muted" in result + assert "rule_id" in result + assert isinstance(result["findings_muted"], int) + assert isinstance(result["rule_id"], str) + assert result["rule_id"] == str(mute_rule.id) diff --git a/api/src/backend/tasks/tests/test_report.py b/api/src/backend/tasks/tests/test_report.py index a472067b2d..91d3b1145d 100644 --- a/api/src/backend/tasks/tests/test_report.py +++ b/api/src/backend/tasks/tests/test_report.py @@ -1,19 +1,25 @@ import uuid +from datetime import timedelta +from decimal import Decimal from pathlib import Path from unittest.mock import MagicMock, patch import matplotlib import pytest +from django.utils import timezone +from freezegun import freeze_time from tasks.jobs.report import ( - _aggregate_requirement_statistics_from_database, - _calculate_requirements_data_from_statistics, _load_findings_for_requirement_checks, generate_threatscore_report, generate_threatscore_report_job, ) +from tasks.jobs.threatscore_utils import ( + _aggregate_requirement_statistics_from_database, + _calculate_requirements_data_from_statistics, +) from tasks.tasks import generate_threatscore_report_task -from api.models import Finding, StatusChoices +from api.models import Finding, Scan, StateChoices, StatusChoices, ThreatScoreSnapshot from prowler.lib.check.models import Severity matplotlib.use("Agg") # Use non-interactive backend for tests @@ -39,6 +45,7 @@ class TestGenerateThreatscoreReport: assert result == {"upload": False} mock_filter.assert_called_once_with(scan_id=self.scan_id) + @patch("tasks.jobs.report.ThreatScoreSnapshot.objects.create") @patch("tasks.jobs.report.rmtree") @patch("tasks.jobs.report._upload_to_s3") @patch("tasks.jobs.report.generate_threatscore_report") @@ -53,6 +60,7 @@ class TestGenerateThreatscoreReport: mock_generate_report, mock_upload, mock_rmtree, + mock_snapshot_create, ): mock_scan_summary_filter.return_value.exists.return_value = True @@ -95,8 +103,10 @@ class TestGenerateThreatscoreReport: Path("/tmp/threatscore_path_threatscore_report.pdf").parent, ignore_errors=True, ) + mock_snapshot_create.assert_called_once() - def test_generate_threatscore_report_fails_upload(self): + @patch("tasks.jobs.report.ThreatScoreSnapshot.objects.create") + def test_generate_threatscore_report_fails_upload(self, mock_snapshot_create): with ( patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter, patch("tasks.jobs.report.Provider.objects.get") as mock_provider_get, @@ -125,8 +135,12 @@ class TestGenerateThreatscoreReport: ) assert result == {"upload": False} + mock_snapshot_create.assert_called_once() - def test_generate_threatscore_report_logs_rmtree_exception(self, caplog): + @patch("tasks.jobs.report.ThreatScoreSnapshot.objects.create") + def test_generate_threatscore_report_logs_rmtree_exception( + self, mock_snapshot_create, caplog + ): with ( patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter, patch("tasks.jobs.report.Provider.objects.get") as mock_provider_get, @@ -160,8 +174,10 @@ class TestGenerateThreatscoreReport: provider_id=self.provider_id, ) assert "Error deleting output files" in caplog.text + mock_snapshot_create.assert_called_once() - def test_generate_threatscore_report_azure_provider(self): + @patch("tasks.jobs.report.ThreatScoreSnapshot.objects.create") + def test_generate_threatscore_report_azure_provider(self, mock_snapshot_create): with ( patch("tasks.jobs.report.ScanSummary.objects.filter") as mock_filter, patch("tasks.jobs.report.Provider.objects.get") as mock_provider_get, @@ -200,6 +216,135 @@ class TestGenerateThreatscoreReport: only_failed=True, min_risk_level=4, ) + mock_snapshot_create.assert_called_once() + + @patch("tasks.jobs.report.rmtree") + @patch( + "tasks.jobs.report._upload_to_s3", + return_value="s3://bucket/threatscore/threatscore_report.pdf", + ) + @patch("tasks.jobs.report.generate_threatscore_report") + @patch("tasks.jobs.report._generate_output_directory") + @patch("tasks.jobs.report.ScanSummary.objects.filter") + @patch("tasks.jobs.report.compute_threatscore_metrics") + @pytest.mark.django_db + @freeze_time("2025-01-10T12:00:00Z") + def test_generate_threatscore_report_persists_snapshot_and_delta( + self, + mock_compute_metrics, + mock_scan_summary_filter, + mock_generate_output_directory, + mock_generate_report, + mock_upload, + mock_rmtree, + tenants_fixture, + providers_fixture, + ): + tenant = tenants_fixture[0] + provider = providers_fixture[0] + + scan_previous = Scan.objects.create( + tenant=tenant, + provider=provider, + name="previous-threatscore-scan", + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + started_at=timezone.now() - timedelta(hours=4), + completed_at=timezone.now() - timedelta(hours=3), + ) + ThreatScoreSnapshot.objects.create( + tenant=tenant, + scan=scan_previous, + provider=provider, + compliance_id="prowler_threatscore_aws", + overall_score=Decimal("70.00"), + score_delta=None, + section_scores={"1. IAM": "65.00"}, + critical_requirements=[], + total_requirements=50, + passed_requirements=35, + failed_requirements=15, + manual_requirements=0, + total_findings=40, + passed_findings=25, + failed_findings=15, + ) + + scan_current = Scan.objects.create( + tenant=tenant, + provider=provider, + name="current-threatscore-scan", + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + started_at=timezone.now() - timedelta(hours=2), + completed_at=timezone.now() - timedelta(hours=1), + ) + + mock_scan_summary_filter.return_value.exists.return_value = True + mock_generate_output_directory.return_value = ( + "/tmp/output", + "/tmp/compressed", + "/tmp/threatscore_path", + ) + + metrics = { + "overall_score": 85.5, + "score_delta": 10.0, + "section_scores": {"1. IAM": 82.3, "2. Attack Surface": 60.0}, + "critical_requirements": [ + { + "requirement_id": "req_new", + "title": "New high-risk requirement", + "section": "1. IAM", + "subsection": "Root Account", + "risk_level": 5, + "weight": 150, + "passed_findings": 7, + "total_findings": 10, + "description": "Critical requirement description", + } + ], + "total_requirements": 140, + "passed_requirements": 100, + "failed_requirements": 40, + "manual_requirements": 0, + "total_findings": 200, + "passed_findings": 150, + "failed_findings": 50, + } + mock_compute_metrics.return_value = metrics + + result = generate_threatscore_report_job( + tenant_id=str(tenant.id), + scan_id=str(scan_current.id), + provider_id=str(provider.id), + ) + + assert result == {"upload": True} + mock_compute_metrics.assert_called_once_with( + tenant_id=str(tenant.id), + scan_id=str(scan_current.id), + provider_id=str(provider.id), + compliance_id="prowler_threatscore_aws", + min_risk_level=4, + ) + mock_generate_report.assert_called_once() + mock_upload.assert_called_once() + mock_rmtree.assert_called_once() + + snapshots = ThreatScoreSnapshot.objects.filter( + tenant=tenant, provider=provider + ).order_by("inserted_at") + assert snapshots.count() == 2 + + new_snapshot = ThreatScoreSnapshot.objects.get(scan=scan_current) + assert new_snapshot.compliance_id == "prowler_threatscore_aws" + assert Decimal(new_snapshot.overall_score) == Decimal("85.50") + assert Decimal(new_snapshot.score_delta) == Decimal("15.50") + assert new_snapshot.section_scores == metrics["section_scores"] + assert new_snapshot.critical_requirements == metrics["critical_requirements"] + assert new_snapshot.total_requirements == metrics["total_requirements"] + assert new_snapshot.total_findings == metrics["total_findings"] @pytest.mark.django_db diff --git a/api/src/backend/tasks/tests/test_scan.py b/api/src/backend/tasks/tests/test_scan.py index 62c9a2b7d1..f2724de4e6 100644 --- a/api/src/backend/tasks/tests/test_scan.py +++ b/api/src/backend/tasks/tests/test_scan.py @@ -1,16 +1,23 @@ import csv import json +import re import uuid +from contextlib import contextmanager from datetime import datetime, timezone from io import StringIO from unittest.mock import MagicMock, patch import pytest from tasks.jobs.scan import ( + _aggregate_findings_by_region, _copy_compliance_requirement_rows, + _create_compliance_summaries, _create_finding_delta, + _normalized_compliance_key, _persist_compliance_requirement_rows, + _process_finding_micro_batch, _store_resources, + aggregate_findings, create_compliance_requirements, perform_prowler_scan, ) @@ -18,8 +25,40 @@ from tasks.utils import CustomEncoder from api.db_router import MainRouter from api.exceptions import ProviderConnectionError -from api.models import Finding, Provider, Resource, Scan, StateChoices, StatusChoices +from api.models import ( + Finding, + MuteRule, + Provider, + Resource, + Scan, + StateChoices, + StatusChoices, +) from prowler.lib.check.models import Severity +from prowler.lib.outputs.finding import Status + + +@contextmanager +def noop_rls_transaction(*args, **kwargs): + yield + + +class FakeFinding: + def __init__(self, **attrs): + self.metadata = attrs.pop("metadata", {}) + for key, value in attrs.items(): + setattr(self, key, value) + + self.resource_tags = getattr(self, "resource_tags", {}) + self.resource_metadata = getattr(self, "resource_metadata", {}) + self.resource_details = getattr(self, "resource_details", {}) + self.compliance = getattr(self, "compliance", {}) + self.raw = getattr(self, "raw", {}) + self.partition = getattr(self, "partition", "") + self.muted = getattr(self, "muted", False) + + def get_metadata(self): + return self.metadata @pytest.mark.django_db @@ -739,10 +778,874 @@ class TestPerformScan: # Assert that failed_findings_count was reset to 0 during the scan assert resource.failed_findings_count == 0 + def test_perform_prowler_scan_with_active_mute_rules( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test active MuteRule mutes findings with correct reason""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create active MuteRule with specific finding UIDs + mute_rule_reason = "Accepted risk - production exception" + finding_uid_1 = "finding_to_mute_1" + finding_uid_2 = "finding_to_mute_2" + + MuteRule.objects.create( + tenant_id=tenant_id, + name="Production Exception Rule", + reason=mute_rule_reason, + enabled=True, + finding_uids=[finding_uid_1, finding_uid_2], + ) + + # Mock findings: one FAIL and one PASS, both should be muted + muted_fail_finding = MagicMock() + muted_fail_finding.uid = finding_uid_1 + muted_fail_finding.status = StatusChoices.FAIL + muted_fail_finding.status_extended = "muted fail" + muted_fail_finding.severity = Severity.high + muted_fail_finding.check_id = "muted_fail_check" + muted_fail_finding.get_metadata.return_value = {"key": "value"} + muted_fail_finding.resource_uid = "resource_uid_1" + muted_fail_finding.resource_name = "resource_1" + muted_fail_finding.region = "us-east-1" + muted_fail_finding.service_name = "ec2" + muted_fail_finding.resource_type = "instance" + muted_fail_finding.resource_tags = {} + muted_fail_finding.muted = False + muted_fail_finding.raw = {} + muted_fail_finding.resource_metadata = {} + muted_fail_finding.resource_details = {} + muted_fail_finding.partition = "aws" + muted_fail_finding.compliance = {} + + muted_pass_finding = MagicMock() + muted_pass_finding.uid = finding_uid_2 + muted_pass_finding.status = StatusChoices.PASS + muted_pass_finding.status_extended = "muted pass" + muted_pass_finding.severity = Severity.medium + muted_pass_finding.check_id = "muted_pass_check" + muted_pass_finding.get_metadata.return_value = {"key": "value"} + muted_pass_finding.resource_uid = "resource_uid_2" + muted_pass_finding.resource_name = "resource_2" + muted_pass_finding.region = "us-east-1" + muted_pass_finding.service_name = "s3" + muted_pass_finding.resource_type = "bucket" + muted_pass_finding.resource_tags = {} + muted_pass_finding.muted = False + muted_pass_finding.raw = {} + muted_pass_finding.resource_metadata = {} + muted_pass_finding.resource_details = {} + muted_pass_finding.partition = "aws" + muted_pass_finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [ + (100, [muted_fail_finding, muted_pass_finding]) + ] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify findings are muted with correct reason + fail_finding_db = Finding.objects.get(uid=finding_uid_1) + pass_finding_db = Finding.objects.get(uid=finding_uid_2) + + assert fail_finding_db.muted + assert fail_finding_db.muted_reason == mute_rule_reason + assert fail_finding_db.muted_at is not None + + assert pass_finding_db.muted + assert pass_finding_db.muted_reason == mute_rule_reason + assert pass_finding_db.muted_at is not None + + # Verify failed_findings_count is 0 for muted FAIL finding + resource_1 = Resource.objects.get(uid="resource_uid_1") + assert resource_1.failed_findings_count == 0 + + def test_perform_prowler_scan_with_inactive_mute_rules( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test inactive MuteRule does not mute findings""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create inactive MuteRule + finding_uid = "finding_inactive_rule" + MuteRule.objects.create( + tenant_id=tenant_id, + name="Inactive Rule", + reason="Should not apply", + enabled=False, + finding_uids=[finding_uid], + ) + + # Mock FAIL finding + fail_finding = MagicMock() + fail_finding.uid = finding_uid + fail_finding.status = StatusChoices.FAIL + fail_finding.status_extended = "test fail" + fail_finding.severity = Severity.high + fail_finding.check_id = "fail_check" + fail_finding.get_metadata.return_value = {"key": "value"} + fail_finding.resource_uid = "resource_uid_inactive" + fail_finding.resource_name = "resource_inactive" + fail_finding.region = "us-east-1" + fail_finding.service_name = "ec2" + fail_finding.resource_type = "instance" + fail_finding.resource_tags = {} + fail_finding.muted = False + fail_finding.raw = {} + fail_finding.resource_metadata = {} + fail_finding.resource_details = {} + fail_finding.partition = "aws" + fail_finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, [fail_finding])] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify finding is NOT muted + finding_db = Finding.objects.get(uid=finding_uid) + assert not finding_db.muted + assert finding_db.muted_reason is None + assert finding_db.muted_at is None + + # Verify failed_findings_count increments for FAIL finding + resource = Resource.objects.get(uid="resource_uid_inactive") + assert resource.failed_findings_count == 1 + + def test_perform_prowler_scan_mutelist_overrides_mute_rules( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test mutelist processor takes precedence over MuteRule""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create active MuteRule + finding_uid = "finding_both_rules" + MuteRule.objects.create( + tenant_id=tenant_id, + name="Manual Mute Rule", + reason="Muted by manual rule", + enabled=True, + finding_uids=[finding_uid], + ) + + # Mock finding with mutelist processor muted=True + muted_finding = MagicMock() + muted_finding.uid = finding_uid + muted_finding.status = StatusChoices.FAIL + muted_finding.status_extended = "test" + muted_finding.severity = Severity.high + muted_finding.check_id = "test_check" + muted_finding.get_metadata.return_value = {"key": "value"} + muted_finding.resource_uid = "resource_both" + muted_finding.resource_name = "resource_both" + muted_finding.region = "us-east-1" + muted_finding.service_name = "ec2" + muted_finding.resource_type = "instance" + muted_finding.resource_tags = {} + muted_finding.muted = True + muted_finding.raw = {} + muted_finding.resource_metadata = {} + muted_finding.resource_details = {} + muted_finding.partition = "aws" + muted_finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, [muted_finding])] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify mutelist reason takes precedence + finding_db = Finding.objects.get(uid=finding_uid) + assert finding_db.muted + assert finding_db.muted_reason == "Muted by mutelist" + assert finding_db.muted_at is not None + + # Verify failed_findings_count is 0 + resource = Resource.objects.get(uid="resource_both") + assert resource.failed_findings_count == 0 + + def test_perform_prowler_scan_mute_rules_multiple_findings( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test MuteRule with multiple finding UIDs mutes all findings""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create MuteRule with multiple finding UIDs + mute_rule_reason = "Bulk exception for dev environment" + finding_uids = [ + "bulk_finding_1", + "bulk_finding_2", + "bulk_finding_3", + "bulk_finding_4", + ] + MuteRule.objects.create( + tenant_id=tenant_id, + name="Bulk Mute Rule", + reason=mute_rule_reason, + enabled=True, + finding_uids=finding_uids, + ) + + # Mock multiple findings with mixed statuses + findings = [] + for i, uid in enumerate(finding_uids): + finding = MagicMock() + finding.uid = uid + finding.status = ( + StatusChoices.FAIL if i % 2 == 0 else StatusChoices.PASS + ) + finding.status_extended = f"test {i}" + finding.severity = Severity.medium + finding.check_id = f"check_{i}" + finding.get_metadata.return_value = {"key": f"value_{i}"} + finding.resource_uid = f"resource_bulk_{i}" + finding.resource_name = f"resource_{i}" + finding.region = "us-west-2" + finding.service_name = "lambda" + finding.resource_type = "function" + finding.resource_tags = {} + finding.muted = False + finding.raw = {} + finding.resource_metadata = {} + finding.resource_details = {} + finding.partition = "aws" + finding.compliance = {} + findings.append(finding) + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, findings)] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-west-2"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify all findings are muted with same reason + for uid in finding_uids: + finding_db = Finding.objects.get(uid=uid) + assert finding_db.muted + assert finding_db.muted_reason == mute_rule_reason + assert finding_db.muted_at is not None + + # Verify all resources have failed_findings_count = 0 + for i in range(len(finding_uids)): + resource = Resource.objects.get(uid=f"resource_bulk_{i}") + assert resource.failed_findings_count == 0 + + def test_perform_prowler_scan_mute_rules_error_handling( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test scan continues when MuteRule loading fails""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + patch("api.models.MuteRule.objects.filter") as mock_mute_rule_filter, + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Mock MuteRule.objects.filter to raise exception + mock_mute_rule_filter.side_effect = Exception("Database error") + + # Mock finding + finding = MagicMock() + finding.uid = "finding_error_handling" + finding.status = StatusChoices.FAIL + finding.status_extended = "test" + finding.severity = Severity.high + finding.check_id = "test_check" + finding.get_metadata.return_value = {"key": "value"} + finding.resource_uid = "resource_error" + finding.resource_name = "resource_error" + finding.region = "us-east-1" + finding.service_name = "ec2" + finding.resource_type = "instance" + finding.resource_tags = {} + finding.muted = False + finding.raw = {} + finding.resource_metadata = {} + finding.resource_details = {} + finding.partition = "aws" + finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, [finding])] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Call the function under test - should not raise + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + + # Verify scan completed successfully + scan.refresh_from_db() + assert scan.state == StateChoices.COMPLETED + + # Verify finding is not muted (mute_rules_cache was empty dict) + finding_db = Finding.objects.get(uid="finding_error_handling") + assert not finding_db.muted + assert finding_db.muted_reason is None + + # Verify failed_findings_count increments + resource = Resource.objects.get(uid="resource_error") + assert resource.failed_findings_count == 1 + + def test_perform_prowler_scan_muted_at_timestamp( + self, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + """Test muted_at timestamp is set correctly for muted findings""" + with ( + patch("api.db_utils.rls_transaction"), + patch( + "tasks.jobs.scan.initialize_prowler_provider" + ) as mock_initialize_prowler_provider, + patch("tasks.jobs.scan.ProwlerScan") as mock_prowler_scan_class, + patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE", + new_callable=dict, + ), + patch("api.compliance.PROWLER_CHECKS", new_callable=dict), + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + provider.provider = Provider.ProviderChoices.AWS + provider.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + + # Create active MuteRule + finding_uid = "finding_timestamp_test" + MuteRule.objects.create( + tenant_id=tenant_id, + name="Timestamp Test Rule", + reason="Testing timestamp", + enabled=True, + finding_uids=[finding_uid], + ) + + # Mock finding + finding = MagicMock() + finding.uid = finding_uid + finding.status = StatusChoices.FAIL + finding.status_extended = "test" + finding.severity = Severity.high + finding.check_id = "test_check" + finding.get_metadata.return_value = {"key": "value"} + finding.resource_uid = "resource_timestamp" + finding.resource_name = "resource_timestamp" + finding.region = "us-east-1" + finding.service_name = "ec2" + finding.resource_type = "instance" + finding.resource_tags = {} + finding.muted = False + finding.raw = {} + finding.resource_metadata = {} + finding.resource_details = {} + finding.partition = "aws" + finding.compliance = {} + + # Mock the ProwlerScan instance + mock_prowler_scan_instance = MagicMock() + mock_prowler_scan_instance.scan.return_value = [(100, [finding])] + mock_prowler_scan_class.return_value = mock_prowler_scan_instance + + # Mock prowler_provider + mock_prowler_provider_instance = MagicMock() + mock_prowler_provider_instance.get_regions.return_value = ["us-east-1"] + mock_initialize_prowler_provider.return_value = ( + mock_prowler_provider_instance + ) + + # Capture time before and after scan + before_scan = datetime.now(timezone.utc) + perform_prowler_scan(tenant_id, scan_id, provider_id, []) + after_scan = datetime.now(timezone.utc) + + # Verify muted_at is within the scan time window + finding_db = Finding.objects.get(uid=finding_uid) + assert finding_db.muted + assert finding_db.muted_at is not None + assert before_scan <= finding_db.muted_at <= after_scan + # TODO Add tests for aggregations +@pytest.mark.django_db +class TestProcessFindingMicroBatch: + def test_process_finding_micro_batch_creates_records_and_updates_caches( + self, tenants_fixture, scans_fixture + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = scan.provider + + finding = FakeFinding( + uid="finding-new", + status=StatusChoices.PASS, + status_extended="all good", + severity=Severity.low, + check_id="s3_public_buckets", + resource_uid="arn:aws:s3:::bucket-1", + resource_name="bucket-1", + region="us-east-1", + service_name="s3", + resource_type="bucket", + resource_tags={"env": "dev", "team": "security"}, + resource_metadata={"owner": "secops"}, + resource_details={"arn": "arn:aws:s3:::bucket-1"}, + partition="aws", + raw={"status": "PASS"}, + compliance={"cis": {"1.1": "PASS"}}, + metadata={"source": "prowler"}, + muted=False, + ) + + resource_cache = {} + tag_cache = {} + last_status_cache = {} + resource_failed_findings_cache = {} + unique_resources: set[tuple[str, str]] = set() + scan_resource_cache: set[tuple[str, str, str, str]] = set() + mute_rules_cache = {} + + with ( + patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction), + patch("api.db_utils.rls_transaction", new=noop_rls_transaction), + ): + _process_finding_micro_batch( + str(tenant.id), + [finding], + scan, + provider, + resource_cache, + tag_cache, + last_status_cache, + resource_failed_findings_cache, + unique_resources, + scan_resource_cache, + mute_rules_cache, + ) + + created_finding = Finding.objects.get(uid=finding.uid) + resource = Resource.objects.get(uid=finding.resource_uid) + + assert created_finding.scan_id == scan.id + assert created_finding.status == StatusChoices.PASS + assert created_finding.delta == Finding.DeltaChoices.NEW + assert created_finding.muted is False + assert created_finding.check_metadata == finding.metadata + assert created_finding.resource_regions == [finding.region] + assert created_finding.resource_services == [finding.service_name] + assert created_finding.resource_types == [finding.resource_type] + assert created_finding.first_seen_at is not None + assert created_finding.compliance == finding.compliance + + assert resource.provider_id == provider.id + assert resource.region == finding.region + assert resource.service == finding.service_name + assert resource.type == finding.resource_type + assert resource.name == finding.resource_name + assert resource.metadata == json.dumps( + finding.resource_metadata, cls=CustomEncoder + ) + assert resource.details == f"{finding.resource_details}" + assert resource.partition == finding.partition + assert set(resource.tags.values_list("key", "value")) == set( + finding.resource_tags.items() + ) + assert resource.findings.filter(uid=finding.uid).exists() + + assert resource_cache[finding.resource_uid].id == resource.id + assert resource_failed_findings_cache[finding.resource_uid] == 0 + assert (resource.uid, resource.region) in unique_resources + assert ( + str(resource.id), + resource.service, + resource.region, + resource.type, + ) in scan_resource_cache + assert set(tag_cache.keys()) == set(finding.resource_tags.items()) + + def test_process_finding_micro_batch_manual_mute_and_dirty_resources( + self, tenants_fixture, scans_fixture + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = scan.provider + + existing_resource = Resource.objects.create( + tenant_id=tenant.id, + provider=provider, + uid="arn:aws:ec2:us-east-1:123456789012:instance/i-001", + name="i-001", + region="us-east-1", + service="ec2", + type="instance", + metadata=json.dumps({"old": "meta"}), + details="old-details", + partition="aws-old", + ) + + previous_first_seen = datetime(2024, 1, 1, tzinfo=timezone.utc) + + finding = FakeFinding( + uid="finding-muted", + status=StatusChoices.FAIL, + status_extended="failing", + severity=Severity.high, + check_id="ec2_public_instance", + resource_uid=existing_resource.uid, + resource_name=existing_resource.name, + region="eu-west-1", + service_name="eks", + resource_type="cluster", + resource_tags={"team": "devsec"}, + resource_metadata={"owner": "platform"}, + resource_details={"id": existing_resource.name}, + partition="aws", + raw={"status": "FAIL"}, + compliance={"cis": {"1.2": "FAIL"}}, + metadata={"source": "prowler"}, + muted=False, + ) + + resource_cache = {existing_resource.uid: existing_resource} + tag_cache = {} + last_status_cache = {finding.uid: (StatusChoices.PASS, previous_first_seen)} + resource_failed_findings_cache = {existing_resource.uid: 2} + unique_resources: set[tuple[str, str]] = set() + scan_resource_cache: set[tuple[str, str, str, str]] = set() + mute_rules_cache = {finding.uid: "Muted via rule"} + + with ( + patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction), + patch("api.db_utils.rls_transaction", new=noop_rls_transaction), + ): + _process_finding_micro_batch( + str(tenant.id), + [finding], + scan, + provider, + resource_cache, + tag_cache, + last_status_cache, + resource_failed_findings_cache, + unique_resources, + scan_resource_cache, + mute_rules_cache, + ) + + existing_resource.refresh_from_db() + created_finding = Finding.objects.get(uid=finding.uid) + + assert created_finding.delta == Finding.DeltaChoices.CHANGED + assert created_finding.status == StatusChoices.FAIL + assert created_finding.muted is True + assert created_finding.muted_reason == "Muted via rule" + assert created_finding.muted_at is not None + assert created_finding.first_seen_at == previous_first_seen + assert created_finding.compliance == finding.compliance + assert created_finding.resource_regions == [finding.region] + assert created_finding.resource_services == [finding.service_name] + assert created_finding.resource_types == [finding.resource_type] + assert created_finding.scan_id == scan.id + + assert resource_failed_findings_cache[finding.resource_uid] == 2 + assert (finding.resource_uid, finding.region) in unique_resources + assert ( + str(existing_resource.id), + finding.service_name, + finding.region, + finding.resource_type, + ) in scan_resource_cache + + assert existing_resource.region == finding.region + assert existing_resource.service == finding.service_name + assert existing_resource.type == finding.resource_type + assert existing_resource.metadata == json.dumps( + finding.resource_metadata, cls=CustomEncoder + ) + assert existing_resource.details == f"{finding.resource_details}" + assert existing_resource.partition == finding.partition + assert set(existing_resource.tags.values_list("key", "value")) == { + ("team", "devsec") + } + assert existing_resource.findings.filter(uid=finding.uid).exists() + + assert resource_cache[finding.resource_uid].region == finding.region + assert resource_cache[finding.resource_uid].service == finding.service_name + assert tag_cache.keys() == {("team", "devsec")} + + def test_process_finding_micro_batch_skips_long_uid( + self, tenants_fixture, scans_fixture + ): + """Test that findings with UID > 300 chars are skipped (temporary workaround).""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = scan.provider + + # Create a finding with UID > 300 chars + long_uid = ( + "prowler-aws-ec2_instance_public_ip-123456789012-us-east-1-" + "x" * 250 + ) + assert len(long_uid) > 300 + + finding_with_long_uid = FakeFinding( + uid=long_uid, + status=StatusChoices.FAIL, + status_extended="public instance", + severity=Severity.high, + check_id="ec2_instance_public_ip", + resource_uid="arn:aws:ec2:us-east-1:123456789012:instance/i-long", + resource_name="i-long-uid-instance", + region="us-east-1", + service_name="ec2", + resource_type="instance", + resource_tags={}, + resource_metadata={}, + resource_details={}, + partition="aws", + raw={}, + compliance={}, + metadata={}, + muted=False, + ) + + # Create a normal finding that should be processed + normal_finding = FakeFinding( + uid="finding-normal", + status=StatusChoices.PASS, + status_extended="all good", + severity=Severity.low, + check_id="s3_bucket_encryption", + resource_uid="arn:aws:s3:::bucket-normal", + resource_name="bucket-normal", + region="us-east-1", + service_name="s3", + resource_type="bucket", + resource_tags={}, + resource_metadata={}, + resource_details={}, + partition="aws", + raw={}, + compliance={}, + metadata={}, + muted=False, + ) + + resource_cache = {} + tag_cache = {} + last_status_cache = {} + resource_failed_findings_cache = {} + unique_resources: set[tuple[str, str]] = set() + scan_resource_cache: set[tuple[str, str, str, str]] = set() + mute_rules_cache = {} + + with ( + patch("tasks.jobs.scan.rls_transaction", new=noop_rls_transaction), + patch("api.db_utils.rls_transaction", new=noop_rls_transaction), + patch("tasks.jobs.scan.logger") as mock_logger, + ): + _process_finding_micro_batch( + str(tenant.id), + [finding_with_long_uid, normal_finding], + scan, + provider, + resource_cache, + tag_cache, + last_status_cache, + resource_failed_findings_cache, + unique_resources, + scan_resource_cache, + mute_rules_cache, + ) + + # Verify the long UID finding was NOT created + assert not Finding.objects.filter(uid=long_uid).exists() + + # Verify the normal finding WAS created + assert Finding.objects.filter(uid=normal_finding.uid).exists() + + # Verify logging was called for skipped finding + assert mock_logger.warning.called + warning_calls = [str(call) for call in mock_logger.warning.call_args_list] + assert any( + "Skipping finding with UID exceeding 300 characters" in str(call) + for call in warning_calls + ) + assert any( + f"Scan {scan.id}: Skipped 1 finding(s)" in str(call) + for call in warning_calls + ) + + @pytest.mark.django_db class TestCreateComplianceRequirements: def test_create_compliance_requirements_success( @@ -753,12 +1656,9 @@ class TestCreateComplianceRequirements: findings_fixture, resources_fixture, ): - with ( - patch( - "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" - ) as mock_compliance_template, - patch("tasks.jobs.scan.generate_scan_compliance"), - ): + with patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" + ) as mock_compliance_template: tenant_id = str(tenants_fixture[0].id) scan_id = str(scans_fixture[0].id) @@ -769,6 +1669,7 @@ class TestCreateComplianceRequirements: "requirements": { "1.1": { "description": "Ensure root access key does not exist", + "checks": {"test_check_id": None}, "checks_status": { "pass": 0, "fail": 0, @@ -779,6 +1680,7 @@ class TestCreateComplianceRequirements: }, "1.2": { "description": "Ensure MFA is enabled for root account", + "checks": {"test_check_id": None}, "checks_status": { "pass": 0, "fail": 1, @@ -804,12 +1706,9 @@ class TestCreateComplianceRequirements: providers_fixture, findings_fixture, ): - with ( - patch( - "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" - ) as mock_compliance_template, - patch("tasks.jobs.scan.generate_scan_compliance"), - ): + with patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" + ) as mock_compliance_template: tenant_id = str(tenants_fixture[0].id) scan_id = str(scans_fixture[0].id) @@ -820,6 +1719,7 @@ class TestCreateComplianceRequirements: "requirements": { "req_1": { "description": "Test Requirement 1", + "checks": {"test_check_id": None}, "checks_status": { "pass": 2, "fail": 1, @@ -843,12 +1743,9 @@ class TestCreateComplianceRequirements: providers_fixture, findings_fixture, ): - with ( - patch( - "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" - ) as mock_compliance_template, - patch("tasks.jobs.scan.generate_scan_compliance"), - ): + with patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" + ) as mock_compliance_template: tenant = tenants_fixture[0] scan = scans_fixture[0] provider = providers_fixture[0] @@ -868,6 +1765,7 @@ class TestCreateComplianceRequirements: "requirements": { "1.1": { "description": "Test requirement", + "checks": {"test_check_id": None}, "checks_status": { "pass": 0, "fail": 0, @@ -891,12 +1789,9 @@ class TestCreateComplianceRequirements: providers_fixture, findings_fixture, ): - with ( - patch( - "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" - ) as mock_compliance_template, - patch("tasks.jobs.scan.generate_scan_compliance"), - ): + with patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" + ) as mock_compliance_template: tenant_id = str(tenants_fixture[0].id) scan_id = str(scans_fixture[0].id) @@ -925,18 +1820,41 @@ class TestCreateComplianceRequirements: create_compliance_requirements(tenant_id, scan_id) def test_create_compliance_requirements_check_status_priority( - self, tenants_fixture, scans_fixture, providers_fixture, findings_fixture + self, tenants_fixture, scans_fixture, findings_fixture ): with ( patch( "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" ) as mock_compliance_template, patch( - "tasks.jobs.scan.generate_scan_compliance" - ) as mock_generate_compliance, + "tasks.jobs.scan._persist_compliance_requirement_rows" + ) as mock_persist, + patch("tasks.jobs.scan._create_compliance_summaries"), ): tenant_id = str(tenants_fixture[0].id) - scan_id = str(scans_fixture[0].id) + scan = scans_fixture[0] + scan_id = str(scan.id) + existing_finding = findings_fixture[0] + + pass_finding = Finding.objects.create( + tenant_id=scan.tenant_id, + uid="pass-finding", + scan=scan, + delta=None, + status=Status.PASS, + status_extended="pass status", + impact=Severity.low, + impact_extended="", + severity=Severity.low, + raw_result={"status": Status.PASS}, + tags={}, + check_id=existing_finding.check_id, + check_metadata={"CheckId": existing_finding.check_id}, + first_seen_at=datetime.now(timezone.utc), + muted=False, + ) + resource = existing_finding.resources.first() + pass_finding.add_resources([resource]) mock_compliance_template.__getitem__.return_value = { "cis_1.4_aws": { @@ -945,6 +1863,7 @@ class TestCreateComplianceRequirements: "requirements": { "1.1": { "description": "Test requirement", + "checks": {existing_finding.check_id: None}, "checks_status": { "pass": 0, "fail": 0, @@ -959,7 +1878,12 @@ class TestCreateComplianceRequirements: create_compliance_requirements(tenant_id, scan_id) - assert mock_generate_compliance.call_count == 1 + mock_persist.assert_called_once() + persisted_rows = mock_persist.call_args[0][1] + requirement_row = next( + row for row in persisted_rows if row["requirement_id"] == "1.1" + ) + assert requirement_row["requirement_status"] == "FAIL" def test_create_compliance_requirements_multiple_regions( self, @@ -968,12 +1892,9 @@ class TestCreateComplianceRequirements: providers_fixture, findings_fixture, ): - with ( - patch( - "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" - ) as mock_compliance_template, - patch("tasks.jobs.scan.generate_scan_compliance"), - ): + with patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" + ) as mock_compliance_template: tenant_id = str(tenants_fixture[0].id) scan_id = str(scans_fixture[0].id) @@ -984,6 +1905,7 @@ class TestCreateComplianceRequirements: "requirements": { "req_1": { "description": "Test Requirement 1", + "checks": {"test_check_id": None}, "checks_status": { "pass": 2, "fail": 0, @@ -1008,12 +1930,9 @@ class TestCreateComplianceRequirements: providers_fixture, findings_fixture, ): - with ( - patch( - "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" - ) as mock_compliance_template, - patch("tasks.jobs.scan.generate_scan_compliance"), - ): + with patch( + "tasks.jobs.scan.PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE" + ) as mock_compliance_template: tenant_id = str(tenants_fixture[0].id) scan_id = str(scans_fixture[0].id) @@ -1024,6 +1943,7 @@ class TestCreateComplianceRequirements: "requirements": { "req_1": { "description": "Test Requirement 1", + "checks": {"test_check_id": None}, "checks_status": { "pass": 2, "fail": 0, @@ -1034,6 +1954,7 @@ class TestCreateComplianceRequirements: }, "req_2": { "description": "Test Requirement 2", + "checks": {"test_check_id": None}, "checks_status": { "pass": 1, "fail": 1, @@ -1820,3 +2741,733 @@ class TestComplianceRequirementCopy: assert obj.failed_checks == 5 assert obj.total_checks == 15 assert obj.scan_id == scan_id + + +@pytest.mark.django_db +class TestCreateComplianceSummaries: + """Test _create_compliance_summaries function.""" + + @patch("tasks.jobs.scan.ComplianceOverviewSummary.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + def test_create_compliance_summaries_mixed_statuses( + self, mock_rls_transaction, mock_bulk_create + ): + """Test creating summaries with mixed requirement statuses (PASS/FAIL/MANUAL).""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + + # Simulate pre-computed requirement statuses + requirement_statuses = { + ("compliance1", "req1"): { + "fail_count": 0, + "pass_count": 5, + "total_count": 5, + }, + ("compliance1", "req2"): { + "fail_count": 2, + "pass_count": 3, + "total_count": 5, + }, + ("compliance1", "req3"): { + "fail_count": 0, + "pass_count": 3, + "total_count": 5, + }, + ("compliance2", "req1"): { + "fail_count": 1, + "pass_count": 0, + "total_count": 5, + }, + } + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + _create_compliance_summaries(tenant_id, scan_id, requirement_statuses) + + mock_rls_transaction.assert_called_once_with(tenant_id) + mock_bulk_create.assert_called_once() + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + assert len(objects) == 2 + assert kwargs["batch_size"] == 500 + + # Find compliance1 and compliance2 summaries + comp1 = next(obj for obj in objects if obj.compliance_id == "compliance1") + comp2 = next(obj for obj in objects if obj.compliance_id == "compliance2") + + # compliance1: req1=PASS, req2=FAIL (has fail_count), req3=MANUAL (pass < total) + assert comp1.total_requirements == 3 + assert comp1.requirements_passed == 1 + assert comp1.requirements_failed == 1 + assert comp1.requirements_manual == 1 + + # compliance2: req1=FAIL (has fail_count) + assert comp2.total_requirements == 1 + assert comp2.requirements_passed == 0 + assert comp2.requirements_failed == 1 + assert comp2.requirements_manual == 0 + + @patch("tasks.jobs.scan.ComplianceOverviewSummary.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + def test_create_compliance_summaries_empty_input( + self, mock_rls_transaction, mock_bulk_create + ): + """Test with empty requirement_statuses dict - should not create any summaries.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + requirement_statuses = {} + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + _create_compliance_summaries(tenant_id, scan_id, requirement_statuses) + + # Should not call bulk_create with empty list + mock_bulk_create.assert_not_called() + + @patch("tasks.jobs.scan.ComplianceOverviewSummary.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + def test_create_compliance_summaries_all_pass( + self, mock_rls_transaction, mock_bulk_create + ): + """Test creating summaries when all requirements pass.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + + requirement_statuses = { + ("comp1", "req1"): {"fail_count": 0, "pass_count": 10, "total_count": 10}, + ("comp1", "req2"): {"fail_count": 0, "pass_count": 5, "total_count": 5}, + } + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + _create_compliance_summaries(tenant_id, scan_id, requirement_statuses) + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + assert len(objects) == 1 + + obj = objects[0] + assert obj.compliance_id == "comp1" + assert obj.total_requirements == 2 + assert obj.requirements_passed == 2 + assert obj.requirements_failed == 0 + assert obj.requirements_manual == 0 + + @patch("tasks.jobs.scan.ComplianceOverviewSummary.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + def test_create_compliance_summaries_all_fail( + self, mock_rls_transaction, mock_bulk_create + ): + """Test creating summaries when all requirements fail.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + + requirement_statuses = { + ("comp1", "req1"): {"fail_count": 3, "pass_count": 7, "total_count": 10}, + ("comp1", "req2"): {"fail_count": 1, "pass_count": 4, "total_count": 5}, + } + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + _create_compliance_summaries(tenant_id, scan_id, requirement_statuses) + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + assert len(objects) == 1 + + obj = objects[0] + assert obj.compliance_id == "comp1" + assert obj.total_requirements == 2 + assert obj.requirements_passed == 0 + assert obj.requirements_failed == 2 + assert obj.requirements_manual == 0 + + @patch("tasks.jobs.scan.ComplianceOverviewSummary.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + def test_create_compliance_summaries_correct_aggregation( + self, mock_rls_transaction, mock_bulk_create + ): + """Test that requirements are correctly aggregated to compliance level.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + + requirement_statuses = { + ("compliance_a", "req1"): { + "fail_count": 0, + "pass_count": 10, + "total_count": 10, + }, + ("compliance_a", "req2"): { + "fail_count": 1, + "pass_count": 9, + "total_count": 10, + }, + ("compliance_a", "req3"): { + "fail_count": 0, + "pass_count": 5, + "total_count": 10, + }, + ("compliance_b", "req1"): { + "fail_count": 0, + "pass_count": 8, + "total_count": 8, + }, + } + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + _create_compliance_summaries(tenant_id, scan_id, requirement_statuses) + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + assert len(objects) == 2 + + comp_a = next(obj for obj in objects if obj.compliance_id == "compliance_a") + comp_b = next(obj for obj in objects if obj.compliance_id == "compliance_b") + + # compliance_a: req1=PASS, req2=FAIL, req3=MANUAL + assert comp_a.total_requirements == 3 + assert comp_a.requirements_passed == 1 + assert comp_a.requirements_failed == 1 + assert comp_a.requirements_manual == 1 + + # compliance_b: req1=PASS + assert comp_b.total_requirements == 1 + assert comp_b.requirements_passed == 1 + assert comp_b.requirements_failed == 0 + assert comp_b.requirements_manual == 0 + + +@pytest.mark.django_db +class TestNormalizedComplianceKey: + """Test _normalized_compliance_key function.""" + + def test_normalized_compliance_key_normal_strings(self): + """Test normalization with normal framework and version strings.""" + result = _normalized_compliance_key("AWS-Foundational-Security", "2.0") + assert result == "awsfoundationalsecurity20" + + def test_normalized_compliance_key_with_underscores(self): + """Test normalization removes underscores.""" + result = _normalized_compliance_key("CIS_AWS_Foundations", "1_5_0") + assert result == "cisawsfoundations150" + + def test_normalized_compliance_key_none_framework(self): + """Test normalization with None framework.""" + result = _normalized_compliance_key(None, "1.0") + assert result == "10" + + def test_normalized_compliance_key_none_version(self): + """Test normalization with None version.""" + result = _normalized_compliance_key("AWS-Security", None) + assert result == "awssecurity" + + def test_normalized_compliance_key_both_none(self): + """Test normalization with both framework and version as None.""" + result = _normalized_compliance_key(None, None) + assert result == "" + + def test_normalized_compliance_key_empty_strings(self): + """Test normalization with empty strings.""" + result = _normalized_compliance_key("", "") + assert result == "" + + def test_normalized_compliance_key_mixed_case(self): + """Test normalization lowercases strings.""" + result = _normalized_compliance_key("AWS-FOUNDATIONAL", "V2.0") + assert result == "awsfoundationalv20" + + def test_normalized_compliance_key_complex_pattern(self): + """Test normalization with complex patterns.""" + result = _normalized_compliance_key("PCI-DSS_v3-2-1", "2023-Update") + assert result == "pcidssv3212023update" + + +@pytest.mark.django_db +class TestAggregateFindings: + """Test aggregate_findings function.""" + + @patch("tasks.jobs.scan.ScanSummary.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_creates_scan_summaries( + self, + mock_rls_transaction, + mock_bulk_create, + tenants_fixture, + scans_fixture, + findings_fixture, + ): + """Test that aggregate_findings creates ScanSummary records.""" + tenant = tenants_fixture[0] + scan = scans_fixture[0] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + + aggregate_findings(str(tenant.id), str(scan.id)) + + mock_rls_transaction.assert_called() + mock_bulk_create.assert_called_once() + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + assert kwargs["batch_size"] == 3000 + # Should have created at least one summary + assert len(objects) > 0 + + @patch("tasks.jobs.scan.Finding.objects.filter") + @patch("tasks.jobs.scan.ScanSummary.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_excludes_muted_from_counts( + self, mock_rls_transaction, mock_bulk_create, mock_findings_filter + ): + """Test that muted findings are excluded from fail/pass counts but counted separately.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + + # Mock findings queryset + mock_queryset = MagicMock() + mock_queryset.values.return_value = mock_queryset + mock_queryset.annotate.return_value = [ + { + "check_id": "check1", + "resources__service": "s3", + "severity": "high", + "resources__region": "us-east-1", + "fail": 5, + "_pass": 10, + "muted_count": 3, + "total": 18, + "new": 2, + "changed": 1, + "unchanged": 12, + "fail_new": 1, + "fail_changed": 0, + "pass_new": 1, + "pass_changed": 0, + "muted_new": 0, + "muted_changed": 1, + } + ] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + aggregate_findings(tenant_id, scan_id) + + mock_bulk_create.assert_called_once() + args, kwargs = mock_bulk_create.call_args + objects = args[0] + + summary = list(objects)[0] + assert summary.fail == 5 + assert summary._pass == 10 + assert summary.muted == 3 + assert summary.total == 18 + + @patch("tasks.jobs.scan.Finding.objects.filter") + @patch("tasks.jobs.scan.ScanSummary.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_computes_deltas_correctly( + self, mock_rls_transaction, mock_bulk_create, mock_findings_filter + ): + """Test that delta counts (new, changed, unchanged) are computed correctly.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + + mock_queryset = MagicMock() + mock_queryset.values.return_value = mock_queryset + mock_queryset.annotate.return_value = [ + { + "check_id": "check1", + "resources__service": "ec2", + "severity": "critical", + "resources__region": "us-west-2", + "fail": 8, + "_pass": 12, + "muted_count": 2, + "total": 22, + "new": 5, + "changed": 3, + "unchanged": 12, + "fail_new": 3, + "fail_changed": 2, + "pass_new": 2, + "pass_changed": 1, + "muted_new": 1, + "muted_changed": 0, + } + ] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + aggregate_findings(tenant_id, scan_id) + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + + summary = list(objects)[0] + assert summary.new == 5 + assert summary.changed == 3 + assert summary.unchanged == 12 + assert summary.fail_new == 3 + assert summary.fail_changed == 2 + assert summary.pass_new == 2 + assert summary.pass_changed == 1 + assert summary.muted_new == 1 + assert summary.muted_changed == 0 + + @patch("tasks.jobs.scan.Finding.objects.filter") + @patch("tasks.jobs.scan.ScanSummary.objects.bulk_create") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_groups_by_dimensions( + self, mock_rls_transaction, mock_bulk_create, mock_findings_filter + ): + """Test that findings are grouped by check_id, service, severity, and region.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + + mock_queryset = MagicMock() + mock_queryset.values.return_value = mock_queryset + mock_queryset.annotate.return_value = [ + { + "check_id": "check1", + "resources__service": "s3", + "severity": "high", + "resources__region": "us-east-1", + "fail": 5, + "_pass": 10, + "muted_count": 0, + "total": 15, + "new": 2, + "changed": 1, + "unchanged": 12, + "fail_new": 1, + "fail_changed": 0, + "pass_new": 1, + "pass_changed": 1, + "muted_new": 0, + "muted_changed": 0, + }, + { + "check_id": "check1", + "resources__service": "s3", + "severity": "high", + "resources__region": "us-west-2", + "fail": 3, + "_pass": 7, + "muted_count": 1, + "total": 11, + "new": 1, + "changed": 0, + "unchanged": 9, + "fail_new": 1, + "fail_changed": 0, + "pass_new": 0, + "pass_changed": 0, + "muted_new": 0, + "muted_changed": 1, + }, + ] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + aggregate_findings(tenant_id, scan_id) + + args, kwargs = mock_bulk_create.call_args + objects = args[0] + + # Should create 2 summaries (different regions) + assert len(list(objects)) == 2 + + summaries = list(objects) + assert all(s.check_id == "check1" for s in summaries) + assert all(s.service == "s3" for s in summaries) + assert all(s.severity == "high" for s in summaries) + + regions = {s.region for s in summaries} + assert regions == {"us-east-1", "us-west-2"} + + +@pytest.mark.django_db +class TestAggregateFindingsByRegion: + """Test _aggregate_findings_by_region function.""" + + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_by_region_returns_correct_structure( + self, mock_rls_transaction, mock_findings_filter + ): + """Test function returns correct data structure.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" + + # Mock findings with resources + mock_finding1 = MagicMock() + mock_finding1.check_id = "check1" + mock_finding1.status = "FAIL" + mock_finding1.compliance = {modeled_threatscore_compliance_id: ["req1", "req2"]} + + mock_resource1 = MagicMock() + mock_resource1.region = "us-east-1" + mock_finding1.small_resources = [mock_resource1] + + mock_queryset = MagicMock() + mock_queryset.only.return_value = mock_queryset + mock_queryset.prefetch_related.return_value = [mock_finding1] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + check_status_by_region, findings_count_by_compliance = ( + _aggregate_findings_by_region( + tenant_id, scan_id, modeled_threatscore_compliance_id + ) + ) + + # Verify structure of check_status_by_region + assert isinstance(check_status_by_region, dict) + assert "us-east-1" in check_status_by_region + assert "check1" in check_status_by_region["us-east-1"] + assert check_status_by_region["us-east-1"]["check1"] == "FAIL" + + # Verify structure of findings_count_by_compliance + assert isinstance(findings_count_by_compliance, dict) + + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_by_region_fail_status_priority( + self, mock_rls_transaction, mock_findings_filter + ): + """Test that FAIL status takes priority over other statuses.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" + + # First finding with PASS status + mock_finding1 = MagicMock() + mock_finding1.check_id = "check1" + mock_finding1.status = "PASS" + mock_finding1.compliance = {} + mock_resource1 = MagicMock() + mock_resource1.region = "us-east-1" + mock_finding1.small_resources = [mock_resource1] + + # Second finding with FAIL status for same check/region + mock_finding2 = MagicMock() + mock_finding2.check_id = "check1" + mock_finding2.status = "FAIL" + mock_finding2.compliance = {} + mock_resource2 = MagicMock() + mock_resource2.region = "us-east-1" + mock_finding2.small_resources = [mock_resource2] + + mock_queryset = MagicMock() + mock_queryset.only.return_value = mock_queryset + mock_queryset.prefetch_related.return_value = [mock_finding1, mock_finding2] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + check_status_by_region, _ = _aggregate_findings_by_region( + tenant_id, scan_id, modeled_threatscore_compliance_id + ) + + # FAIL should override PASS + assert check_status_by_region["us-east-1"]["check1"] == "FAIL" + + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_by_region_filters_muted( + self, mock_rls_transaction, mock_findings_filter + ): + """Test that muted findings are filtered out (muted=False in query).""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" + + mock_queryset = MagicMock() + mock_queryset.only.return_value = mock_queryset + mock_queryset.prefetch_related.return_value = [] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + _aggregate_findings_by_region( + tenant_id, scan_id, modeled_threatscore_compliance_id + ) + + # Verify filter was called with muted=False + mock_findings_filter.assert_called_once_with( + tenant_id=tenant_id, scan_id=scan_id, muted=False + ) + + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_by_region_processes_compliance_counts( + self, mock_rls_transaction, mock_findings_filter + ): + """Test that ThreatScore compliance counts are processed correctly.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" + + # Finding with PASS status + mock_finding1 = MagicMock() + mock_finding1.check_id = "check1" + mock_finding1.status = "PASS" + mock_finding1.compliance = {modeled_threatscore_compliance_id: ["req1"]} + mock_resource1 = MagicMock() + mock_resource1.region = "us-east-1" + mock_finding1.small_resources = [mock_resource1] + + # Finding with FAIL status + mock_finding2 = MagicMock() + mock_finding2.check_id = "check2" + mock_finding2.status = "FAIL" + mock_finding2.compliance = {modeled_threatscore_compliance_id: ["req1"]} + mock_resource2 = MagicMock() + mock_resource2.region = "us-east-1" + mock_finding2.small_resources = [mock_resource2] + + mock_queryset = MagicMock() + mock_queryset.only.return_value = mock_queryset + mock_queryset.prefetch_related.return_value = [mock_finding1, mock_finding2] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + _, findings_count_by_compliance = _aggregate_findings_by_region( + tenant_id, scan_id, modeled_threatscore_compliance_id + ) + + # Verify compliance counts + normalized_id = re.sub( + r"[^a-z0-9]", "", modeled_threatscore_compliance_id.lower() + ) + assert "us-east-1" in findings_count_by_compliance + assert normalized_id in findings_count_by_compliance["us-east-1"] + assert "req1" in findings_count_by_compliance["us-east-1"][normalized_id] + + req_stats = findings_count_by_compliance["us-east-1"][normalized_id]["req1"] + assert req_stats["total"] == 2 + assert req_stats["pass"] == 1 + + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_by_region_multiple_regions( + self, mock_rls_transaction, mock_findings_filter + ): + """Test aggregation across multiple regions.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" + + # Finding in us-east-1 + mock_finding1 = MagicMock() + mock_finding1.check_id = "check1" + mock_finding1.status = "FAIL" + mock_finding1.compliance = {} + mock_resource1 = MagicMock() + mock_resource1.region = "us-east-1" + mock_finding1.small_resources = [mock_resource1] + + # Finding in us-west-2 + mock_finding2 = MagicMock() + mock_finding2.check_id = "check1" + mock_finding2.status = "PASS" + mock_finding2.compliance = {} + mock_resource2 = MagicMock() + mock_resource2.region = "us-west-2" + mock_finding2.small_resources = [mock_resource2] + + mock_queryset = MagicMock() + mock_queryset.only.return_value = mock_queryset + mock_queryset.prefetch_related.return_value = [mock_finding1, mock_finding2] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + check_status_by_region, _ = _aggregate_findings_by_region( + tenant_id, scan_id, modeled_threatscore_compliance_id + ) + + # Verify both regions are present with correct statuses + assert "us-east-1" in check_status_by_region + assert "us-west-2" in check_status_by_region + assert check_status_by_region["us-east-1"]["check1"] == "FAIL" + assert check_status_by_region["us-west-2"]["check1"] == "PASS" + + @patch("tasks.jobs.scan.Finding.all_objects.filter") + @patch("tasks.jobs.scan.rls_transaction") + def test_aggregate_findings_by_region_empty_findings( + self, mock_rls_transaction, mock_findings_filter + ): + """Test with no findings - should return empty dicts.""" + tenant_id = str(uuid.uuid4()) + scan_id = str(uuid.uuid4()) + modeled_threatscore_compliance_id = "ProwlerThreatScore-1.0" + + mock_queryset = MagicMock() + mock_queryset.only.return_value = mock_queryset + mock_queryset.prefetch_related.return_value = [] + + ctx = MagicMock() + ctx.__enter__.return_value = None + ctx.__exit__.return_value = False + mock_rls_transaction.return_value = ctx + mock_findings_filter.return_value = mock_queryset + + check_status_by_region, findings_count_by_compliance = ( + _aggregate_findings_by_region( + tenant_id, scan_id, modeled_threatscore_compliance_id + ) + ) + + assert check_status_by_region == {} + assert findings_count_by_compliance == {} diff --git a/api/src/backend/tasks/tests/test_tasks.py b/api/src/backend/tasks/tests/test_tasks.py index a98341ef35..b2d8b7bc2b 100644 --- a/api/src/backend/tasks/tests/test_tasks.py +++ b/api/src/backend/tasks/tests/test_tasks.py @@ -1,16 +1,24 @@ import uuid from unittest.mock import MagicMock, patch +import openai import pytest +from botocore.exceptions import ClientError from tasks.tasks import ( _perform_scan_complete_tasks, check_integrations_task, + check_lighthouse_provider_connection_task, generate_outputs_task, + refresh_lighthouse_provider_models_task, s3_integration_task, security_hub_integration_task, ) -from api.models import Integration +from api.models import ( + Integration, + LighthouseProviderConfiguration, + LighthouseProviderModels, +) # TODO Move this to outputs/reports jobs @@ -1097,3 +1105,363 @@ class TestCheckIntegrationsTask: assert result is False mock_upload.assert_called_once_with(self.tenant_id, self.provider_id, scan_id) + + +@pytest.mark.django_db +class TestCheckLighthouseProviderConnectionTask: + def setup_method(self): + self.tenant_id = str(uuid.uuid4()) + + @pytest.mark.parametrize( + "provider_type,credentials,base_url,expected_result", + [ + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + {"api_key": "sk-test123"}, + None, + {"connected": True, "error": None}, + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + {"api_key": "sk-test123"}, + "https://openrouter.ai/api/v1", + {"connected": True, "error": None}, + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + { + "access_key_id": "AKIA123", + "secret_access_key": "secret", + "region": "us-east-1", + }, + None, + {"connected": True, "error": None}, + ), + ], + ) + def test_check_connection_success_all_providers( + self, tenants_fixture, provider_type, credentials, base_url, expected_result + ): + """Test successful connection check for all provider types.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=provider_type, + base_url=base_url, + is_active=False, + ) + provider_cfg.credentials_decoded = credentials + provider_cfg.save() + + # Mock the appropriate API calls + with ( + patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai, + patch("tasks.jobs.lighthouse_providers.boto3.client") as mock_boto3, + ): + mock_client = MagicMock() + mock_client.models.list.return_value = MagicMock() + mock_client.list_foundation_models.return_value = {} + mock_openai.return_value = mock_client + mock_boto3.return_value = mock_client + + # Execute + result = check_lighthouse_provider_connection_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result == expected_result + provider_cfg.refresh_from_db() + assert provider_cfg.is_active is True + + @pytest.mark.parametrize( + "provider_type,credentials,base_url,exception_to_raise", + [ + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + {"api_key": "sk-invalid"}, + None, + openai.AuthenticationError( + "Invalid API key", response=MagicMock(), body=None + ), + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + {"api_key": "sk-invalid"}, + "https://openrouter.ai/api/v1", + openai.APIConnectionError(request=MagicMock()), + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + { + "access_key_id": "AKIA123", + "secret_access_key": "secret", + "region": "us-east-1", + }, + None, + ClientError( + {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, + "list_foundation_models", + ), + ), + ], + ) + def test_check_connection_api_failure( + self, + tenants_fixture, + provider_type, + credentials, + base_url, + exception_to_raise, + ): + """Test connection check when API calls fail.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=provider_type, + base_url=base_url, + is_active=True, + ) + provider_cfg.credentials_decoded = credentials + provider_cfg.save() + + # Mock the API to raise exception + with ( + patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai, + patch("tasks.jobs.lighthouse_providers.boto3.client") as mock_boto3, + ): + mock_client = MagicMock() + if ( + provider_type + == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK + ): + mock_client.list_foundation_models.side_effect = exception_to_raise + mock_boto3.return_value = mock_client + else: + mock_client.models.list.side_effect = exception_to_raise + mock_openai.return_value = mock_client + + # Execute + result = check_lighthouse_provider_connection_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result["connected"] is False + assert result["error"] is not None + provider_cfg.refresh_from_db() + assert provider_cfg.is_active is False + + def test_check_connection_updates_active_status(self, tenants_fixture): + """Test that connection check toggles is_active from True to False on failure.""" + # Create provider with is_active=True + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + base_url=None, + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "sk-test123"} + provider_cfg.save() + + # Mock API to fail + with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai: + mock_client = MagicMock() + mock_client.models.list.side_effect = openai.AuthenticationError( + "Invalid", response=MagicMock(), body=None + ) + mock_openai.return_value = mock_client + + # Execute + result = check_lighthouse_provider_connection_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert status changed + assert result["connected"] is False + provider_cfg.refresh_from_db() + assert provider_cfg.is_active is False + + def test_check_connection_provider_does_not_exist(self, tenants_fixture): + """Test that checking non-existent provider raises DoesNotExist.""" + non_existent_id = str(uuid.uuid4()) + + with pytest.raises(LighthouseProviderConfiguration.DoesNotExist): + check_lighthouse_provider_connection_task( + provider_config_id=non_existent_id, + tenant_id=str(tenants_fixture[0].id), + ) + + +@pytest.mark.django_db +class TestRefreshLighthouseProviderModelsTask: + def setup_method(self): + self.tenant_id = str(uuid.uuid4()) + + @pytest.mark.parametrize( + "provider_type,credentials,base_url,mock_models,expected_count", + [ + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + {"api_key": "sk-test123"}, + None, + {"gpt-5": "gpt-5", "gpt-4o": "gpt-4o"}, + 2, + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE, + {"api_key": "sk-test123"}, + "https://openrouter.ai/api/v1", + {"model-1": "Model One", "model-2": "Model Two"}, + 2, + ), + ( + LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK, + { + "access_key_id": "AKIA123", + "secret_access_key": "secret", + "region": "us-east-1", + }, + None, + {"openai.gpt-oss-120b-1:0": "gpt-oss-120b"}, + 1, + ), + ], + ) + def test_refresh_models_create_new( + self, + tenants_fixture, + provider_type, + credentials, + base_url, + mock_models, + expected_count, + ): + """Test creating new models for all provider types.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=provider_type, + base_url=base_url, + is_active=True, + ) + provider_cfg.credentials_decoded = credentials + provider_cfg.save() + + # Mock the fetch functions + with ( + patch( + "tasks.jobs.lighthouse_providers._fetch_openai_models", + return_value=mock_models, + ), + patch( + "tasks.jobs.lighthouse_providers._fetch_openai_compatible_models", + return_value=mock_models, + ), + patch( + "tasks.jobs.lighthouse_providers._fetch_bedrock_models", + return_value=mock_models, + ), + ): + # Execute + result = refresh_lighthouse_provider_models_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result["created"] == expected_count + assert result["updated"] == 0 + assert result["deleted"] == 0 + assert ( + LighthouseProviderModels.objects.filter( + provider_configuration=provider_cfg + ).count() + == expected_count + ) + + def test_refresh_models_mixed_operations(self, tenants_fixture): + """Test mixed create, update, and delete operations.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + base_url=None, + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "sk-test123"} + provider_cfg.save() + + # Create 2 existing models (A, B) + LighthouseProviderModels.objects.create( + tenant_id=tenants_fixture[0].id, + provider_configuration=provider_cfg, + model_id="model-a", + model_name="Model A", + ) + LighthouseProviderModels.objects.create( + tenant_id=tenants_fixture[0].id, + provider_configuration=provider_cfg, + model_id="model-b", + model_name="Model B", + ) + + # Mock API to return models B (existing), C (new) - A will be deleted + mock_models = {"model-b": "Model B", "model-c": "Model C"} + with patch( + "tasks.jobs.lighthouse_providers._fetch_openai_models", + return_value=mock_models, + ): + # Execute + result = refresh_lighthouse_provider_models_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result["created"] == 1 # model-c created + assert result["updated"] == 1 # model-b updated + assert result["deleted"] == 1 # model-a deleted + + # Verify only B and C exist + remaining_models = LighthouseProviderModels.objects.filter( + provider_configuration=provider_cfg + ) + assert remaining_models.count() == 2 + assert set(remaining_models.values_list("model_id", flat=True)) == { + "model-b", + "model-c", + } + + def test_refresh_models_api_exception(self, tenants_fixture): + """Test refresh when API raises an exception.""" + # Create provider configuration + provider_cfg = LighthouseProviderConfiguration( + tenant_id=tenants_fixture[0].id, + provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI, + base_url=None, + is_active=True, + ) + provider_cfg.credentials_decoded = {"api_key": "sk-test123"} + provider_cfg.save() + + # Mock fetch to raise exception + with patch( + "tasks.jobs.lighthouse_providers._fetch_openai_models", + side_effect=openai.APIError("API Error", request=MagicMock(), body=None), + ): + # Execute + result = refresh_lighthouse_provider_models_task( + provider_config_id=str(provider_cfg.id), + tenant_id=str(tenants_fixture[0].id), + ) + + # Assert + assert result["created"] == 0 + assert result["updated"] == 0 + assert result["deleted"] == 0 + assert "error" in result + assert result["error"] is not None diff --git a/dashboard/__main__.py b/dashboard/__main__.py index dac536ccdf..304e4afdf2 100644 --- a/dashboard/__main__.py +++ b/dashboard/__main__.py @@ -35,7 +35,8 @@ dashboard = dash.Dash( # Logo prowler_logo = html.Img( - src="https://prowler.com/wp-content/uploads/logo-dashboard.png", alt="Prowler Logo" + src="https://cdn.prod.website-files.com/68c4ec3f9fb7b154fbcb6e36/68ffb46d40ed7faa37a592a5_prowler-logo.png", + alt="Prowler Logo", ) menu_icons = { diff --git a/dashboard/compliance/c5_azure.py b/dashboard/compliance/c5_azure.py new file mode 100644 index 0000000000..8baac9a9a5 --- /dev/null +++ b/dashboard/compliance/c5_azure.py @@ -0,0 +1,43 @@ +import warnings + +from dashboard.common_methods import get_section_containers_3_levels + +warnings.filterwarnings("ignore") + + +def get_table(data): + data["REQUIREMENTS_DESCRIPTION"] = ( + data["REQUIREMENTS_ID"] + " - " + data["REQUIREMENTS_DESCRIPTION"] + ) + + data["REQUIREMENTS_DESCRIPTION"] = data["REQUIREMENTS_DESCRIPTION"].apply( + lambda x: x[:150] + "..." if len(str(x)) > 150 else x + ) + + data["REQUIREMENTS_ATTRIBUTES_SECTION"] = data[ + "REQUIREMENTS_ATTRIBUTES_SECTION" + ].apply(lambda x: x[:80] + "..." if len(str(x)) > 80 else x) + + data["REQUIREMENTS_ATTRIBUTES_SUBSECTION"] = data[ + "REQUIREMENTS_ATTRIBUTES_SUBSECTION" + ].apply(lambda x: x[:150] + "..." if len(str(x)) > 150 else x) + + aux = data[ + [ + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ] + + return get_section_containers_3_levels( + aux, + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_ATTRIBUTES_SUBSECTION", + "REQUIREMENTS_DESCRIPTION", + ) diff --git a/dashboard/compliance/fedramp_20x_ksi_low_aws.py b/dashboard/compliance/fedramp_20x_ksi_low_aws.py new file mode 100644 index 0000000000..5ca220301f --- /dev/null +++ b/dashboard/compliance/fedramp_20x_ksi_low_aws.py @@ -0,0 +1,46 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + # Shorten the long FedRAMP KSI descriptions for better display + ksi_short_names = { + "A secure cloud service offering will protect user data, control access, and apply zero trust principles": "Identity and Access Management", + "A secure cloud service offering will use cloud native architecture and design principles to enforce and enhance the Confidentiality, Integrity and Availability of the system": "Cloud Native Architecture", + "A secure cloud service provider will ensure that all system changes are properly documented and configuration baselines are updated accordingly": "Change Management", + "A secure cloud service provider will continuously educate their employees on cybersecurity measures, testing them regularly": "Cybersecurity Education", + "A secure cloud service offering will document, report, and analyze security incidents to ensure regulatory compliance and continuous security improvement": "Incident Reporting", + "A secure cloud service offering will monitor, log, and audit all important events, activity, and changes": "Monitoring, Logging, and Auditing", + "A secure cloud service offering will have intentional, organized, universal guidance for how every information resource, including personnel, is secured": "Policy and Inventory", + "A secure cloud service offering will define, maintain, and test incident response plan(s) and recovery capabilities to ensure minimal service disruption and data loss": "Recovery Planning", + "A secure cloud service offering will follow FedRAMP encryption policies, continuously verify information resource integrity, and restrict access to third-party information resources": "Service Configuration", + "A secure cloud service offering will understand, monitor, and manage supply chain risks from third-party information resources": "Third-Party Information Resources", + } + + # Replace long descriptions with short names - use contains for partial matching + if not aux.empty: + for long_desc, short_name in ksi_short_names.items(): + mask = aux["REQUIREMENTS_DESCRIPTION"].str.contains( + long_desc, na=False, regex=False + ) + aux.loc[mask, "REQUIREMENTS_DESCRIPTION"] = short_name + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/fedramp_20x_ksi_low_azure.py b/dashboard/compliance/fedramp_20x_ksi_low_azure.py new file mode 100644 index 0000000000..5ca220301f --- /dev/null +++ b/dashboard/compliance/fedramp_20x_ksi_low_azure.py @@ -0,0 +1,46 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + # Shorten the long FedRAMP KSI descriptions for better display + ksi_short_names = { + "A secure cloud service offering will protect user data, control access, and apply zero trust principles": "Identity and Access Management", + "A secure cloud service offering will use cloud native architecture and design principles to enforce and enhance the Confidentiality, Integrity and Availability of the system": "Cloud Native Architecture", + "A secure cloud service provider will ensure that all system changes are properly documented and configuration baselines are updated accordingly": "Change Management", + "A secure cloud service provider will continuously educate their employees on cybersecurity measures, testing them regularly": "Cybersecurity Education", + "A secure cloud service offering will document, report, and analyze security incidents to ensure regulatory compliance and continuous security improvement": "Incident Reporting", + "A secure cloud service offering will monitor, log, and audit all important events, activity, and changes": "Monitoring, Logging, and Auditing", + "A secure cloud service offering will have intentional, organized, universal guidance for how every information resource, including personnel, is secured": "Policy and Inventory", + "A secure cloud service offering will define, maintain, and test incident response plan(s) and recovery capabilities to ensure minimal service disruption and data loss": "Recovery Planning", + "A secure cloud service offering will follow FedRAMP encryption policies, continuously verify information resource integrity, and restrict access to third-party information resources": "Service Configuration", + "A secure cloud service offering will understand, monitor, and manage supply chain risks from third-party information resources": "Third-Party Information Resources", + } + + # Replace long descriptions with short names - use contains for partial matching + if not aux.empty: + for long_desc, short_name in ksi_short_names.items(): + mask = aux["REQUIREMENTS_DESCRIPTION"].str.contains( + long_desc, na=False, regex=False + ) + aux.loc[mask, "REQUIREMENTS_DESCRIPTION"] = short_name + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/fedramp_20x_ksi_low_gcp.py b/dashboard/compliance/fedramp_20x_ksi_low_gcp.py new file mode 100644 index 0000000000..5ca220301f --- /dev/null +++ b/dashboard/compliance/fedramp_20x_ksi_low_gcp.py @@ -0,0 +1,46 @@ +import warnings + +from dashboard.common_methods import get_section_containers_cis + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_DESCRIPTION", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + # Shorten the long FedRAMP KSI descriptions for better display + ksi_short_names = { + "A secure cloud service offering will protect user data, control access, and apply zero trust principles": "Identity and Access Management", + "A secure cloud service offering will use cloud native architecture and design principles to enforce and enhance the Confidentiality, Integrity and Availability of the system": "Cloud Native Architecture", + "A secure cloud service provider will ensure that all system changes are properly documented and configuration baselines are updated accordingly": "Change Management", + "A secure cloud service provider will continuously educate their employees on cybersecurity measures, testing them regularly": "Cybersecurity Education", + "A secure cloud service offering will document, report, and analyze security incidents to ensure regulatory compliance and continuous security improvement": "Incident Reporting", + "A secure cloud service offering will monitor, log, and audit all important events, activity, and changes": "Monitoring, Logging, and Auditing", + "A secure cloud service offering will have intentional, organized, universal guidance for how every information resource, including personnel, is secured": "Policy and Inventory", + "A secure cloud service offering will define, maintain, and test incident response plan(s) and recovery capabilities to ensure minimal service disruption and data loss": "Recovery Planning", + "A secure cloud service offering will follow FedRAMP encryption policies, continuously verify information resource integrity, and restrict access to third-party information resources": "Service Configuration", + "A secure cloud service offering will understand, monitor, and manage supply chain risks from third-party information resources": "Third-Party Information Resources", + } + + # Replace long descriptions with short names - use contains for partial matching + if not aux.empty: + for long_desc, short_name in ksi_short_names.items(): + mask = aux["REQUIREMENTS_DESCRIPTION"].str.contains( + long_desc, na=False, regex=False + ) + aux.loc[mask, "REQUIREMENTS_DESCRIPTION"] = short_name + + return get_section_containers_cis( + aux, "REQUIREMENTS_ID", "REQUIREMENTS_ATTRIBUTES_SECTION" + ) diff --git a/dashboard/compliance/hipaa_gcp.py b/dashboard/compliance/hipaa_gcp.py new file mode 100644 index 0000000000..b0a8eb6582 --- /dev/null +++ b/dashboard/compliance/hipaa_gcp.py @@ -0,0 +1,25 @@ +import warnings + +from dashboard.common_methods import get_section_containers_format3 + +warnings.filterwarnings("ignore") + + +def get_table(data): + + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_DESCRIPTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_format3( + aux, "REQUIREMENTS_ATTRIBUTES_SECTION", "REQUIREMENTS_ID" + ) diff --git a/dashboard/compliance/nist_csf_2_0_aws.py b/dashboard/compliance/nist_csf_2_0_aws.py new file mode 100644 index 0000000000..07b956b104 --- /dev/null +++ b/dashboard/compliance/nist_csf_2_0_aws.py @@ -0,0 +1,24 @@ +import warnings + +from dashboard.common_methods import get_section_containers_format3 + +warnings.filterwarnings("ignore") + + +def get_table(data): + aux = data[ + [ + "REQUIREMENTS_ID", + "REQUIREMENTS_ATTRIBUTES_SECTION", + "REQUIREMENTS_DESCRIPTION", + "CHECKID", + "STATUS", + "REGION", + "ACCOUNTID", + "RESOURCEID", + ] + ].copy() + + return get_section_containers_format3( + aux, "REQUIREMENTS_ATTRIBUTES_SECTION", "REQUIREMENTS_ID" + ) diff --git a/docs/developer-guide/checks.mdx b/docs/developer-guide/checks.mdx index deeb7b50a5..7a86686071 100644 --- a/docs/developer-guide/checks.mdx +++ b/docs/developer-guide/checks.mdx @@ -125,11 +125,11 @@ Each check **must** populate the `report.status` and `report.status_extended` fi The severity of each check is defined in the metadata file using the `Severity` field. Severity values are always lowercase and must be one of the predefined categories below. -- `critical` – Issue that must be addressed immediately. -- `high` – Issue that should be addressed as soon as possible. -- `medium` – Issue that should be addressed within a reasonable timeframe. -- `low` – Issue that can be addressed in the future. -- `informational` – Not an issue but provides valuable information. +- `critical` – Highest potential impact with broad exposure that could affect core security boundaries or business operations. +- `high` – Substantial potential impact with significant exposure that could affect important security controls or resources. +- `medium` – Moderate potential impact with limited exposure that weakens defense layers but has contained scope. +- `low` – Minimal potential impact with negligible exposure that represents minor gaps in security posture. +- `informational` – Provides valuable information but does not affect the security posture. If the check involves multiple scenarios that may alter its severity, adjustments can be made dynamically within the check's logic using the severity `report.check_metadata.Severity` attribute: diff --git a/docs/developer-guide/documentation.mdx b/docs/developer-guide/documentation.mdx index a7a4e97d4e..f1fae30d35 100644 --- a/docs/developer-guide/documentation.mdx +++ b/docs/developer-guide/documentation.mdx @@ -4,7 +4,26 @@ title: 'Contributing to Documentation' Prowler documentation is built using [Mintlify](https://www.mintlify.com/docs), allowing contributors to easily add or enhance documentation. -## Installation and Setup +## Documentation Structure + +The Prowler documentation is organized into several sections. The main ones are: + +- **Getting Started**: Provides an overview of the Prowler platform and its different solutions, including Prowler Cloud/App, Prowler CLI, Prowler MCP Server, Prowler Hub, and Prowler Lighthouse AI. This section helps new users understand which Prowler solution best fits their needs and includes product comparisons. + +- **Guides**: Contains practical tutorials and how-to guides organized by product (Prowler Cloud/App, CLI) and provider (AWS, Azure, GCP, Kubernetes, Microsoft 365, GitHub, etc.). This section covers authentication, integrations, compliance, and advanced usage scenarios. + +- **Developer Guide**: Documentation for contributors looking to extend Prowler functionality. This includes guides on creating providers, services, checks, output formats, integrations, and compliance frameworks. Provider-specific implementation details and testing strategies are also covered here. + +- **Troubleshooting**: Common issues, error messages, and their solutions. This section helps users resolve problems encountered during installation, configuration, or execution. + + +## AI-Driven Documentation + +As mentioned in the [Introduction](/developer-guide/introduction#ai-driven-contributions), we have specialized resources to enhance AI-driven development. + +This includes the [AGENTS.md](https://github.com/prowler-cloud/prowler/blob/master/docs/AGENTS.md) file that contains the guidelines and style guide for the AI agents in the Prowler documentation. + +## Local Development @@ -33,10 +52,10 @@ Prowler documentation is built using [Mintlify](https://www.mintlify.com/docs), - Once documentation updates are complete, submit a pull request for review. + Once documentation updates are complete, [submit a pull request for review](/developer-guide/introduction#sending-the-pull-request). The Prowler team will assess and merge contributions. -Your efforts help improve Prowler documentation—thank you for contributing! +Your efforts help improve Prowler documentation. Thank you for contributing! 🤘 diff --git a/docs/developer-guide/introduction.mdx b/docs/developer-guide/introduction.mdx index 5361d4460a..9d1ea742b1 100644 --- a/docs/developer-guide/introduction.mdx +++ b/docs/developer-guide/introduction.mdx @@ -2,19 +2,70 @@ title: 'Introduction to developing in Prowler' --- -Extending Prowler +Thanks for your interest in contributing to Prowler! -Prowler can be extended in various ways, with common use cases including: +Prowler can be extended in various ways. This guide provides the different ways to contribute and how to get started. -- New security checks -- New compliance frameworks -- New output formats -- New integrations -- New proposed features +## Contributing to Prowler -All the relevant information for these cases is included in this guide. +### Review Current Issues +Check out our [GitHub Issues](https://github.com/prowler-cloud/prowler/issues) page for ideas to contribute. + + + We tag issues as `good first issue` for new contributors. These are typically well-defined and manageable in scope. + + + We tag issues as `help wanted` for other issues that require more time to complete. + + -## Getting the Code and Installing All Dependencies +### Expand Prowler's Capabilities +Prowler is constantly evolving. Contributions to checks, services, or integrations help improve the tool for everyone. Here is how to get involved: + + + + Want to improve Prowler's detection capabilities for your favorite cloud provider? You can contribute by writing new checks. + + + One key service for your favorite cloud provider is missing? Add it to Prowler! Do not forget to include relevant checks to validate functionality. + + + If you would like to extend Prowler to work with a new cloud provider, this typically involves setting up new services and checks to ensure compatibility. + + + Want to tailor how results are displayed or exported? You can add custom output formats. + + + Prowler can work with other tools and platforms through integrations. + + + Propose brand-new features or enhancements to existing ones, or help implement community-requested improvements. + + + +### Improve Documentation +Help make Prowler more accessible by enhancing our documentation, fixing typos, or adding examples/tutorials. + + + + Enhance our documentation, fix typos, or add examples/tutorials. + + + +### Bug Fixes +If you find any issues or bugs, you can report them in the [GitHub Issues](https://github.com/prowler-cloud/prowler/issues) page and if you want you can also fix them. + + + + Report or fix issues or bugs. + + + +Remember, our community is here to help! If you need guidance, do not hesitate to ask questions in the issues or join our [ Slack workspace](https://goto.prowler.com/slack). + + + +## Setting up your development environment ### Prerequisites @@ -26,11 +77,11 @@ Before proceeding, ensure the following: ### Forking the Prowler Repository -To contribute to Prowler, fork the Prowler GitHub repository. This allows you to propose changes, submit new features, and fix bugs. For guidance on forking, refer to the [official GitHub documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo?tool=webui#forking-a-repository). +Fork the Prowler GitHub repository to contribute to Prowler. This allows proposing changes, submitting new features, and fixing bugs. For guidance on forking, refer to the [official GitHub documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo?tool=webui#forking-a-repository). ### Cloning Your Forked Repository -Once your fork is created, clone it using the following commands: +Once your fork is created, clone it using the following commands (replace `` with your GitHub username): ``` git clone https://github.com//prowler @@ -56,39 +107,7 @@ If your poetry version is below 2.0.0 you must keep using `poetry shell` to acti In case you have any doubts, consult the [Poetry environment activation guide](https://python-poetry.org/docs/managing-environments/#activating-the-environment). -## Contributing to Prowler -### Ways to Contribute - -Here are some ideas for collaborating with Prowler: - -1. **Review Current Issues**: Check out our [GitHub Issues](https://github.com/prowler-cloud/prowler/issues) page. We often tag issues as `good first issue` - these are perfect for new contributors as they are typically well-defined and manageable in scope. - -2. **Expand Prowler's Capabilities**: Prowler is constantly evolving, and you can be a part of its growth. Whether you are adding checks, supporting new services, or introducing integrations, your contributions help improve the tool for everyone. Here is how you can get involved: - - - **Adding New Checks** - Want to improve Prowler's detection capabilities for your favorite cloud provider? You can contribute by writing new checks. To get started, follow the [create a new check guide](/developer-guide/checks). - - - **Adding New Services** - One key service for your favorite cloud provider is missing? Add it to Prowler! To add a new service, check out the [create a new service guide](/developer-guide/services). Do not forget to include relevant checks to validate functionality. - - - **Adding New Providers** - If you would like to extend Prowler to work with a new cloud provider, follow the [create a new provider guide](/developer-guide/provider). This typically involves setting up new services and checks to ensure compatibility. - - - **Adding New Output Formats** - Want to tailor how results are displayed or exported? You can add custom output formats by following the [create a new output format guide](/developer-guide/outputs). - - - **Adding New Integrations** - Prowler can work with other tools and platforms through integrations. If you would like to add one, see the [create a new integration guide](/developer-guide/integrations). - - - **Proposing or Implementing Features** - Got an idea to make Prowler better? Whether it is a brand-new feature or an enhancement to an existing one, you are welcome to propose it or help implement community-requested improvements. - -3. **Improve Documentation**: Help make Prowler more accessible by enhancing our documentation, fixing typos, or adding examples/tutorials. See the tutorial of how we write our documentation [here](/developer-guide/documentation). - -4. **Bug Fixes**: If you find any issues or bugs, you can report them in the [GitHub Issues](https://github.com/prowler-cloud/prowler/issues) page and if you want you can also fix them. - -Remember, our community is here to help! If you need guidance, do not hesitate to ask questions in the issues or join our [Slack workspace](https://goto.prowler.com/slack). ### Pre-Commit Hooks @@ -121,6 +140,16 @@ These should have been already installed if `poetry install --with dev` was alre Additionally, ensure the latest version of [`TruffleHog`](https://github.com/trufflesecurity/trufflehog) is installed to scan for sensitive data in the code. Follow the official [installation guide](https://github.com/trufflesecurity/trufflehog?tab=readme-ov-file#floppy_disk-installation) for setup. +### AI-Driven Contributions + +If you are using AI assistants to help with your contributions, Prowler provides specialized resources to enhance AI-driven development: + +- **Prowler MCP Server**: The [Prowler MCP Server](/getting-started/products/prowler-mcp) provides AI assistants with access to the entire Prowler ecosystem, including security checks, compliance frameworks, documentation, and more. This enables AI tools to better understand Prowler's architecture and help you create contributions that align with project standards. + +- **AGENTS.md Files**: Each component of the Prowler monorepo includes an `AGENTS.md` file that contains specific guidelines for AI agents working on that component. These files provide context about project structure, coding standards, and best practices. When working on a specific component, refer to the relevant `AGENTS.md` file (e.g., `prowler/AGENTS.md`, `ui/AGENTS.md`, `api/AGENTS.md`) to ensure your AI assistant follows the appropriate guidelines. + +These resources help ensure that AI-assisted contributions maintain consistency with Prowler's codebase and development practices. + ### Dependency Management All dependencies are listed in the `pyproject.toml` file. @@ -133,7 +162,7 @@ If you encounter issues when committing to the Prowler repository, use the `--no ### Repository Folder Structure -Understanding the layout of the Prowler codebase will help you quickly find where to add new features, checks, or integrations. The following is a high-level overview from the root of the repository: +The Prowler codebase layout helps quickly locate where to add new features, checks, or integrations. The following is a high-level overview from the root of the repository: ``` prowler/ @@ -148,7 +177,7 @@ prowler/ ├── permissions/ # Permission-related files and policies ├── contrib/ # Community-contributed scripts or modules ├── kubernetes/ # Kubernetes deployment files -├── .github/ # GitHub related files (workflows, issue templates, etc.) +├── .github/ # GitHub-related files (workflows, issue templates, etc.) ├── pyproject.toml # Python project configuration (Poetry) ├── poetry.lock # Poetry lock file ├── README.md # Project overview and getting started @@ -158,19 +187,23 @@ prowler/ └── ... # Other supporting files ``` -## Pull Request Checklist +## Sending the Pull Request -When creating or reviewing a pull request in https://github.com/prowler-cloud/prowler, follow [this checklist](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md#checklist). +When creating or reviewing a pull request in [Prowler](https://github.com/prowler-cloud/prowler), follow [this template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md) and fill it with the relevant information: + +- **Context** and **Description** of the change: This will help the reviewers to understand the change and the purpose of the pull request. +- **Steps to review**: A detailed description of how to review the change. +- **Checklist**: A mandatory checklist of the things that should be reviewed before merging the pull request. ## Contribution Appreciation -If you enjoy swag, we’d love to thank you for your contribution with laptop stickers or other Prowler merchandise! +If you enjoy swag, we'd love to thank you for your contribution with laptop stickers or other Prowler merchandise! To request swag: Share your pull request details in our [Slack workspace](https://goto.prowler.com/slack). You can also reach out to Toni de la Fuente on [Twitter](https://twitter.com/ToniBlyx)—his DMs are open! -# Testing a Pull Request from a Specific Branch +## Testing a Pull Request from a Specific Branch To test Prowler from a specific branch (for example, to try out changes from a pull request before it is merged), you can use `pipx` to install directly from GitHub: @@ -179,3 +212,5 @@ pipx install "git+https://github.com/prowler-cloud/prowler.git@branch-name" ``` Replace `branch-name` with the name of the branch you want to test. This will install Prowler in an isolated environment, allowing you to try out the changes safely. + +For more details on testing go to the [Testing section](/developer-guide/unit-testing) of this documentation. \ No newline at end of file diff --git a/docs/developer-guide/kubernetes-details.mdx b/docs/developer-guide/kubernetes-details.mdx index fa0c9291f6..0ccda94f38 100644 --- a/docs/developer-guide/kubernetes-details.mdx +++ b/docs/developer-guide/kubernetes-details.mdx @@ -4,7 +4,7 @@ title: 'Kubernetes Provider' This page details the [Kubernetes](https://kubernetes.io/) provider implementation in Prowler. -By default, Prowler will audit all namespaces in the Kubernetes cluster accessible by the configured context. To configure it, see the [In-Cluster Execution](/user-guide/providers/kubernetes/in-cluster) or [Non In-Cluster Execution](/user-guide/providers/kubernetes/outside-cluster) guides. +By default, Prowler will audit all namespaces in the Kubernetes cluster accessible by the configured context. To configure it, see the [In-Cluster Execution](/user-guide/providers/kubernetes/getting-started-k8s#in-cluster-execution) or [Non In-Cluster Execution](/user-guide/providers/kubernetes/getting-started-k8s#non-in-cluster-execution) guides. ## Kubernetes Provider Classes Architecture diff --git a/docs/docs.json b/docs/docs.json index 54402e5d8a..7d3f795482 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -110,6 +110,7 @@ ] }, "user-guide/tutorials/prowler-app-lighthouse", + "user-guide/tutorials/prowler-cloud-public-ips", { "group": "Tutorials", "pages": [ @@ -194,8 +195,7 @@ { "group": "Kubernetes", "pages": [ - "user-guide/providers/kubernetes/in-cluster", - "user-guide/providers/kubernetes/outside-cluster", + "user-guide/providers/kubernetes/getting-started-k8s", "user-guide/providers/kubernetes/misc" ] }, diff --git a/docs/getting-started/basic-usage/prowler-mcp.mdx b/docs/getting-started/basic-usage/prowler-mcp.mdx index 5d89756acf..b6d59093e7 100644 --- a/docs/getting-started/basic-usage/prowler-mcp.mdx +++ b/docs/getting-started/basic-usage/prowler-mcp.mdx @@ -4,7 +4,7 @@ title: "Configuration" Configure your MCP client to connect to Prowler MCP Server. -## Step 1: Get Your API Key (Optional) +## Step 1: Get Your API Key **Authentication is optional**: Prowler Hub and Prowler Documentation features work without authentication. An API key is only required for Prowler Cloud and Prowler App (Self-Managed) features. @@ -16,21 +16,17 @@ To use Prowler Cloud or Prowler App (Self-Managed) features. To get the API key, Keep the API key secure. Never share it publicly or commit it to version control. -## Step 2: Configure Your MCP Client +## Step 2: Configure Your MCP Host/Client Choose the configuration based on your deployment: -- **STDIO Mode**: Local installation only (runs as subprocess). - **HTTP Mode**: Prowler Cloud MCP Server or self-hosted Prowler MCP Server. +- **STDIO Mode**: Local installation only (runs as subprocess of your MCP client). -### HTTP Mode (Prowler Cloud MCP Server or self-hosted Prowler MCP Server) +### HTTP Mode - - **Clients that support HTTP with custom headers natively** - - For example: Cursor, VSCode, LobeChat, etc. - + **Configuration:** ```json { @@ -38,20 +34,15 @@ Choose the configuration based on your deployment: "prowler": { "url": "https://mcp.prowler.com/mcp", // or your self-hosted Prowler MCP Server URL "headers": { - "Authorization": "Bearer pk_your_api_key_here" + "Authorization": "Bearer " } } } } ``` - - - **For clients without native HTTP support (like Claude Desktop)** - - For example: Claude Desktop. - + **Configuration:** ```json { @@ -65,26 +56,79 @@ Choose the configuration based on your deployment: "Authorization: Bearer ${PROWLER_APP_API_KEY}" ], "env": { - "PROWLER_APP_API_KEY": "pk_your_api_key_here" + "PROWLER_APP_API_KEY": "" } } } } ``` - The `mcp-remote` tool acts as a bridge for clients that don't support HTTP natively. Learn more at [mcp-remote on npm](https://www.npmjs.com/package/mcp-remote). - + + + 1. Open Claude Desktop settings + 2. Go to "Developer" tab + 3. Click in "Edit Config" button + 4. Edit the `claude_desktop_config.json` file with your favorite editor + 5. Add the following configuration: + ```json + { + "mcpServers": { + "prowler": { + "command": "npx", + "args": [ + "mcp-remote", + "https://mcp.prowler.com/mcp", + "--header", + "Authorization: Bearer ${PROWLER_APP_API_KEY}" + ], + "env": { + "PROWLER_APP_API_KEY": "" + } + } + } + } + ``` + + + + Run the following command: + ```bash + export PROWLER_APP_API_KEY="" + claude mcp add --transport http prowler https://mcp.prowler.com/mcp --header "Authorization: Bearer $PROWLER_APP_API_KEY" --scope user + ``` + + + + 1. Open Cursor settings + 2. Go to "Tools & MCP" + 3. Click in "New MCP Server" button + 4. Add to the JSON Configuration the following: + ```json + { + "mcpServers": { + "prowler": { + "url": "https://mcp.prowler.com/mcp", + "headers": { + "Authorization": "Bearer " + } + } + } + } + ``` + + + -### STDIO Mode (Local Installation Only) +### STDIO Mode STDIO mode is only available when running the MCP server locally. - + **Run from source or local installation** ```json @@ -94,7 +138,7 @@ STDIO mode is only available when running the MCP server locally. "command": "uvx", "args": ["/absolute/path/to/prowler/mcp_server/"], "env": { - "PROWLER_APP_API_KEY": "pk_your_api_key_here", + "PROWLER_APP_API_KEY": "", "PROWLER_API_BASE_URL": "https://api.prowler.com" } } @@ -108,7 +152,7 @@ STDIO mode is only available when running the MCP server locally. - + **Run with Docker image** ```json @@ -121,7 +165,7 @@ STDIO mode is only available when running the MCP server locally. "--rm", "-i", "--env", - "PROWLER_APP_API_KEY=pk_your_api_key_here", + "PROWLER_APP_API_KEY=", "--env", "PROWLER_API_BASE_URL=https://api.prowler.com", "prowlercloud/prowler-mcp" @@ -154,7 +198,7 @@ Prowler MCP Server supports two authentication methods to connect to Prowler Clo Use your Prowler API key directly in the Bearer token: ``` -Authorization: Bearer pk_your_api_key_here +Authorization: Bearer ``` This is the recommended method for most users. diff --git a/docs/getting-started/installation/prowler-app.mdx b/docs/getting-started/installation/prowler-app.mdx index 4c49b8d4f0..f404180791 100644 --- a/docs/getting-started/installation/prowler-app.mdx +++ b/docs/getting-started/installation/prowler-app.mdx @@ -1,5 +1,5 @@ --- -title: 'Installation' +title: "Installation" --- ### Installation @@ -9,41 +9,40 @@ Prowler App supports multiple installation methods based on your environment. Refer to the [Prowler App Tutorial](/user-guide/tutorials/prowler-app) for detailed usage instructions. -Prowler configuration is based in `.env` files. Every version of Prowler can have differences on that file, so, please, use the file that corresponds with that version or repository branch or tag. - + Prowler configuration is based in `.env` files. Every version of Prowler can have differences on that file, so, please, use the file that corresponds with that version or repository branch or tag. + _Requirements_: - * `Docker Compose` installed: https://docs.docker.com/compose/install/. + - `Docker Compose` installed: https://docs.docker.com/compose/install/. _Commands_: - ``` bash - curl -LO https://raw.githubusercontent.com/prowler-cloud/prowler/refs/heads/master/docker-compose.yml - curl -LO https://raw.githubusercontent.com/prowler-cloud/prowler/refs/heads/master/.env + ```bash + VERSION=$(curl -s https://api.github.com/repos/prowler-cloud/prowler/releases/latest | jq -r .tag_name) + curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${VERSION}/docker-compose.yml" + curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${VERSION}/.env" docker compose up -d ``` > Containers are built for `linux/amd64`. If your workstation's architecture is different, please set `DOCKER_DEFAULT_PLATFORM=linux/amd64` in your environment or use the `--platform linux/amd64` flag in the docker command. - _Requirements_: - * `git` installed. - * `poetry` installed: [poetry installation](https://python-poetry.org/docs/#installation). - * `npm` installed: [npm installation](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). - * `Docker Compose` installed: https://docs.docker.com/compose/install/. + - `git` installed. + - `poetry` installed: [poetry installation](https://python-poetry.org/docs/#installation). + - `npm` installed: [npm installation](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). + - `Docker Compose` installed: https://docs.docker.com/compose/install/. - Make sure to have `api/.env` and `ui/.env.local` files with the required environment variables. You can find the required environment variables in the [`api/.env.template`](https://github.com/prowler-cloud/prowler/blob/master/api/.env.example) and [`ui/.env.template`](https://github.com/prowler-cloud/prowler/blob/master/ui/.env.template) files. + Make sure to have `api/.env` and `ui/.env.local` files with the required environment variables. You can find the required environment variables in the [`api/.env.template`](https://github.com/prowler-cloud/prowler/blob/master/api/.env.example) and [`ui/.env.template`](https://github.com/prowler-cloud/prowler/blob/master/ui/.env.template) files. - _Commands to run the API_: - ``` bash + ```bash git clone https://github.com/prowler-cloud/prowler \ cd prowler/api \ poetry install \ @@ -57,17 +56,15 @@ Prowler configuration is based in `.env` files. Every version of Prowler can hav ``` - Starting from Poetry v2.0.0, `poetry shell` has been deprecated in favor of `poetry env activate`. + Starting from Poetry v2.0.0, `poetry shell` has been deprecated in favor of `poetry env activate`. - If your poetry version is below 2.0.0 you must keep using `poetry shell` to activate your environment. - In case you have any doubts, consult the Poetry environment activation guide: https://python-poetry.org/docs/managing-environments/#activating-the-environment + If your poetry version is below 2.0.0 you must keep using `poetry shell` to activate your environment. In case you have any doubts, consult the Poetry environment activation guide: https://python-poetry.org/docs/managing-environments/#activating-the-environment - > Now, you can access the API documentation at http://localhost:8080/api/v1/docs. _Commands to run the API Worker_: - ``` bash + ```bash git clone https://github.com/prowler-cloud/prowler \ cd prowler/api \ poetry install \ @@ -80,7 +77,7 @@ Prowler configuration is based in `.env` files. Every version of Prowler can hav _Commands to run the API Scheduler_: - ``` bash + ```bash git clone https://github.com/prowler-cloud/prowler \ cd prowler/api \ poetry install \ @@ -93,7 +90,7 @@ Prowler configuration is based in `.env` files. Every version of Prowler can hav _Commands to run the UI_: - ``` bash + ```bash git clone https://github.com/prowler-cloud/prowler \ cd prowler/ui \ npm install \ @@ -104,10 +101,11 @@ Prowler configuration is based in `.env` files. Every version of Prowler can hav > Enjoy Prowler App at http://localhost:3000 by signing up with your email and password. - Google and GitHub authentication is only available in [Prowler Cloud](https://prowler.com). + Google and GitHub authentication is only available in [Prowler Cloud](https://prowler.com). + ### Update Prowler App Upgrade Prowler App installation using one of two options: @@ -129,13 +127,12 @@ docker compose pull --policy always The `--policy always` flag ensures that Docker pulls the latest images even if they already exist locally. - -**What Gets Preserved During Upgrade** - -Everything is preserved, nothing will be deleted after the update. + **What Gets Preserved During Upgrade** + Everything is preserved, nothing will be deleted after the update. + ### Troubleshooting If containers don't start, check logs for errors: @@ -155,7 +152,6 @@ docker compose pull docker compose up -d ``` - ### Container versions The available versions of Prowler CLI are the following: @@ -171,6 +167,5 @@ The available versions of Prowler CLI are the following: The container images are available here: - Prowler App: - - - [DockerHub - Prowler UI](https://hub.docker.com/r/prowlercloud/prowler-ui/tags) - - [DockerHub - Prowler API](https://hub.docker.com/r/prowlercloud/prowler-api/tags) + - [DockerHub - Prowler UI](https://hub.docker.com/r/prowlercloud/prowler-ui/tags) + - [DockerHub - Prowler API](https://hub.docker.com/r/prowlercloud/prowler-api/tags) diff --git a/docs/getting-started/products/prowler-mcp.mdx b/docs/getting-started/products/prowler-mcp.mdx index cb97efd5ba..94a5466edb 100644 --- a/docs/getting-started/products/prowler-mcp.mdx +++ b/docs/getting-started/products/prowler-mcp.mdx @@ -5,7 +5,7 @@ title: "Overview" **Prowler MCP Server** brings the entire Prowler ecosystem to AI assistants through the Model Context Protocol (MCP). It enables seamless integration with AI tools like Claude Desktop, Cursor, and other MCP clients, allowing interaction with Prowler's security capabilities through natural language. -**Preview Feature**: This MCP server is currently in preview and under active development. Features and functionality may change. We welcome your feedback—please report any issues on [GitHub](https://github.com/prowler-cloud/prowler/issues) or join our [Slack community](https://goto.prowler.com/slack) to discuss and share your thoughts. +**Preview Feature**: This MCP server is currently under active development. Features and functionality may change. We welcome your feedback—please report any issues on [GitHub](https://github.com/prowler-cloud/prowler/issues) or join our [Slack community](https://goto.prowler.com/slack) to discuss and share your thoughts. ## What is the Model Context Protocol? @@ -42,6 +42,15 @@ Search and retrieve official Prowler documentation: - **Contextual Results**: Get relevant documentation pages with highlighted snippets. - **Document Retrieval**: Access complete markdown content of any documentation file. +## MCP Server Architecture + +The following diagram illustrates the Prowler MCP Server architecture and its integration points: + +Prowler MCP Server Schema +Prowler MCP Server Schema + +The architecture shows how AI assistants connect through the MCP protocol to access Prowler's three main components: Prowler Cloud/App for security operations, Prowler Hub for security knowledge, and Prowler Documentation for guidance and reference. + ## Use Cases The Prowler MCP Server enables powerful workflows through AI assistants: @@ -61,6 +70,59 @@ The Prowler MCP Server enables powerful workflows through AI assistants: - "What authentication methods does Prowler support for Azure?" - "How can I contribute with a new security check to Prowler?" +### Example: Creating a custom dashboard with Prowler extracted data + +In the next example you can see how to create a dashboard using Prowler MCP Server and Claude Desktop. + +**Used Prompt:** +``` +Generate me a security dashboard for the Prowler open source project using live data from Prowler MCP tools. + +REQUIREMENTS: +1. Fetch real-time data from Prowler Findings using MCP tools +2. Create a single self-contained HTML file and display it +3. Dashboard must be production-ready with modern design + +DATA TO FETCH: +Use these MCP tools in this order: +1. Prowler app list providers - To get all available configured provider in the account +2. Prowler app get latest findings - To get findings information, if there are so many you can use the filter_fields to get less information, or pagination to get in different batches +3. For most critical findings you can get more context and remediation with Prowler Hub to get remediations for example + +DESIGN REQUIREMENTS: +- Dark theme (gradient background: #0a0e27 to #131830) +- Card-based layout with glassmorphism effects +- Color scheme: + * Primary green + * Secondary purple +- Modern, professional look +- Animated "LIVE DATA" indicator (pulsing green badge) +- Hover effects on all cards (lift, glow, border color change) +- Responsive grid layout +- Mobile-responsive breakpoints at 768px +- Single HTML file with all CSS and JavaScript embedded +- No external dependencies + +SPECIFIC DETAILS TO INCLUDE: +- Show actual counts from the data (don't hardcode numbers) +- Add timestamp showing when dashboard was generated +- Link to GitHub repository: https://github.com/prowler-cloud/prowler + +OUTPUT: +Generate the complete HTML file and display it +``` + +**Video:** + + + ## Deployment Options Prowler MCP Server can be used in three ways: diff --git a/docs/images/providers/add-iac-repo.png b/docs/images/providers/add-iac-repo.png new file mode 100644 index 0000000000..31981e7fd1 Binary files /dev/null and b/docs/images/providers/add-iac-repo.png differ diff --git a/docs/images/providers/iac-authentication.png b/docs/images/providers/iac-authentication.png new file mode 100644 index 0000000000..c6caad4ae1 Binary files /dev/null and b/docs/images/providers/iac-authentication.png differ diff --git a/docs/images/providers/iac-verify-connection.png b/docs/images/providers/iac-verify-connection.png new file mode 100644 index 0000000000..bc918f6c24 Binary files /dev/null and b/docs/images/providers/iac-verify-connection.png differ diff --git a/docs/images/providers/select-iac.png b/docs/images/providers/select-iac.png new file mode 100644 index 0000000000..1dc474cbe8 Binary files /dev/null and b/docs/images/providers/select-iac.png differ diff --git a/docs/images/prowler_mcp_schema_dark.png b/docs/images/prowler_mcp_schema_dark.png new file mode 100644 index 0000000000..7771557601 Binary files /dev/null and b/docs/images/prowler_mcp_schema_dark.png differ diff --git a/docs/images/prowler_mcp_schema_light.png b/docs/images/prowler_mcp_schema_light.png new file mode 100644 index 0000000000..c542d84ed2 Binary files /dev/null and b/docs/images/prowler_mcp_schema_light.png differ diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 811dc794b7..603bd1afbd 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -28,12 +28,12 @@ The supported providers right now are: | [AWS](/user-guide/providers/aws/getting-started-aws) | Official | UI, API, CLI | | [Azure](/user-guide/providers/azure/getting-started-azure) | Official | UI, API, CLI | | [Google Cloud](/user-guide/providers/gcp/getting-started-gcp) | Official | UI, API, CLI | -| [Kubernetes](/user-guide/providers/kubernetes/in-cluster) | Official | UI, API, CLI | +| [Kubernetes](/user-guide/providers/kubernetes/getting-started-k8s) | Official | UI, API, CLI | | [M365](/user-guide/providers/microsoft365/getting-started-m365) | Official | UI, API, CLI | | [Github](/user-guide/providers/github/getting-started-github) | Official | UI, API, CLI | -| [Oracle Cloud](/user-guide/providers/oci/getting-started-oci) | Official | CLI | -| [Infra as Code](/user-guide/providers/iac/getting-started-iac) | Official | CLI | -| [MongoDB Atlas](/user-guide/providers/mongodbatlas/getting-started-mongodbatlas) | Official | CLI | +| [Oracle Cloud](/user-guide/providers/oci/getting-started-oci) | Official | UI, API, CLI | +| [Infra as Code](/user-guide/providers/iac/getting-started-iac) | Official | UI, API, CLI | +| [MongoDB Atlas](/user-guide/providers/mongodbatlas/getting-started-mongodbatlas) | Official | CLI, API | | [LLM](/user-guide/providers/llm/getting-started-llm) | Official | CLI | | **NHN** | Unofficial | CLI | @@ -48,4 +48,4 @@ For more information about the checks and compliance of each provider visit [Pro Interested in contributing to Prowler? - \ No newline at end of file + diff --git a/docs/user-guide/cli/tutorials/compliance.mdx b/docs/user-guide/cli/tutorials/compliance.mdx index da4d15e5c7..8754be6237 100644 --- a/docs/user-guide/cli/tutorials/compliance.mdx +++ b/docs/user-guide/cli/tutorials/compliance.mdx @@ -24,7 +24,7 @@ Standard results will be shown and additionally the framework information as the **If Prowler can't find a resource related with a check from a compliance requirement, this requirement won't appear on the output** -## List Available Compliance Frameworks +## List Available Compliance Frameworks To see which compliance frameworks are covered by Prowler, use the `--list-compliance` option: @@ -34,7 +34,7 @@ prowler --list-compliance Or you can visit [Prowler Hub](https://hub.prowler.com/compliance). -## List Requirements of Compliance Frameworks +## List Requirements of Compliance Frameworks To list requirements for a compliance framework, use the `--list-compliance-requirements` option: ```sh diff --git a/docs/user-guide/cli/tutorials/configuration_file.mdx b/docs/user-guide/cli/tutorials/configuration_file.mdx index 819c17e9ca..c2f886b3f3 100644 --- a/docs/user-guide/cli/tutorials/configuration_file.mdx +++ b/docs/user-guide/cli/tutorials/configuration_file.mdx @@ -94,7 +94,7 @@ The following list includes all the Azure checks with configurable variables tha ### Configurable Checks -## Kubernetes +## Kubernetes ### Configurable Checks The following list includes all the Kubernetes checks with configurable variables that can be changed in the configuration yaml file: diff --git a/docs/user-guide/cli/tutorials/integrations.mdx b/docs/user-guide/cli/tutorials/integrations.mdx index 2667b81c50..f3e698d799 100644 --- a/docs/user-guide/cli/tutorials/integrations.mdx +++ b/docs/user-guide/cli/tutorials/integrations.mdx @@ -2,7 +2,7 @@ title: 'Integrations' --- -## Integration with Slack +## Integration with Slack Prowler can be integrated with [Slack](https://slack.com/) to send a summary of the execution having configured a Slack APP in your channel with the following command: diff --git a/docs/user-guide/cli/tutorials/misc.mdx b/docs/user-guide/cli/tutorials/misc.mdx index 9921f89032..0940508ed7 100644 --- a/docs/user-guide/cli/tutorials/misc.mdx +++ b/docs/user-guide/cli/tutorials/misc.mdx @@ -14,7 +14,7 @@ prowler -V/-v/--version Prowler provides various execution settings. -### Verbose Execution +### Verbose Execution To enable verbose mode in Prowler, similar to Version 2, use: @@ -54,7 +54,7 @@ To run Prowler without color formatting: prowler --no-color ``` -### Checks in Prowler +### Checks in Prowler Prowler provides various security checks per cloud provider. Use the following options to list, execute, or exclude specific checks: @@ -96,7 +96,7 @@ prowler -e/--excluded-checks ec2 rds prowler -C/--checks-file .json ``` -## Custom Checks in Prowler +## Custom Checks in Prowler Prowler supports custom security checks, allowing users to define their own logic. diff --git a/docs/user-guide/cli/tutorials/mutelist.mdx b/docs/user-guide/cli/tutorials/mutelist.mdx index 4bb7524fa6..79292f49b3 100644 --- a/docs/user-guide/cli/tutorials/mutelist.mdx +++ b/docs/user-guide/cli/tutorials/mutelist.mdx @@ -19,15 +19,45 @@ The Mutelist option works in combination with other filtering mechanisms and mod ## How the Mutelist Works -The **Mutelist** uses both "AND" and "OR" logic to determine which resources, checks, regions, and tags should be muted. For each check, the Mutelist evaluates whether the account, region, and resource match the specified criteria using "AND" logic. If tags are specified, the Mutelist can apply either "AND" or "OR" logic. +The **Mutelist** uses **AND logic** to evaluate whether a finding should be muted. For a finding to be muted, **ALL** of the following conditions must match: -If any of the criteria do not match, the check is not muted. +- **Account** matches (exact match or `*`) +- **Check** matches (exact match, regex pattern, or `*`) +- **Region** matches (exact match, regex pattern, or `*`) +- **Resource** matches (exact match, regex pattern, or `*`) +- **Tags** match (if specified) + +If **any** of these criteria do not match, the finding is **not muted**. + +### Tag Matching Logic + +Tags have special matching behavior: + +- **Multiple tags in the list = AND logic**: ALL tags must be present on the resource + ```yaml + Tags: + - "environment=dev" + - "team=backend" # BOTH tags required + ``` + +- **Regex alternation within a single tag = OR logic**: Use the pipe operator `|` for OR + ```yaml + Tags: + - "environment=dev|environment=stg" # Matches EITHER dev OR stg + ``` + +- **Complex tag patterns**: Combine AND and OR using regex + ```yaml + Tags: + - "team=backend" # Required + - "environment=dev|environment=stg" # AND (dev OR stg) + ``` Remember that mutelist can be used with regular expressions. -## Mutelist Specification +## Mutelist Specification - For Azure provider, the Account ID is the Subscription Name and the Region is the Location. @@ -40,9 +70,10 @@ The Mutelist file uses the [YAML](https://en.wikipedia.org/wiki/YAML) format wit ```yaml ### Account, Check and/or Region can be * to apply for all the cases. ### Resources and tags are lists that can have either Regex or Keywords. -### Tags is an optional list that matches on tuples of 'key=value' and are "ANDed" together. -### Use an alternation Regex to match one of multiple tags with "ORed" logic. -### For each check you can except Accounts, Regions, Resources and/or Tags. +### Multiple tags in the list are "ANDed" together (ALL must match). +### Use regex alternation (|) within a single tag for "OR" logic (e.g., "env=dev|env=stg"). +### For each check you can use Exceptions to unmute specific Accounts, Regions, Resources and/or Tags. +### All conditions (Account, Check, Region, Resource, Tags) are ANDed together. ########################### MUTELIST EXAMPLE ########################### Mutelist: Accounts: @@ -148,11 +179,11 @@ Mutelist: | Field| Description| Logic |----------|----------|---------- -| `account_id`| Use `*` to apply the mutelist to all accounts.| `ANDed` -| `check_name`| The name of the Prowler check. Use `*` to apply the mutelist to all checks, or `service_*` to apply it to all service's checks.| `ANDed` -| `region`| The region identifier. Use `*` to apply the mutelist to all regions.| `ANDed` -| `resource`| The resource identifier. Use `*` to apply the mutelist to all resources.| `ANDed` -| `tag`| The tag value.| `ORed` +| `account_id`| Use `*` to apply the mutelist to all accounts. Supports exact match or wildcard.| `AND` (with other fields) +| `check_name`| The name of the Prowler check. Use `*` to apply the mutelist to all checks, or `service_*` to apply it to all service's checks. Supports regex patterns.| `AND` (with other fields) +| `region`| The region identifier. Use `*` to apply the mutelist to all regions. Supports regex patterns.| `AND` (with other fields) +| `resource`| The resource identifier. Use `*` to apply the mutelist to all resources. Supports regex patterns.| `AND` (with other fields) +| `tags`| List of tag patterns in `key=value` format. **Multiple tags = AND** (all must match). **Regex alternation within single tag = OR** (use `tag1\|tag2`).| `AND` between tags, `OR` within regex ### Description @@ -173,6 +204,68 @@ Replace `` with the appropriate provider name. - The Mutelist can be used in combination with other Prowler options, such as the `--service` or `--checks` option, to further customize the scanning process. - Make sure to review and update the Mutelist regularly to ensure it reflects the desired exclusions and remains up to date with your infrastructure. +## Current Limitations and Workarounds + +### Limitation: No OR Logic Between Different Rule Sets + +The current Mutelist schema **does not support OR logic** between different condition sets. Each check can have only **one rule object**, and all conditions are **ANDed** together. + +**Example of unsupported scenario:** +```yaml +# ❌ INVALID: Cannot have multiple rule blocks for the same check +Accounts: + "*": + Checks: + "*": # Rule 1 + Regions: ["eu-west-1", "us-west-2"] + Resources: ["*"] + "*": # Rule 2 - This will OVERWRITE Rule 1 (YAML duplicate key) + Regions: ["us-east-1"] + Tags: ["environment=dev"] +``` + +**Workaround: Use multiple scans with different mutelists** + +For complex scenarios requiring OR logic, run separate scans: + +```bash +# Scan 1: Mute findings in non-critical regions +prowler aws --mutelist-file mutelist_noncritical.yaml + +# Scan 2: Mute dev/stg in critical regions +prowler aws --mutelist-file mutelist_critical.yaml --regions us-east-1,sa-east-1 +``` + +Then merge the outputs in your reporting pipeline. + +### Limitation: Cannot Negate Regions + +You cannot express "all regions **except** X and Y". You must explicitly list all regions you want to mute. + +**Workaround:** +```yaml +# Must enumerate all unwanted regions +Accounts: + "*": + Checks: + "*": + Regions: + - "af-south-1" + - "ap-east-1" + # ... list all regions EXCEPT the ones you want to monitor + Resources: ["*"] +``` + +### Best Practices + +1. **Use regex patterns for flexibility**: Instead of listing multiple resources, use regex patterns like `"dev-.*"` or `"test-instance-[0-9]+"` + +2. **Combine tag OR logic with regex**: Use `"environment=dev|environment=stg|environment=test"` instead of multiple tag entries + +3. **Be specific with exceptions**: Use the `Exceptions` field to unmute specific resources within a broader muting rule + +4. **Test your mutelist**: Run Prowler with `--output-modes json` and verify that the expected findings are muted + ## AWS Mutelist ### Muting specific AWS regions diff --git a/docs/user-guide/cli/tutorials/parallel-execution.mdx b/docs/user-guide/cli/tutorials/parallel-execution.mdx index b364483ace..93b8ef381b 100644 --- a/docs/user-guide/cli/tutorials/parallel-execution.mdx +++ b/docs/user-guide/cli/tutorials/parallel-execution.mdx @@ -10,7 +10,7 @@ This can help for really large accounts, but please be aware of AWS API rate lim 2. **API Rate Limits**: Most of the rate limits in AWS are applied at the API level. Each API call to an AWS service counts towards the rate limit for that service. 3. **Throttling Responses**: When you exceed the rate limit for a service, AWS responds with a throttling error. In AWS SDKs, these are typically represented as `ThrottlingException` or `RateLimitExceeded` errors. -For information on Prowler's retrier configuration please refer to this [page](https://docs.prowler.cloud/en/latest/tutorials/aws/boto3-configuration/). +For information on Prowler's retrier configuration please refer to this [page](https://docs.prowler.com/user-guide/providers/aws/boto3-configuration/). You might need to increase the `--aws-retries-max-attempts` parameter from the default value of 3. The retrier follows an exponential backoff strategy. diff --git a/docs/user-guide/cli/tutorials/quick-inventory.mdx b/docs/user-guide/cli/tutorials/quick-inventory.mdx index 33ce719d19..08519c305c 100644 --- a/docs/user-guide/cli/tutorials/quick-inventory.mdx +++ b/docs/user-guide/cli/tutorials/quick-inventory.mdx @@ -24,6 +24,6 @@ By default, it extracts resources from all the regions, you could use `-f`/`--fi ![Quick Inventory Example](/images/quick-inventory.jpg) -## Objections +## Objections The inventorying process is carried out with `resourcegroupstaggingapi` calls, which means that only resources they have or have had tags will appear (except for the IAM and S3 resources which are done with Boto3 API calls). diff --git a/docs/user-guide/cli/tutorials/reporting.mdx b/docs/user-guide/cli/tutorials/reporting.mdx index 2e2fed88c0..65eec93263 100644 --- a/docs/user-guide/cli/tutorials/reporting.mdx +++ b/docs/user-guide/cli/tutorials/reporting.mdx @@ -22,7 +22,7 @@ prowler --output-formats json-asff All compliance-related reports are automatically generated when Prowler is executed. These outputs are stored in the `/output/compliance` directory. -## Custom Output Flags +## Custom Output Flags By default, Prowler creates a file inside the `output` directory named: `prowler-output-ACCOUNT_NUM-OUTPUT_DATE.format`. @@ -53,13 +53,13 @@ Both flags can be used simultaneously to provide a custom directory and filename By default, the timestamp format of the output files is ISO 8601. This can be changed with the flag `--unix-timestamp` generating the timestamp fields in pure unix timestamp format. -## Supported Output Formats +## Supported Output Formats Prowler natively supports the following reporting output formats: - CSV - JSON-OCSF -- JSON-ASFF +- JSON-ASFF (AWS only) - HTML Hereunder is the structure for each of the supported report formats by Prowler: @@ -285,10 +285,10 @@ The JSON-OCSF output format implements the [Detection Finding](https://schema.oc Each finding is a `json` object within a list. -### JSON-ASFF +### JSON-ASFF (AWS Only) -Only available when using `--security-hub` or `--output-formats json-asff` +Only available when using `--security-hub` or `--output-formats json-asff` with the AWS provider. The following code is an example output of the [JSON-ASFF](https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format-syntax.html) format: diff --git a/docs/user-guide/cli/tutorials/scan-unused-services.mdx b/docs/user-guide/cli/tutorials/scan-unused-services.mdx index 32756cdce0..6d675e159f 100644 --- a/docs/user-guide/cli/tutorials/scan-unused-services.mdx +++ b/docs/user-guide/cli/tutorials/scan-unused-services.mdx @@ -14,7 +14,7 @@ prowler --scan-unused-services ## Services Ignored -### AWS +### AWS #### ACM (AWS Certificate Manager) @@ -22,21 +22,21 @@ Certificates stored in ACM without active usage in AWS resources are excluded. B - `acm_certificates_expiration_check` -#### Athena +#### Athena Upon AWS account creation, Athena provisions a default primary workgroup for the user. Prowler verifies if this workgroup is enabled and used by checking for queries within the last 45 days. If Athena is unused, findings related to its checks will not appear. - `athena_workgroup_encryption` - `athena_workgroup_enforce_configuration` -#### AWS CloudTrail +#### AWS CloudTrail AWS CloudTrail should have at least one trail with a data event to record all S3 object-level API operations. Before flagging this issue, Prowler verifies if S3 buckets exist in the account. - `cloudtrail_s3_dataevents_read_enabled` - `cloudtrail_s3_dataevents_write_enabled` -#### AWS Elastic Compute Cloud (EC2) +#### AWS Elastic Compute Cloud (EC2) If Amazon Elastic Block Store (EBS) default encyption is not enabled, sensitive data at rest will remain unprotected in EC2. However, Prowler will only generate a finding if EBS volumes exist where default encryption could be enforced. @@ -56,7 +56,7 @@ Prowler scans only attached security groups to report vulnerabilities in activel - `ec2_networkacl_allow_ingress_X_port` -#### AWS Glue +#### AWS Glue AWS Glue best practices recommend encrypting metadata and connection passwords in Data Catalogs. @@ -71,7 +71,7 @@ Amazon Inspector is a vulnerability discovery service that automates continuous - `inspector2_is_enabled` -#### Amazon Macie +#### Amazon Macie Amazon Macie leverages machine learning to automatically discover, classify, and protect sensitive data in S3 buckets. Prowler only generates findings if Macie is disabled and there are S3 buckets in the AWS account. @@ -83,7 +83,7 @@ A network firewall is essential for monitoring and controlling traffic within a - `networkfirewall_in_all_vpc` -#### Amazon S3 +#### Amazon S3 To prevent unintended data exposure: @@ -91,7 +91,7 @@ Public Access Block should be enabled at the account level. Prowler only checks - `s3_account_level_public_access_blocks` -#### Virtual Private Cloud (VPC) +#### Virtual Private Cloud (VPC) VPC settings directly impact network security and availability. diff --git a/docs/user-guide/compliance/tutorials/threatscore.mdx b/docs/user-guide/compliance/tutorials/threatscore.mdx index 0a2f2f77b9..c67b98c8c7 100644 --- a/docs/user-guide/compliance/tutorials/threatscore.mdx +++ b/docs/user-guide/compliance/tutorials/threatscore.mdx @@ -2,6 +2,10 @@ title: "Prowler ThreatScore Documentation" --- + + +This feature is only available in Prowler Cloud/App. + ## Introduction The **Prowler ThreatScore** is a comprehensive compliance scoring system that provides a unified metric for assessing your organization's security posture across compliance frameworks. It aggregates findings from individual security checks into a single, normalized score ranging from 0 to 100. diff --git a/docs/user-guide/providers/aws/getting-started-aws.mdx b/docs/user-guide/providers/aws/getting-started-aws.mdx index e6cc636a42..f502981b08 100644 --- a/docs/user-guide/providers/aws/getting-started-aws.mdx +++ b/docs/user-guide/providers/aws/getting-started-aws.mdx @@ -84,6 +84,17 @@ For detailed instructions on how to create the role, see [Authentication > Assum ![Next button in Prowler Cloud](/images/providers/next-button-prowler-cloud.png) ![Launch Scan](/images/providers/launch-scan-button-prowler-cloud.png) + +Check if your AWS Security Token Service (STS) has the EU (Ireland) endpoint active. If not, we will not be able to connect to your AWS account. + +If that is the case your STS configuration may look like this: + +AWS Role + +To solve this issue, please activate the EU (Ireland) STS endpoint. + + + --- #### Credentials (Static Access Keys) diff --git a/docs/user-guide/providers/azure/getting-started-azure.mdx b/docs/user-guide/providers/azure/getting-started-azure.mdx index f9f37b9f9b..a5299c614b 100644 --- a/docs/user-guide/providers/azure/getting-started-azure.mdx +++ b/docs/user-guide/providers/azure/getting-started-azure.mdx @@ -53,7 +53,8 @@ For detailed instructions on how to create the Service Principal and configure p ### Step 3: Add Credentials to Prowler App -Having completed the [Service Principal setup from the Authentication guide](/user-guide/providers/azure/authentication#service-principal-application-authentication-recommended): +For Azure, Prowler App uses a service principal application to authenticate. For more information about the process of creating and adding permissions to a service principal refer to this [section](/user-guide/providers/azure/authentication). When you finish creating and adding the [Entra](/user-guide/providers/azure/create-prowler-service-principal#assigning-proper-permissions) and [Subscription](/user-guide/providers/azure/subscriptions) scope permissions to the service principal, enter the `Tenant ID`, `Client ID` and `Client Secret` of the service principal application. + 1. Go to your App Registration overview and copy the `Client ID` and `Tenant ID` diff --git a/docs/user-guide/providers/gcp/authentication.mdx b/docs/user-guide/providers/gcp/authentication.mdx index 7eea8ac5dd..9447d73a20 100644 --- a/docs/user-guide/providers/gcp/authentication.mdx +++ b/docs/user-guide/providers/gcp/authentication.mdx @@ -14,7 +14,10 @@ Prowler for Google Cloud supports multiple authentication methods. To use a spec Prowler for Google Cloud requires the following permissions: ### IAM Roles -- **Reader (`roles/reader`)** – Must be granted at the **project, folder, or organization** level to allow scanning of target projects. +- **Viewer (`roles/viewer`)** – Must be granted at the **project, folder, or organization** level to allow scanning of target projects. +- **Service Usage Consumer (`roles/serviceusage.serviceUsageConsumer`)** IAM Role – Required for resource scanning. +- **Custom `ProwlerRole`** – Include granular permissions that are not included in the Viewer role: + - `storage.buckets.getIamPolicy` ### Project-Level Settings @@ -106,18 +109,46 @@ prowler gcp --project-ids This method uses a service account with a downloaded key file for authentication. -### Create Service Account and Key +### Step 1: Create ProwlerRole -1. Go to the [Service Accounts page](https://console.cloud.google.com/iam-admin/serviceaccounts) in the GCP Console -2. Click "Create Service Account" -3. Fill in the service account details and click "Create and Continue" -4. Grant the service account the "Reader" role -5. Click "Done" -6. Find your service account in the list and click on it -7. Go to the "Keys" tab -8. Click "Add Key" > "Create new key" -9. Select "JSON" and click "Create" -10. Save the downloaded key file securely +To keep permissions focused: +1. Create a custom role named **ProwlerRole** that explicitly includes the permissions your compliance team approves. Click **Create role**, set the title to *ProwlerRole*, keep the ID readable (for example, `prowler_role`) +2. Add the required permission `storage.buckets.getIamPolicy` (the permission highlighted in the screenshots). To make it easier, filter the permissions by `Storage Admin` role. + +![Create a custom Prowler role](/user-guide/providers/gcp/img/roles-section.png) + +![Sample permissions for a custom Prowler role](/user-guide/providers/gcp/img/prowler-role.png) + +### Step 2: Create the Service Account + +1. Navigate to **IAM & Admin > Service Accounts** and make sure the correct project is selected. + + ![Service accounts landing page](/user-guide/providers/gcp/img/service-account-page.png) + +2. Select **Create service account**, provide a name, ID, and a short description that states the purpose (for example, “Service account to execute Prowler”), then click **Create and continue**. + + ![Create service account wizard](/user-guide/providers/gcp/img/create-service-account.png) + +3. Assign the roles you prepared earlier: + - **ProwlerRole** for `cloudstorage` service checks. + - **Viewer** for broad read-only visibility. + - **Service Usage Consumer** so Prowler can inspect API states. + + ![Assign roles to the service account](/user-guide/providers/gcp/img/service-account-permissions.png) + +4. Continue through the wizard and finish. No principals need to be granted access in step 3 unless you want other identities to impersonate this account. + +### Step 3: Generate a JSON Key + +1. Open the newly created service account, move to the **Keys** tab, and choose **Add key > Create new key**. + + ![Add a new key to the service account](/user-guide/providers/gcp/img/create-new-key.png) + +2. Select **JSON** as the key type and click **Create**. The browser downloads the file exactly once. + + ![Select JSON as the key type](/user-guide/providers/gcp/img/json-key.png) + +3. Once created, make sure to store the Key securely. ### Using with Prowler CLI diff --git a/docs/user-guide/providers/gcp/getting-started-gcp.mdx b/docs/user-guide/providers/gcp/getting-started-gcp.mdx index 218272793a..c2f8a9363c 100644 --- a/docs/user-guide/providers/gcp/getting-started-gcp.mdx +++ b/docs/user-guide/providers/gcp/getting-started-gcp.mdx @@ -32,35 +32,45 @@ title: 'Getting Started With GCP on Prowler' ### Step 3: Set Up GCP Authentication -Choose the preferred authentication mode before proceeding: +For Google Cloud, first enter your `GCP Project ID` and then select the authentication method you want to use: -**User Credentials (Application Default Credentials)** +- **Service Account Authentication** (**Recommended**) + * Authenticates as a service identity + * Stable and auditable + * Recommended for production +- **Application Default Credentials** + * Quick scan as current user + * Uses Google Cloud CLI authentication + * Credentials may time out -* Quick scan as current user -* Uses Google Cloud CLI authentication -* Credentials may time out +**Service Account Authentication** is the recommended authentication method for automated systems and machine-to-machine interactions, like Prowler. For detailed information about this, refer to the [Google Cloud documentation](https://cloud.google.com/iam/docs/service-account-overview). -**Service Account Key File** +GCP Authentication Methods -* Authenticates as a service identity -* Stable and auditable -* Recommended for production + + + First of all, in the same project that you selected in the previous step, you need to create a service account and then generate a key in JSON format for it. For more information about this, you can follow the next Google Cloud documentation tutorials: -For detailed instructions on how to set up authentication, see [Authentication](/user-guide/providers/gcp/authentication). + - [Create a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) + - [Generate a key for a service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) -6. Once credentials are configured, return to Prowler App and enter the required values: + GCP Service Account Credentials + + + 1. Run the following command in your terminal to authenticate with GCP: - For "Service Account Key": + ```bash + gcloud auth application-default login + ``` - - `Service Account Key JSON` + 2. Once authenticated, get the `Client ID`, `Client Secret` and `Refresh Token` from `~/.config/gcloud/application_default_credentials`. - For "Application Default Credentials": + 3. Paste the `Client ID`, `Client Secret` and `Refresh Token` into Prowler App. - - `client_id` - - `client_secret` - - `refresh_token` + GCP Credentials + + - ![Enter the Credentials](/images/providers/enter-credentials-prowler-cloud.png) 7. Click "Next", then "Launch Scan" diff --git a/docs/user-guide/providers/gcp/img/create-new-key.png b/docs/user-guide/providers/gcp/img/create-new-key.png new file mode 100644 index 0000000000..db925004d5 Binary files /dev/null and b/docs/user-guide/providers/gcp/img/create-new-key.png differ diff --git a/docs/user-guide/providers/gcp/img/create-service-account.png b/docs/user-guide/providers/gcp/img/create-service-account.png new file mode 100644 index 0000000000..5c6bc516ca Binary files /dev/null and b/docs/user-guide/providers/gcp/img/create-service-account.png differ diff --git a/docs/user-guide/providers/gcp/img/json-key.png b/docs/user-guide/providers/gcp/img/json-key.png new file mode 100644 index 0000000000..1466000b7e Binary files /dev/null and b/docs/user-guide/providers/gcp/img/json-key.png differ diff --git a/docs/user-guide/providers/gcp/img/prowler-role.png b/docs/user-guide/providers/gcp/img/prowler-role.png new file mode 100644 index 0000000000..99da2f6edb Binary files /dev/null and b/docs/user-guide/providers/gcp/img/prowler-role.png differ diff --git a/docs/user-guide/providers/gcp/img/roles-section.png b/docs/user-guide/providers/gcp/img/roles-section.png new file mode 100644 index 0000000000..dcf068d967 Binary files /dev/null and b/docs/user-guide/providers/gcp/img/roles-section.png differ diff --git a/docs/user-guide/providers/gcp/img/service-account-page.png b/docs/user-guide/providers/gcp/img/service-account-page.png new file mode 100644 index 0000000000..588bb54816 Binary files /dev/null and b/docs/user-guide/providers/gcp/img/service-account-page.png differ diff --git a/docs/user-guide/providers/gcp/img/service-account-permissions.png b/docs/user-guide/providers/gcp/img/service-account-permissions.png new file mode 100644 index 0000000000..a4fce5e408 Binary files /dev/null and b/docs/user-guide/providers/gcp/img/service-account-permissions.png differ diff --git a/docs/user-guide/providers/iac/getting-started-iac.mdx b/docs/user-guide/providers/iac/getting-started-iac.mdx index a3efb0f2df..315afafed4 100644 --- a/docs/user-guide/providers/iac/getting-started-iac.mdx +++ b/docs/user-guide/providers/iac/getting-started-iac.mdx @@ -1,6 +1,7 @@ --- title: "Getting Started with the IaC Provider" --- +import { VersionBadge } from "/snippets/version-badge.mdx" Prowler's Infrastructure as Code (IaC) provider enables scanning of local or remote infrastructure code for security and compliance issues using [Trivy](https://trivy.dev/). This provider supports a wide range of IaC frameworks, allowing assessment of code before deployment. @@ -22,8 +23,46 @@ The IaC provider leverages [Trivy](https://trivy.dev/latest/docs/scanner/vulnera - Mutelist logic ([filtering](https://trivy.dev/latest/docs/configuration/filtering/)) is handled by Trivy, not Prowler. - Results are output in the same formats as other Prowler providers (CSV, JSON, HTML, etc.). +## Prowler App + + + +### Step 1: Access Prowler Cloud/App + +1. Navigate to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](/user-guide/tutorials/prowler-app) +2. Go to "Configuration" > "Cloud Providers" + + ![Cloud Providers Page](/images/prowler-app/cloud-providers-page.png) + +3. Click "Add Cloud Provider" + + ![Add a Cloud Provider](/images/prowler-app/add-cloud-provider.png) + +4. Select "Infrastructure as Code" + + ![Select Infrastructure as Code](/images/providers/select-iac.png) + +5. Add the Repository URL and an optional alias, then click "Next" + + ![Add IaC Repository URL](/images/providers/add-iac-repo.png) + +### Step 2: Enter Authentication Details + +6. Optionally provide the [authentication](/user-guide/providers/iac/authentication) details for private repositories, then click "Next" + + ![IaC Authentication](/images/providers/iac-authentication.png) + +### Step 3: Verify Connection & Start Scan + +7. Review the provider configuration and click "Launch scan" to initiate the scan + + ![Verify Connection & Start Scan](/images/providers/iac-verify-connection.png) + + ## Prowler CLI + + ### Usage Use the `iac` argument to run Prowler with the IaC provider. Specify the directory or repository to scan, frameworks to include, and paths to exclude. diff --git a/docs/user-guide/providers/kubernetes/getting-started-k8s.mdx b/docs/user-guide/providers/kubernetes/getting-started-k8s.mdx new file mode 100644 index 0000000000..ae70c06263 --- /dev/null +++ b/docs/user-guide/providers/kubernetes/getting-started-k8s.mdx @@ -0,0 +1,131 @@ +--- +title: 'Getting Started with Kubernetes' +--- + +## Prowler App + +### Step 1: Access Prowler Cloud/App + +1. Navigate to [Prowler Cloud](https://cloud.prowler.com/) or launch [Prowler App](/user-guide/tutorials/prowler-app) +2. Go to "Configuration" > "Cloud Providers" + + ![Cloud Providers Page](/images/prowler-app/cloud-providers-page.png) + +3. Click "Add Cloud Provider" + + ![Add a Cloud Provider](/images/prowler-app/add-cloud-provider.png) + +4. Select "Kubernetes" + +5. Enter your Kubernetes Cluster context from your kubeconfig file and optionally provide a friendly alias + +### Step 2: Configure Kubernetes Authentication + +For Kubernetes, Prowler App uses a `kubeconfig` file to authenticate. Paste the contents of your `kubeconfig` file into the `Kubeconfig content` field. + +By default, the `kubeconfig` file is located at `~/.kube/config`. + +![Kubernetes Credentials](/images/kubernetes-credentials.png) + +### Step 3: Additional Setup for EKS, GKE, AKS, or External Clusters + +If you are adding an **EKS**, **GKE**, **AKS** or external cluster, follow these additional steps to ensure proper authentication: + +**Make sure your cluster allows traffic from the Prowler Cloud IP address `52.48.254.174/32`** + +1. Apply the necessary Kubernetes resources to your EKS, GKE, AKS or external cluster (you can find the files in the [`kubernetes` directory of the Prowler repository](https://github.com/prowler-cloud/prowler/tree/master/kubernetes)): + + ```console + kubectl apply -f kubernetes/prowler-sa.yaml + kubectl apply -f kubernetes/prowler-role.yaml + kubectl apply -f kubernetes/prowler-rolebinding.yaml + ``` + +2. Generate a long-lived token for authentication: + + ```console + kubectl create token prowler-sa -n prowler-ns --duration=0 + ``` + + - **Security Note:** The `--duration=0` option generates a non-expiring token, which may pose a security risk if not managed properly. Users should decide on an appropriate expiration time based on their security policies. If a limited-time token is preferred, set `--duration= ## Mutelist Ready To Use Examples -Below are examples for different cloud providers supported by Prowler App. Check how the mutelist works [here](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/mutelist/#how-the-mutelist-works). +Below are examples for different cloud providers supported by Prowler App. Check how the mutelist works [here](/user-guide/cli/tutorials/mutelist#how-the-mutelist-works). ### AWS Provider diff --git a/docs/user-guide/tutorials/prowler-app-sso.mdx b/docs/user-guide/tutorials/prowler-app-sso.mdx index f56336f415..fa062100d1 100644 --- a/docs/user-guide/tutorials/prowler-app-sso.mdx +++ b/docs/user-guide/tutorials/prowler-app-sso.mdx @@ -10,9 +10,9 @@ This guide provides comprehensive instructions to configure SAML-based Single Si This document is divided into two main sections: -- **User Guide**: For organization administrators to configure SAML SSO through Prowler App. +- **[User Guide](#user-guide-configuration)**: For organization administrators to configure SAML SSO through Prowler App. -- **Developer and Administrator Guide**: For developers and system administrators running self-hosted Prowler App instances, providing technical details on environment configuration, API usage, and testing. +- **[Developer and Administrator Guide](#developer-and-administrator-guide)**: For developers and system administrators running self-hosted Prowler App instances, providing technical details on environment configuration, API usage, and testing. --- @@ -53,58 +53,65 @@ On the profile page, find the "SAML SSO Integration" card and click "Enable" to ![Enable SAML Integration](/images/prowler-app/saml/saml-step-2.png) - -**Choose Your Method** +Next section will explain how to fill the IdP configuration based on your Identity Provider. -**Use Step 3A (Generic Method)** for any SAML 2.0 compliant Identity Provider or when you need custom configuration. +#### Step 3: Configure the Identity Provider (IdP) +Choose a Method: -**Use Step 3B (Okta App Catalog)** if you're using Okta and want a simplified setup process with pre-configured settings. +- Use [**Generic Method**](#generic-method) for any SAML 2.0 compliant Identity Provider or when you need custom configuration. +- Use [**Okta App Catalog**](#okta-app-catalog) if you're using Okta and want a simplified setup process with pre-configured settings. - -#### Step 3A: Configure the Identity Provider (IdP) - Generic + + + Prowler App displays the SAML configuration information needed to configure the IdP. Use this information to create a new SAML application in the IdP. -Prowler App displays the SAML configuration information needed to configure the IdP. Use this information to create a new SAML application in the IdP. + 1. **Assertion Consumer Service (ACS) URL**: The endpoint in Prowler that will receive the SAML assertion from the IdP. + 2. **Audience URI (Entity ID)**: A unique identifier for the Prowler application (Service Provider). -1. **Assertion Consumer Service (ACS) URL**: The endpoint in Prowler that will receive the SAML assertion from the IdP. -2. **Audience URI (Entity ID)**: A unique identifier for the Prowler application (Service Provider). + To configure the IdP, copy the **ACS URL** and **Audience URI** from Prowler App and use them to set up a new SAML application. -To configure the IdP, copy the **ACS URL** and **Audience URI** from Prowler App and use them to set up a new SAML application. + ![IdP configuration](/images/prowler-app/saml/idp_config.png) -![IdP configuration](/images/prowler-app/saml/idp_config.png) + + **IdP Configuration** - -**IdP Configuration** + The exact steps for configuring an IdP vary depending on the provider (Okta, Azure AD, etc.). Please refer to the IdP's documentation for instructions on creating a SAML application. For SSO integration with Azure AD / Entra ID, see our [Entra ID configuration instructions](/user-guide/tutorials/prowler-app-sso-entra). -The exact steps for configuring an IdP vary depending on the provider (Okta, Azure AD, etc.). Please refer to the IdP's documentation for instructions on creating a SAML application. For SSO integration with Azure AD / Entra ID, see our [Entra ID configuration instructions](/user-guide/tutorials/prowler-app-sso-entra). + - -#### Step 3B: Configure Prowler from App Catalog - Okta + + + Instead of creating a custom SAML integration, Okta administrators can configure Prowler Cloud directly from Okta's application catalog. -Instead of creating a custom SAML integration, Okta administrators can configure Prowler Cloud directly from Okta's application catalog: + You can find a walkthrough video [here](https://youtu.be/NjSp5owvCdY). -1. **Access App Catalog**: Navigate to the IdP's application catalog (e.g., [Browse App Catalog](https://www.okta.com/integrations/) in Okta). + 1. **Access App Catalog**: Navigate to the IdP's application catalog (e.g., [Browse App Catalog](https://www.okta.com/integrations/) in Okta). - ![Browse App Catalog](/images/prowler-app/saml/app-catalog-browse.png) + ![Browse App Catalog](/images/prowler-app/saml/app-catalog-browse.png) -2. **Search for Prowler Cloud**: Use the search functionality to find "Prowler Cloud" in the app catalog. The official Prowler Cloud application will appear in the search results. + 2. **Search for Prowler Cloud**: Use the search functionality to find "Prowler Cloud" in the app catalog. The official Prowler Cloud application will appear in the search results. - ![Search for Prowler](/images/prowler-app/saml/app-catalog-browse-prowler.png) + ![Search for Prowler](/images/prowler-app/saml/app-catalog-browse-prowler.png) -3. **Select Prowler Cloud Application**: Click on the Prowler Cloud application from the search results to view its details page. + 3. **Select Prowler Cloud Application**: Click the Prowler Cloud application from the search results to view its details page. - ![Prowler Application Details](/images/prowler-app/saml/app-catalog-browse-prowler-add.png) + ![Prowler Application Details](/images/prowler-app/saml/app-catalog-browse-prowler-add.png) -4. **Add Integration**: Click the "Add Integration" button to begin adding Prowler Cloud to the organization's applications. + 4. **Add Integration**: Click the "Add Integration" button to begin adding Prowler Cloud to the organization's applications. -5. **Configure General Settings**: In the "Add Prowler Cloud" configuration screen, the integration automatically configures the necessary settings. + 5. **Configure General Settings**: In the "Add Prowler Cloud" configuration screen, the integration automatically configures the necessary settings. - ![Add Prowler Configuration](/images/prowler-app/saml/app-catalog-browse-prowler-configure.png) + ![Add Prowler Configuration](/images/prowler-app/saml/app-catalog-browse-prowler-configure.png) -6. **Assign Users**: Navigate to the **Assignments** tab and assign the appropriate users or groups to the Prowler application by clicking "Assign" and selecting "Assign to People" or "Assign to Groups". + 6. **Assign Users**: Navigate to the "Assignments" tab and assign the appropriate users or groups to the Prowler application by clicking "Assign" and selecting "Assign to People" or "Assign to Groups". -With this step, the Okta app catalog configuration is complete. Users can now access Prowler Cloud using either [IdP-initiated](#idp-initiated-sso) or [SP-initiated SSO](#sp-initiated-sso) flows. + With this step, the Okta app catalog configuration is complete. Users can now access Prowler Cloud using either [IdP-initiated](#idp-initiated-sso) or [SP-initiated SSO](#sp-initiated-sso) flows. -**If you used Step 3B (Okta App Catalog)**, jump to [Step 6: Save and Verify Configuration](#step-6-save-and-verify-configuration). + 7. **Download Metadata XML**: Inside the "Sign On" section, go to the "Metadata URL" and download the metadata XML file. + + Jump to [Step 5: Upload IdP Metadata to Prowler](#step-5:-upload-idp-metadata-to-prowler). + + #### Step 4: Configure Attribute Mapping in the IdP @@ -120,7 +127,7 @@ For Prowler App to correctly identify and provision users, configure the IdP to **IdP Attribute Mapping** -Note that the attribute name is just an example and may be different depending on the IdP. For instance, if the IdP provides a 'division' attribute, it can be mapped to 'userType'. +Note that the attribute name is just an example and may be different depending on the IdP. For instance, if the IdP provides a `division` attribute, it can be mapped to `userType`. ![IdP configuration](/images/prowler-app/saml/saml_attribute_statements.png) @@ -156,7 +163,8 @@ Click the "Save" button to complete the setup. The "SAML Integration" card will The exact steps for configuring an IdP vary depending on the provider (Okta, Azure AD, etc.). Please refer to the IdP's documentation for instructions on creating a SAML application. -##### Remove SAML Configuration + +### Remove SAML Configuration SAML SSO can be disabled by removing the existing configuration from the integration panel. ![Remove SAML configuration](/images/prowler-app/saml/saml-step-remove.png) diff --git a/docs/user-guide/tutorials/prowler-app.mdx b/docs/user-guide/tutorials/prowler-app.mdx index dd2074c3b9..52210b01bf 100644 --- a/docs/user-guide/tutorials/prowler-app.mdx +++ b/docs/user-guide/tutorials/prowler-app.mdx @@ -77,208 +77,35 @@ Steps to add a provider: ## **Step 4: Configure the Provider** -Select the cloud provider you want to scan. +Select the cloud provider to scan and configure authentication credentials. Each provider has specific requirements and authentication methods. Select a Provider -Once chosen, enter the Provider UID for authentication: - -- **AWS**: Enter your AWS Account ID. -- **GCP**: Enter your GCP Project ID. -- **Azure**: Enter your Azure Subscription ID. -- **Kubernetes**: Enter your Kubernetes Cluster context of your kubeconfig file. -- **M365**: Enter your M365 Domain ID. - -Optionally, provide a **Provider Alias** for easier identification. Follow the instructions provided to add your credentials: - -### **Step 4.1: AWS Credentials** - -For AWS, enter your `AWS Account ID` and choose one of the following methods to connect: - -#### **Step 4.1.1: IAM Access Keys** - -1. Select `Connect via Credentials`. - - AWS Credentials - -2. Enter your `Access Key ID`, `Secret Access Key` and optionally a `Session Token`: - - AWS Credentials - -#### **Step 4.1.2: IAM Role** - -1. Select `Connect assuming IAM Role`. - - AWS Role - -2. Enter the `Role ARN` and any optional field like the AWS Access Keys to assume the role, the `External ID`, the `Role Session Name` or the `Session Duration`: - - AWS Role - - -Check if your AWS Security Token Service (STS) has the EU (Ireland) endpoint active. If not, we will not be able to connect to your AWS account. - -If that is the case your STS configuration may look like this: - -AWS Role - -To solve this issue, please activate the EU (Ireland) STS endpoint. - - -### **Step 4.2: Azure Credentials**: - -For Azure, Prowler App uses a service principal application to authenticate. For more information about the process of creating and adding permissions to a service principal refer to this [section](/user-guide/providers/azure/authentication). When you finish creating and adding the [Entra](/user-guide/providers/azure/create-prowler-service-principal#assigning-proper-permissions) and [Subscription](/user-guide/providers/azure/subscriptions) scope permissions to the service principal, enter the `Tenant ID`, `Client ID` and `Client Secret` of the service principal application. - -Azure Credentials - ---- -### **Step 4.3: GCP Credentials** - -For Google Cloud, first enter your `GCP Project ID` and then select the authentication method you want to use: - -- **Service Account Authentication** (**Recommended**) -- **Application Default Credentials** - -**Service Account Authentication** is the recommended authentication method for automated systems and machine-to-machine interactions, like Prowler. For detailed information about this, refer to the [Google Cloud documentation](https://cloud.google.com/iam/docs/service-account-overview). - -GCP Authentication Methods - -#### **Step 4.3.1: Service Account Authentication** - -First of all, in the same project that you selected in the previous step, you need to create a service account and then generate a key in JSON format for it. For more information about this, you can follow the next Google Cloud documentation tutorials: - -- [Create a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) -- [Generate a key for a service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) - -GCP Service Account Credentials - -#### **Step 4.3.2: Application Default Credentials** - -1. Run the following command in your terminal to authenticate with GCP: - - ```bash - gcloud auth application-default login - ``` - -2. Once authenticated, get the `Client ID`, `Client Secret` and `Refresh Token` from `~/.config/gcloud/application_default_credentials`. - -3. Paste the `Client ID`, `Client Secret` and `Refresh Token` into Prowler App. - -GCP Credentials - -### **Step 4.4: Kubernetes Credentials**: - -For Kubernetes, Prowler App uses a `kubeconfig` file to authenticate, paste the contents of your `kubeconfig` file into the `Kubeconfig content` field. - -By default, the `kubeconfig` file is located at `~/.kube/config`. - -Kubernetes Credentials - -If you are adding an **EKS**, **GKE**, **AKS** or external cluster, follow these additional steps to ensure proper authentication: - -**Make sure your cluster allow traffic from the Prowler Cloud IP address `52.48.254.174/32`** - -1. Apply the necessary Kubernetes resources to your EKS, GKE, AKS or external cluster (you can find the files in the [`kubernetes` directory of the Prowler repository](https://github.com/prowler-cloud/prowler/tree/master/kubernetes)): - - ```console - kubectl apply -f kubernetes/prowler-sa.yaml - kubectl apply -f kubernetes/prowler-role.yaml - kubectl apply -f kubernetes/prowler-rolebinding.yaml - ``` - -2. Generate a long-lived token for authentication: - - ```console - kubectl create token prowler-sa -n prowler-ns --duration=0 - ``` - - - **Security Note:** The `--duration=0` option generates a non-expiring token, which may pose a security risk if not managed properly. Users should decide on an appropriate expiration time based on their security policies. If a limited-time token is preferred, set `--duration=