mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-06-09 21:04:53 +00:00
Compare commits
138 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 510da0154a | |||
| 9a50dffaa0 | |||
| 88926cc052 | |||
| ed17dc4b09 | |||
| e710ebff1c | |||
| b3caee88e4 | |||
| d9f90e50b8 | |||
| 58efb719fa | |||
| 355b7071aa | |||
| b994b0b14e | |||
| 6c559fbb8d | |||
| b2d74711d9 | |||
| 7e60e8f8da | |||
| 62955dd16b | |||
| 1f7caa6394 | |||
| 662e7e9e18 | |||
| e3013d9918 | |||
| 0ea2f6d67e | |||
| 7692a1d76a | |||
| 1c9afc714e | |||
| 466f1a3d73 | |||
| 061fbaa7bb | |||
| 28b045302f | |||
| 5a2226c02c | |||
| 6f172a5c19 | |||
| a7d180ea5b | |||
| d4bbc8b5ad | |||
| a5bc226f11 | |||
| 3a3d9d6146 | |||
| bcd282d3d0 | |||
| eb7949c884 | |||
| e60a4462e5 | |||
| f7f8747512 | |||
| d573af911d | |||
| cf9beb8234 | |||
| 7f67eac1bf | |||
| a652e28b4a | |||
| 1b17304c4a | |||
| c2cef99b33 | |||
| a769e37615 | |||
| 9d2a8d9108 | |||
| e05519ff9f | |||
| 67b26072f8 | |||
| 2222082631 | |||
| 8b0cb4b981 | |||
| 9422eff8ab | |||
| e3c4368d32 | |||
| 2a641b39c8 | |||
| 02b713572b | |||
| 74251350bc | |||
| 8f745cdbe6 | |||
| 81226cd837 | |||
| a2824f7166 | |||
| edbbd86828 | |||
| c58dad2ca4 | |||
| b4befe3a10 | |||
| d98933c2e7 | |||
| 03dfa3816d | |||
| ad1261ce54 | |||
| 3252f9cf19 | |||
| f1cdf3df15 | |||
| 03ddb8a708 | |||
| 2678c6bc9f | |||
| 48c071297f | |||
| 7e9a16d022 | |||
| 84b388f649 | |||
| 671d0c746c | |||
| 0e4b117161 | |||
| a70bc3c1c7 | |||
| 723d161c63 | |||
| d560020592 | |||
| 00451f8239 | |||
| 329dfdf8e6 | |||
| 4c59af93eb | |||
| 6ca8e726f7 | |||
| 546eb2d85a | |||
| ec3efc94f5 | |||
| 6cffd0d17f | |||
| 528d32601b | |||
| 56b3044aae | |||
| 3a096b1750 | |||
| 6f01041178 | |||
| 13e2ede763 | |||
| c53ddfd532 | |||
| f86bd7b52e | |||
| 6177fc6286 | |||
| 0fd952ae2b | |||
| 74622dd576 | |||
| 4dfa2b9748 | |||
| 435424a680 | |||
| dbbefd0558 | |||
| e55d1d470e | |||
| ab69f3b665 | |||
| a28f4994a8 | |||
| 349611d52d | |||
| 10b965e3c7 | |||
| 554a5024c1 | |||
| 7d03bc5e17 | |||
| c660b35ed6 | |||
| f3bac38a55 | |||
| 61330937f7 | |||
| 5ac978b9a3 | |||
| b4159bd590 | |||
| ef4d45d409 | |||
| f210c26c2f | |||
| a55a736363 | |||
| 9f2af5abc2 | |||
| fee98a58eb | |||
| 1ab8f2f0ac | |||
| e7fbc8b391 | |||
| 8caab36c3f | |||
| 0c4794b060 | |||
| 782e3f238b | |||
| e1c7e0a99b | |||
| 6ef70484c7 | |||
| 621170d9c9 | |||
| b6e2255e9e | |||
| 3ce8eae72f | |||
| 81aa1883fd | |||
| 534dedb608 | |||
| cff1704d7b | |||
| 0ca444895f | |||
| a9865209a1 | |||
| 8526e8b4a6 | |||
| a52ef3c04a | |||
| 1f3f5c2e27 | |||
| 6eebfcfe77 | |||
| 9d8b69abda | |||
| 60aa601e92 | |||
| fc1fd538bd | |||
| 40c1761840 | |||
| 0ab0e8671d | |||
| 7a7c828fc7 | |||
| 5cbe473eb9 | |||
| caf2f61563 | |||
| 9dc4deccb6 | |||
| 476e7d1010 | |||
| 1a450aaa70 |
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "prowler-plugins",
|
||||
"description": "Prowler Cloud Security for Claude Code",
|
||||
"owner": {
|
||||
"name": "Prowler",
|
||||
"email": "support@prowler.com"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "prowler",
|
||||
"source": "./claude_plugins/prowler",
|
||||
"description": "Prowler for Claude Code — cloud security and compliance skills powered by the Prowler MCP server. Bundles compliance triage and remediation; more skills coming.",
|
||||
"category": "security",
|
||||
"homepage": "https://prowler.com"
|
||||
}
|
||||
]
|
||||
}
|
||||
+8
-1
@@ -11,7 +11,14 @@ envs = "wt step copy-ignored"
|
||||
[[pre-start]]
|
||||
deps = "uv sync"
|
||||
|
||||
# Block 3: reminder - last visible output before `wt switch` returns.
|
||||
# Block 3: prepare pnpm via corepack.
|
||||
[[pre-start]]
|
||||
corepack-enable = "corepack enable"
|
||||
|
||||
[[pre-start]]
|
||||
corepack-install = "cd ui && corepack install"
|
||||
|
||||
# Block 4: reminder - last visible output before `wt switch` returns.
|
||||
# Hooks can't mutate the parent shell, so venv activation is manual.
|
||||
[[pre-start]]
|
||||
reminder = "echo '>> Reminder: activate the venv in this shell with: source .venv/bin/activate'"
|
||||
|
||||
@@ -145,7 +145,7 @@ SENTRY_RELEASE=local
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT}
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.27.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.30.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
+22
-22
@@ -6,17 +6,17 @@
|
||||
version: 2
|
||||
updates:
|
||||
# v5
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "pip"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "pip"
|
||||
@@ -66,17 +66,17 @@ updates:
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: "pre-commit"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "pre-commit"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
# - package-ecosystem: "pre-commit"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pre-commit"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/04/15
|
||||
# v4.6
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:best-practices",
|
||||
":enablePreCommit",
|
||||
":semanticCommits",
|
||||
":enableVulnerabilityAlertsWithLabel(security)",
|
||||
"docker:enableMajor",
|
||||
"helpers:pinGitHubActionDigestsToSemver",
|
||||
"helpers:disableTypesNodeMajor",
|
||||
"security:openssf-scorecard",
|
||||
"customManagers:githubActionsVersions",
|
||||
"customManagers:dockerfileVersions"
|
||||
],
|
||||
"timezone": "Europe/Madrid",
|
||||
"baseBranchPatterns": [
|
||||
"master"
|
||||
],
|
||||
"labels": [
|
||||
"dependencies"
|
||||
],
|
||||
"dependencyDashboardTitle": "Dependency Dashboard",
|
||||
"prConcurrentLimit": 20,
|
||||
"prHourlyLimit": 10,
|
||||
"vulnerabilityAlerts": {
|
||||
"prHourlyLimit": 0,
|
||||
"prConcurrentLimit": 0
|
||||
},
|
||||
"configMigration": true,
|
||||
"minimumReleaseAge": "7 days",
|
||||
"rangeStrategy": "pin",
|
||||
"packageRules": [
|
||||
{
|
||||
"description": "Patches: 1st of every month, Madrid overnight window (22:00-06:00)",
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
],
|
||||
"schedule": [
|
||||
"* 22-23,0-5 1 * *"
|
||||
],
|
||||
"enabled": false
|
||||
},
|
||||
{
|
||||
"description": "Minors: 8th of every 3 months, Madrid overnight window (22:00-06:00)",
|
||||
"matchUpdateTypes": [
|
||||
"minor"
|
||||
],
|
||||
"schedule": [
|
||||
"* 22-23,0-5 8 */3 *"
|
||||
],
|
||||
"enabled": false
|
||||
},
|
||||
{
|
||||
"description": "Majors: 15th of every 3 months, Madrid overnight window",
|
||||
"matchUpdateTypes": [
|
||||
"major"
|
||||
],
|
||||
"schedule": [
|
||||
"* 22-23,0-5 15 */3 *"
|
||||
],
|
||||
"enabled": false
|
||||
},
|
||||
{
|
||||
"description": "GitHub Actions - single grouped PR, no changelog, scope=ci",
|
||||
"matchManagers": [
|
||||
"github-actions"
|
||||
],
|
||||
"groupName": "github-actions",
|
||||
"semanticCommitScope": "ci",
|
||||
"addLabels": [
|
||||
"no-changelog"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Docker images - single grouped PR, no changelog, scope=docker",
|
||||
"matchManagers": [
|
||||
"dockerfile",
|
||||
"docker-compose"
|
||||
],
|
||||
"groupName": "docker",
|
||||
"semanticCommitScope": "docker",
|
||||
"addLabels": [
|
||||
"no-changelog"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Pre-commit hooks - single grouped PR, scope=pre-commit",
|
||||
"matchManagers": [
|
||||
"pre-commit"
|
||||
],
|
||||
"groupName": "pre-commit hooks",
|
||||
"semanticCommitScope": "pre-commit",
|
||||
"addLabels": [
|
||||
"no-changelog"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "UI - scope=ui",
|
||||
"matchFileNames": [
|
||||
"ui/**"
|
||||
],
|
||||
"semanticCommitScope": "ui"
|
||||
},
|
||||
{
|
||||
"description": "API - scope=api",
|
||||
"matchFileNames": [
|
||||
"api/**"
|
||||
],
|
||||
"semanticCommitScope": "api"
|
||||
},
|
||||
{
|
||||
"description": "MCP server - scope=mcp",
|
||||
"matchFileNames": [
|
||||
"mcp_server/**"
|
||||
],
|
||||
"semanticCommitScope": "mcp"
|
||||
},
|
||||
{
|
||||
"description": "Python SDK (root) - scope=sdk",
|
||||
"matchFileNames": [
|
||||
"pyproject.toml",
|
||||
"poetry.lock",
|
||||
"util/prowler-bulk-provisioning/**"
|
||||
],
|
||||
"semanticCommitScope": "sdk"
|
||||
},
|
||||
{
|
||||
"description": "UI devDependencies - no changelog",
|
||||
"matchFileNames": [
|
||||
"ui/**"
|
||||
],
|
||||
"matchDepTypes": [
|
||||
"devDependencies"
|
||||
],
|
||||
"addLabels": [
|
||||
"no-changelog"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -61,12 +61,12 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/api-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -122,6 +122,7 @@ jobs:
|
||||
github.com:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
pypi.org:443
|
||||
registry-1.docker.io:443
|
||||
release-assets.githubusercontent.com:443
|
||||
@@ -132,14 +133,18 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Pin prowler SDK to latest master commit
|
||||
if: github.event_name == 'push'
|
||||
- name: Refresh prowler SDK pin to current branch tip
|
||||
run: |
|
||||
LATEST_SHA=$(git ls-remote https://github.com/prowler-cloud/prowler.git refs/heads/master | cut -f1)
|
||||
sed -i "s|prowler-cloud/prowler.git@master|prowler-cloud/prowler.git@${LATEST_SHA}|" api/pyproject.toml
|
||||
# api/pyproject.toml has `@master` on master and `@v5.X` on release
|
||||
# branches (set by prepare-release.yml). uv lock --upgrade-package
|
||||
# re-resolves whichever ref is present against the current branch tip
|
||||
# and writes the SHA into api/uv.lock. The Dockerfile runs
|
||||
# `uv sync --locked`, which is what actually drives the install.
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
(cd api && uv lock --upgrade-package prowler)
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -150,7 +155,7 @@ jobs:
|
||||
- name: Build and push API container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
push: true
|
||||
@@ -170,7 +175,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -179,8 +184,9 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -230,7 +236,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -277,7 +283,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: api/Dockerfile
|
||||
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -83,6 +83,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
debian.map.fastlydns.net:80
|
||||
release-assets.githubusercontent.com:443
|
||||
objects.githubusercontent.com:443
|
||||
@@ -103,7 +104,7 @@ jobs:
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: api/**
|
||||
files_ignore: |
|
||||
@@ -118,7 +119,7 @@ jobs:
|
||||
|
||||
- name: Build container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.API_WORKING_DIR }}
|
||||
push: false
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
|
||||
- name: Check for API changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -139,6 +139,17 @@ jobs:
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
- name: Regenerate lockfiles after version bump
|
||||
run: |
|
||||
set -e
|
||||
# The bumps above edit pyproject.toml / api/pyproject.toml but leave
|
||||
# uv.lock / api/uv.lock stale, which makes `uv sync --locked` fail in
|
||||
# the container builds. Refresh both with the uv version the images
|
||||
# pin (plain `uv lock`, no --upgrade: only the version line changes).
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
uv lock
|
||||
(cd api && uv lock)
|
||||
|
||||
- name: Bump UI version (.env)
|
||||
run: |
|
||||
set -e
|
||||
@@ -155,7 +166,7 @@ jobs:
|
||||
run: git --no-pager diff
|
||||
|
||||
- name: Create PR for next versions to master
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
@@ -191,7 +202,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -240,6 +251,17 @@ jobs:
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${FIRST_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${FIRST_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
- name: Regenerate lockfiles after version bump
|
||||
run: |
|
||||
set -e
|
||||
# The bumps above edit pyproject.toml / api/pyproject.toml but leave
|
||||
# uv.lock / api/uv.lock stale, which makes `uv sync --locked` fail in
|
||||
# the container builds. Refresh both with the uv version the images
|
||||
# pin (plain `uv lock`, no --upgrade: only the version line changes).
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
uv lock
|
||||
(cd api && uv lock)
|
||||
|
||||
- name: Bump UI version (.env)
|
||||
run: |
|
||||
set -e
|
||||
@@ -249,7 +271,7 @@ jobs:
|
||||
run: git --no-pager diff
|
||||
|
||||
- name: Create PR for first patch versions to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
@@ -285,7 +307,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -341,6 +363,17 @@ jobs:
|
||||
sed -i "s|version = \"${CURRENT_API_VERSION}\"|version = \"${NEXT_API_PATCH_VERSION}\"|" api/pyproject.toml
|
||||
sed -i "s| version: ${CURRENT_API_VERSION}| version: ${NEXT_API_PATCH_VERSION}|" api/src/backend/api/specs/v1.yaml
|
||||
|
||||
- name: Regenerate lockfiles after version bump
|
||||
run: |
|
||||
set -e
|
||||
# The bumps above edit pyproject.toml / api/pyproject.toml but leave
|
||||
# uv.lock / api/uv.lock stale, which makes `uv sync --locked` fail in
|
||||
# the container builds. Refresh both with the uv version the images
|
||||
# pin (plain `uv lock`, no --upgrade: only the version line changes).
|
||||
pip install --no-cache-dir "uv==0.11.14"
|
||||
uv lock
|
||||
(cd api && uv lock)
|
||||
|
||||
- name: Bump UI version (.env)
|
||||
run: |
|
||||
set -e
|
||||
@@ -350,7 +383,7 @@ jobs:
|
||||
run: git --no-pager diff
|
||||
|
||||
- name: Create PR for next patch versions to version branch
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -51,6 +51,6 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2
|
||||
uses: zizmorcore/zizmor-action@a16621b09c6db4281f81a93cb393b05dcd7b7165 # v0.5.5
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
# We can't block as Trufflehog needs to verify secrets against vendors
|
||||
egress-policy: audit
|
||||
@@ -44,6 +44,6 @@ jobs:
|
||||
|
||||
- name: Scan diff for secrets with TruffleHog
|
||||
# Action auto-injects --since-commit/--branch from event payload; passing them in extra_args produces duplicate flags.
|
||||
uses: trufflesecurity/trufflehog@ef6e76c3c4023279497fab4721ffa071a722fd05 # v3.92.4
|
||||
uses: trufflesecurity/trufflehog@37b77001d0174ebec2fcca2bd83ff83a6d45a3ab # v3.95.3
|
||||
with:
|
||||
extra_args: --results=verified,unknown
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
Generated
+12
-12
@@ -66,12 +66,12 @@ jobs:
|
||||
title: ${{ steps.compute-text.outputs.title }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Check workflow file timestamps
|
||||
@@ -135,12 +135,12 @@ jobs:
|
||||
secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Checkout repository
|
||||
@@ -870,12 +870,12 @@ jobs:
|
||||
total_count: ${{ steps.missing_tool.outputs.total_count }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Download agent output artifact
|
||||
@@ -982,12 +982,12 @@ jobs:
|
||||
success: ${{ steps.parse_results.outputs.success }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Download agent artifacts
|
||||
@@ -1091,12 +1091,12 @@ jobs:
|
||||
activated: ${{ (steps.check_membership.outputs.is_team_member == 'true') && (steps.check_rate_limit.outputs.rate_limit_ok == 'true') }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Add eyes reaction for immediate feedback
|
||||
@@ -1164,12 +1164,12 @@ jobs:
|
||||
process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@9382be3ca9ac18917e111a99d4e6bbff58d0dccc # v0.43.23
|
||||
uses: github/gh-aw/actions/setup@4d44d0e89851a877f4ddc0cb6c0197e42b1016c5 # v0.73.0
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Download agent output artifact
|
||||
|
||||
@@ -27,12 +27,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Apply labels to PR
|
||||
uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1
|
||||
uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0
|
||||
with:
|
||||
sync-labels: true
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
name: 'Docs: Markdown Lint'
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
markdown-lint:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
registry.npmjs.org:443
|
||||
release-assets.githubusercontent.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: ui/.nvmrc
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
|
||||
with:
|
||||
package_json_file: ui/package.json
|
||||
run_install: false
|
||||
|
||||
- name: Run markdownlint
|
||||
# Pin must match .pre-commit-config.yaml so prek and CI behave identically.
|
||||
# pnpm dlx doesn't accept --ignore-scripts as a flag; the env var
|
||||
# disables postinstall scripts on transitives the same way.
|
||||
env:
|
||||
pnpm_config_ignore_scripts: 'true'
|
||||
run: pnpm dlx markdownlint-cli@0.45.0 '**/*.md'
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -114,6 +114,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
ghcr.io:443
|
||||
pkg-containers.githubusercontent.com:443
|
||||
files.pythonhosted.org:443
|
||||
@@ -125,7 +126,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -136,7 +137,7 @@ jobs:
|
||||
- name: Build and push MCP container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
push: true
|
||||
@@ -164,18 +165,19 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -225,7 +227,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -272,7 +274,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: mcp_server/Dockerfile
|
||||
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
|
||||
- name: Check for MCP changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: mcp_server/**
|
||||
files_ignore: |
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
- name: Build MCP container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.MCP_WORKING_DIR }}
|
||||
push: false
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
|
||||
- name: Publish prowler-mcp package to PyPI
|
||||
if: steps.pypi-check.outputs.skip != 'true'
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
||||
with:
|
||||
packages-dir: ${{ env.WORKING_DIRECTORY }}/dist/
|
||||
print-hash: true
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
name: 'MCP: Security'
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/pyproject.toml'
|
||||
- 'mcp_server/uv.lock'
|
||||
- '.github/workflows/mcp-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'mcp_server/pyproject.toml'
|
||||
- 'mcp_server/uv.lock'
|
||||
- '.github/workflows/mcp-security.yml'
|
||||
- '.github/actions/osv-scanner/**'
|
||||
- '.github/scripts/osv-scan.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
mcp-security-scans:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # osv-scanner action posts/updates a PR comment with findings
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
api.github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.osv.dev:443
|
||||
api.deps.dev:443
|
||||
osv-vulnerabilities.storage.googleapis.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
|
||||
- name: Check for MCP dependency changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
mcp_server/pyproject.toml
|
||||
mcp_server/uv.lock
|
||||
.github/workflows/mcp-security.yml
|
||||
.github/actions/osv-scanner/**
|
||||
.github/scripts/osv-scan.sh
|
||||
|
||||
- name: Dependency vulnerability scan with osv-scanner
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/osv-scanner
|
||||
with:
|
||||
lockfile: mcp_server/uv.lock
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build ${{ matrix.component }} container (linux/arm64)
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ matrix.context }}
|
||||
file: ${{ matrix.dockerfile }}
|
||||
@@ -83,7 +83,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
prowler/providers/**/services/**/*.metadata.json
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: '**'
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -338,7 +338,7 @@ jobs:
|
||||
|
||||
- name: Create PR for API dependency update
|
||||
if: ${{ env.PATCH_VERSION == '0' }}
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
commit-message: 'chore(api): update prowler dependency to ${{ env.BRANCH_NAME }} for release ${{ env.PROWLER_VERSION }}'
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
name: 'CI: Renovate Config Validate'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
paths:
|
||||
- '.github/renovate.json'
|
||||
- '.pre-commit-config.yaml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
env:
|
||||
# renovate: datasource=pypi depName=prek
|
||||
PREK_VERSION: '0.4.0'
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: Validate Renovate config
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
codeload.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
registry.npmjs.org:443
|
||||
nodejs.org:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
|
||||
|
||||
- name: Install prek
|
||||
run: uv tool install "prek==${PREK_VERSION}"
|
||||
|
||||
- name: Validate Renovate config
|
||||
run: prek run renovate-config-validator --files .github/renovate.json
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ./**
|
||||
files_ignore: |
|
||||
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -66,12 +66,12 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/sdk-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -141,7 +141,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -149,6 +149,7 @@ jobs:
|
||||
public.ecr.aws:443
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
auth.docker.io:443
|
||||
debian.map.fastlydns.net:80
|
||||
github.com:443
|
||||
@@ -167,13 +168,13 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
@@ -187,7 +188,7 @@ jobs:
|
||||
- name: Build and push SDK container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
file: ${{ env.DOCKERFILE_PATH }}
|
||||
@@ -208,7 +209,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -216,19 +217,20 @@ jobs:
|
||||
auth.docker.io:443
|
||||
public.ecr.aws:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
@@ -265,7 +267,7 @@ jobs:
|
||||
# Push to toniblyx/prowler only for current version (latest/stable/release tags)
|
||||
- name: Login to DockerHub (toniblyx)
|
||||
if: needs.setup.outputs.latest_tag == 'latest'
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.TONIBLYX_DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.TONIBLYX_DOCKERHUB_PASSWORD }}
|
||||
@@ -290,7 +292,7 @@ jobs:
|
||||
# Re-login as prowlercloud for cleanup of intermediate tags
|
||||
- name: Login to DockerHub (prowlercloud)
|
||||
if: always()
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -318,7 +320,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: Dockerfile
|
||||
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -85,6 +85,7 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
api.github.com:443
|
||||
mirror.gcr.io:443
|
||||
check.trivy.dev:443
|
||||
@@ -108,7 +109,7 @@ jobs:
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ./**
|
||||
files_ignore: |
|
||||
@@ -137,7 +138,7 @@ jobs:
|
||||
|
||||
- name: Build SDK container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
push: false
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
run: uv build
|
||||
|
||||
- name: Publish Prowler package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
||||
with:
|
||||
print-hash: true
|
||||
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -129,6 +129,6 @@ jobs:
|
||||
run: uv build
|
||||
|
||||
- name: Publish prowler-cloud package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
||||
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
||||
with:
|
||||
print-hash: true
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
run: pip install boto3
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
aws-region: ${{ env.AWS_REGION }}
|
||||
role-to-assume: ${{ secrets.DEV_IAM_ROLE_ARN }}
|
||||
@@ -58,7 +58,7 @@ jobs:
|
||||
|
||||
- name: Create pull request
|
||||
id: create-pr
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
author: 'prowler-bot <179230569+prowler-bot@users.noreply.github.com>'
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
- name: Create pull request
|
||||
id: create-pr
|
||||
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
author: 'prowler-bot <179230569+prowler-bot@users.noreply.github.com>'
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files:
|
||||
./**
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -46,6 +46,7 @@ jobs:
|
||||
schema.ocsf.io:443
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
o26192.ingest.us.sentry.io:443
|
||||
management.azure.com:443
|
||||
@@ -69,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Check for SDK changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ./**
|
||||
files_ignore: |
|
||||
@@ -102,7 +103,7 @@ jobs:
|
||||
- name: Check if AWS files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-aws
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/aws/**
|
||||
@@ -232,7 +233,7 @@ jobs:
|
||||
- name: Check if Azure files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-azure
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/azure/**
|
||||
@@ -256,7 +257,7 @@ jobs:
|
||||
- name: Check if GCP files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-gcp
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/gcp/**
|
||||
@@ -280,7 +281,7 @@ jobs:
|
||||
- name: Check if Kubernetes files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-kubernetes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/kubernetes/**
|
||||
@@ -304,7 +305,7 @@ jobs:
|
||||
- name: Check if GitHub files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-github
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/github/**
|
||||
@@ -328,7 +329,7 @@ jobs:
|
||||
- name: Check if Okta files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-okta
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/okta/**
|
||||
@@ -352,7 +353,7 @@ jobs:
|
||||
- name: Check if NHN files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-nhn
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/nhn/**
|
||||
@@ -376,7 +377,7 @@ jobs:
|
||||
- name: Check if M365 files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-m365
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/m365/**
|
||||
@@ -400,7 +401,7 @@ jobs:
|
||||
- name: Check if IaC files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-iac
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/iac/**
|
||||
@@ -424,7 +425,7 @@ jobs:
|
||||
- name: Check if MongoDB Atlas files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-mongodbatlas
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/mongodbatlas/**
|
||||
@@ -448,7 +449,7 @@ jobs:
|
||||
- name: Check if OCI files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-oraclecloud
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/oraclecloud/**
|
||||
@@ -472,7 +473,7 @@ jobs:
|
||||
- name: Check if OpenStack files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-openstack
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/openstack/**
|
||||
@@ -496,7 +497,7 @@ jobs:
|
||||
- name: Check if Google Workspace files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-googleworkspace
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/googleworkspace/**
|
||||
@@ -520,7 +521,7 @@ jobs:
|
||||
- name: Check if Vercel files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-vercel
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/vercel/**
|
||||
@@ -539,12 +540,87 @@ jobs:
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-vercel
|
||||
files: ./vercel_coverage.xml
|
||||
|
||||
# Scaleway Provider
|
||||
- name: Check if Scaleway files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-scaleway
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/scaleway/**
|
||||
./tests/**/scaleway/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run Scaleway tests
|
||||
if: steps.changed-scaleway.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/scaleway --cov-report=xml:scaleway_coverage.xml tests/providers/scaleway
|
||||
|
||||
- name: Upload Scaleway coverage to Codecov
|
||||
if: steps.changed-scaleway.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-scaleway
|
||||
files: ./scaleway_coverage.xml
|
||||
|
||||
# StackIT Provider
|
||||
- name: Check if StackIT files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-stackit
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/stackit/**
|
||||
./tests/**/stackit/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run StackIT tests
|
||||
if: steps.changed-stackit.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/stackit --cov-report=xml:stackit_coverage.xml tests/providers/stackit
|
||||
|
||||
- name: Upload StackIT coverage to Codecov
|
||||
if: steps.changed-stackit.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-stackit
|
||||
files: ./stackit_coverage.xml
|
||||
|
||||
# External Provider (dynamic loading)
|
||||
- name: Check if External Provider files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-external
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
with:
|
||||
files: |
|
||||
./prowler/providers/common/**
|
||||
./prowler/config/**
|
||||
./prowler/lib/**
|
||||
./tests/providers/external/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run External Provider tests
|
||||
if: steps.changed-external.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/common --cov=./prowler/config --cov=./prowler/lib --cov-report=xml:external_coverage.xml tests/providers/external
|
||||
|
||||
- name: Upload External Provider coverage to Codecov
|
||||
if: steps.changed-external.outputs.any_changed == 'true'
|
||||
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-external
|
||||
files: ./external_coverage.xml
|
||||
|
||||
# Lib
|
||||
- name: Check if Lib files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-lib
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/lib/**
|
||||
@@ -568,7 +644,7 @@ jobs:
|
||||
- name: Check if Config files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-config
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/config/**
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -62,12 +62,12 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
config-file: ./.github/codeql/ui-codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4
|
||||
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -110,12 +110,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
auth.docker.io:443
|
||||
registry.npmjs.org:443
|
||||
dl-cdn.alpinelinux.org:443
|
||||
@@ -129,7 +130,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -140,7 +141,7 @@ jobs:
|
||||
- name: Build and push UI container for ${{ matrix.arch }}
|
||||
id: container-push
|
||||
if: github.event_name == 'push' || github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.WORKING_DIRECTORY }}
|
||||
build-args: |
|
||||
@@ -163,7 +164,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -172,9 +173,10 @@ jobs:
|
||||
registry-1.docker.io:443
|
||||
auth.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
@@ -224,7 +226,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -271,7 +273,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
|
||||
- name: Check if Dockerfile changed
|
||||
id: dockerfile-changed
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ui/Dockerfile
|
||||
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -100,7 +100,7 @@ jobs:
|
||||
|
||||
- name: Check for UI changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: ui/**
|
||||
files_ignore: |
|
||||
@@ -114,7 +114,7 @@ jobs:
|
||||
|
||||
- name: Build UI container
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ${{ env.UI_WORKING_DIR }}
|
||||
target: prod
|
||||
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: '24.13.0'
|
||||
node-version-file: 'ui/.nvmrc'
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
|
||||
@@ -184,7 +184,7 @@ jobs:
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
|
||||
|
||||
- name: Setup pnpm and Next.js cache
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
${{ env.STORE_PATH }}
|
||||
@@ -204,7 +204,7 @@ jobs:
|
||||
run: pnpm run build
|
||||
|
||||
- name: Cache Playwright browsers
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
id: playwright-cache
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
@@ -295,7 +295,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
|
||||
- name: Check for UI dependency changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
ui/package.json
|
||||
|
||||
@@ -16,7 +16,6 @@ concurrency:
|
||||
|
||||
env:
|
||||
UI_WORKING_DIR: ./ui
|
||||
NODE_VERSION: "24.13.0"
|
||||
|
||||
permissions: {}
|
||||
|
||||
@@ -32,7 +31,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -54,7 +53,7 @@ jobs:
|
||||
|
||||
- name: Check for UI changes
|
||||
id: check-changes
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
ui/**
|
||||
@@ -67,7 +66,7 @@ jobs:
|
||||
- name: Get changed source files for targeted tests
|
||||
id: changed-source
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
ui/**/*.ts
|
||||
@@ -83,7 +82,7 @@ jobs:
|
||||
- name: Check for critical path changes (run all tests)
|
||||
id: critical-changes
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
ui/lib/**
|
||||
@@ -93,11 +92,11 @@ jobs:
|
||||
ui/vitest.config.ts
|
||||
ui/vitest.setup.ts
|
||||
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
- name: Setup Node.js
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
node-version-file: 'ui/.nvmrc'
|
||||
|
||||
- name: Setup pnpm
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
@@ -113,7 +112,7 @@ jobs:
|
||||
|
||||
- name: Setup pnpm and Next.js cache
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
${{ env.STORE_PATH }}
|
||||
@@ -162,7 +161,7 @@ jobs:
|
||||
- name: Cache Playwright browsers
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: playwright-cache
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('ui/pnpm-lock.yaml') }}
|
||||
|
||||
@@ -60,7 +60,6 @@ htmlcov/
|
||||
**/mcp-config.json
|
||||
**/mcpServers.json
|
||||
.mcp/
|
||||
.mcp.json
|
||||
|
||||
# AI Coding Assistants - Cursor
|
||||
.cursorignore
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "markdownlint/style/prettier",
|
||||
"first-line-h1": false,
|
||||
"no-duplicate-heading": {
|
||||
"siblings_only": true
|
||||
},
|
||||
"no-inline-html": false,
|
||||
"line-length": false,
|
||||
"no-bare-urls": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
node_modules/
|
||||
ui/node_modules/
|
||||
.git/
|
||||
.venv/
|
||||
**/.venv/
|
||||
dist/
|
||||
build/
|
||||
htmlcov/
|
||||
.next/
|
||||
ui/.next/
|
||||
ui/out/
|
||||
contrib/
|
||||
|
||||
# Auto-generated content (keepachangelog format legitimately repeats section headings).
|
||||
# Revisit with the team — see beads task on markdownlint rule triage.
|
||||
**/CHANGELOG.md
|
||||
@@ -49,6 +49,14 @@ repos:
|
||||
files: ^\.github/(workflows|actions)/.+\.ya?ml$|^\.github/dependabot\.ya?ml$
|
||||
priority: 30
|
||||
|
||||
## RENOVATE
|
||||
- repo: https://github.com/renovatebot/pre-commit-hooks
|
||||
rev: 43.150.0
|
||||
hooks:
|
||||
- id: renovate-config-validator
|
||||
files: ^\.github/renovate\.json$
|
||||
priority: 10
|
||||
|
||||
## BASH
|
||||
- repo: https://github.com/koalaman/shellcheck-precommit
|
||||
rev: v0.11.0
|
||||
@@ -125,6 +133,13 @@ repos:
|
||||
pass_filenames: false
|
||||
priority: 50
|
||||
|
||||
## MARKDOWN
|
||||
- repo: https://github.com/igorshubovych/markdownlint-cli
|
||||
rev: v0.45.0
|
||||
hooks:
|
||||
- id: markdownlint
|
||||
priority: 30
|
||||
|
||||
## CONTAINERS
|
||||
- repo: https://github.com/hadolint/hadolint
|
||||
rev: v2.14.0
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
.envrc
|
||||
ui/.env.local
|
||||
openspec/
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
Use these skills for detailed patterns on-demand:
|
||||
|
||||
### Generic Skills (Any Project)
|
||||
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `typescript` | Const types, flat interfaces, utility types | [SKILL.md](skills/typescript/SKILL.md) |
|
||||
@@ -28,6 +29,7 @@ Use these skills for detailed patterns on-demand:
|
||||
| `tdd` | Test-Driven Development workflow | [SKILL.md](skills/tdd/SKILL.md) |
|
||||
|
||||
### Prowler-Specific Skills
|
||||
|
||||
| Skill | Description | URL |
|
||||
|-------|-------------|-----|
|
||||
| `prowler` | Project overview, component navigation | [SKILL.md](skills/prowler/SKILL.md) |
|
||||
|
||||
+4
-3
@@ -1,4 +1,4 @@
|
||||
# Do you want to learn on how to...
|
||||
# Do you want to learn on how to
|
||||
|
||||
- [Contribute with your code or fixes to Prowler](https://docs.prowler.com/developer-guide/introduction)
|
||||
- [Create a new provider](https://docs.prowler.com/developer-guide/provider)
|
||||
@@ -32,5 +32,6 @@ Provider-specific developer notes:
|
||||
|
||||
Want some swag as appreciation for your contribution?
|
||||
|
||||
# Prowler Developer Guide
|
||||
https://goto.prowler.com/devguide
|
||||
## Prowler Developer Guide
|
||||
|
||||
<https://goto.prowler.com/devguide>
|
||||
|
||||
+7
-7
@@ -76,11 +76,11 @@ USER prowler
|
||||
WORKDIR /home/prowler
|
||||
|
||||
# Copy necessary files
|
||||
COPY prowler/ /home/prowler/prowler/
|
||||
COPY dashboard/ /home/prowler/dashboard/
|
||||
COPY pyproject.toml uv.lock /home/prowler/
|
||||
COPY README.md /home/prowler/
|
||||
COPY prowler/providers/m365/lib/powershell/m365_powershell.py /home/prowler/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
COPY --chown=prowler:prowler prowler/ /home/prowler/prowler/
|
||||
COPY --chown=prowler:prowler dashboard/ /home/prowler/dashboard/
|
||||
COPY --chown=prowler:prowler pyproject.toml uv.lock /home/prowler/
|
||||
COPY --chown=prowler:prowler README.md /home/prowler/
|
||||
COPY --chown=prowler:prowler prowler/providers/m365/lib/powershell/m365_powershell.py /home/prowler/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
# Install Python dependencies
|
||||
ENV HOME='/home/prowler'
|
||||
@@ -89,7 +89,7 @@ ENV PATH="${HOME}/.local/bin:${PATH}"
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
|
||||
RUN uv sync --compile-bytecode && \
|
||||
RUN uv sync --locked --compile-bytecode && \
|
||||
rm -rf ~/.cache/uv
|
||||
|
||||
# Install PowerShell modules
|
||||
@@ -100,4 +100,4 @@ RUN pip uninstall dash-html-components -y && \
|
||||
pip uninstall dash-core-components -y
|
||||
|
||||
USER prowler
|
||||
ENTRYPOINT [".venv/bin/prowler"]
|
||||
ENTRYPOINT ["/home/prowler/.venv/bin/prowler"]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<p align="center">
|
||||
<img align="center" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-black.png#gh-light-mode-only" width="50%" height="50%">
|
||||
<img align="center" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-white.png#gh-dark-mode-only" width="50%" height="50%">
|
||||
<img align="center" alt="Prowler logo" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-black.png#gh-light-mode-only" width="50%" height="50%">
|
||||
<img align="center" alt="Prowler logo" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-white.png#gh-dark-mode-only" width="50%" height="50%">
|
||||
</p>
|
||||
<p align="center">
|
||||
<b><i>Prowler</b> is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
|
||||
@@ -22,8 +22,8 @@
|
||||
<a href="https://pypistats.org/packages/prowler"><img alt="PyPI Downloads" src="https://img.shields.io/pypi/dw/prowler.svg?label=downloads"></a>
|
||||
<a href="https://hub.docker.com/r/toniblyx/prowler"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/toniblyx/prowler"></a>
|
||||
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height=19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img alt="Codecov coverage" src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img alt="Linux Foundation insights health score" src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://github.com/prowler-cloud/prowler/releases"><img alt="Version" src="https://img.shields.io/github/v/release/prowler-cloud/prowler"></a>
|
||||
@@ -36,7 +36,7 @@
|
||||
</p>
|
||||
<hr>
|
||||
<p align="center">
|
||||
<img align="center" src="/docs/img/prowler-cloud.gif" width="100%" height="100%">
|
||||
<img align="center" alt="Prowler Cloud demo" src="/docs/img/prowler-cloud.gif" width="100%" height="100%">
|
||||
</p>
|
||||
|
||||
# Description
|
||||
@@ -122,6 +122,7 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
| Vercel | 26 | 6 | 0 | 8 | Official | UI, API, CLI |
|
||||
| Okta | 1 | 1 | 0 | 1 | Official | CLI |
|
||||
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 0 | 1 | Unofficial | CLI |
|
||||
| StackIT [Contact us](https://prowler.com/contact) | 4 | 1 | 0 | 1 | Unofficial | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
|
||||
> [!Note]
|
||||
@@ -146,11 +147,13 @@ Prowler App offers flexible installation methods tailored to various environment
|
||||
|
||||
### Docker Compose
|
||||
|
||||
**Requirements**
|
||||
#### Requirements
|
||||
|
||||
* `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
- `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
|
||||
**Commands**
|
||||
#### Commands
|
||||
|
||||
_macOS/Linux:_
|
||||
|
||||
``` console
|
||||
VERSION=$(curl -s https://api.github.com/repos/prowler-cloud/prowler/releases/latest | jq -r .tag_name)
|
||||
@@ -160,6 +163,16 @@ curl -sLO "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/${V
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
_Windows PowerShell:_
|
||||
|
||||
``` powershell
|
||||
$VERSION = (Invoke-RestMethod -Uri "https://api.github.com/repos/prowler-cloud/prowler/releases/latest").tag_name
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/docker-compose.yml" -OutFile "docker-compose.yml"
|
||||
# Environment variables can be customized in the .env file. Using default values in production environments is not recommended.
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/prowler-cloud/prowler/refs/tags/$VERSION/.env" -OutFile ".env"
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> 🔒 For a secure setup, the API auto-generates a unique key pair, `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY`, and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate keys, delete the stored key files and restart the API.
|
||||
|
||||
@@ -175,14 +188,14 @@ You can find more information in the [Troubleshooting](./docs/troubleshooting.md
|
||||
|
||||
### From GitHub
|
||||
|
||||
**Requirements**
|
||||
#### Requirements
|
||||
|
||||
* `git` installed.
|
||||
* `uv` installed: [uv installation](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
* `pnpm` installed: [pnpm installation](https://pnpm.io/installation).
|
||||
* `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
- `git` installed.
|
||||
- `uv` installed: [uv installation](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
- `pnpm` installed: [pnpm installation](https://pnpm.io/installation).
|
||||
- `Docker Compose` installed: https://docs.docker.com/compose/install/.
|
||||
|
||||
**Commands to run the API**
|
||||
#### Commands to run the API
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -199,7 +212,7 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
|
||||
> After completing the setup, access the API documentation at http://localhost:8080/api/v1/docs.
|
||||
|
||||
**Commands to run the API Worker**
|
||||
#### Commands to run the API Worker
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -212,7 +225,7 @@ cd src/backend
|
||||
python -m celery -A config.celery worker -l info -E
|
||||
```
|
||||
|
||||
**Commands to run the API Scheduler**
|
||||
#### Commands to run the API Scheduler
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -225,7 +238,7 @@ cd src/backend
|
||||
python -m celery -A config.celery beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
```
|
||||
|
||||
**Commands to run the UI**
|
||||
#### Commands to run the UI
|
||||
|
||||
``` console
|
||||
git clone https://github.com/prowler-cloud/prowler
|
||||
@@ -237,7 +250,7 @@ pnpm start
|
||||
|
||||
> Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
|
||||
|
||||
**Pre-commit Hooks Setup**
|
||||
#### Pre-commit Hooks Setup
|
||||
|
||||
Some pre-commit hooks require tools installed on your system:
|
||||
|
||||
@@ -257,14 +270,14 @@ prowler -v
|
||||
|
||||
### Containers
|
||||
|
||||
**Available Versions of Prowler CLI**
|
||||
#### Available Versions of Prowler CLI
|
||||
|
||||
The following versions of Prowler CLI are available, depending on your requirements:
|
||||
|
||||
- `latest`: Synchronizes with the `master` branch. Note that this version is not stable.
|
||||
- `v4-latest`: Synchronizes with the `v4` branch. Note that this version is not stable.
|
||||
- `v3-latest`: Synchronizes with the `v3` branch. Note that this version is not stable.
|
||||
- `<x.y.z>` (release): Stable releases corresponding to specific versions. You can find the complete list of releases [here](https://github.com/prowler-cloud/prowler/releases).
|
||||
- `<x.y.z>` (release): Stable releases corresponding to specific versions. See the [complete list of Prowler releases](https://github.com/prowler-cloud/prowler/releases).
|
||||
- `stable`: Always points to the latest release.
|
||||
- `v4-stable`: Always points to the latest release for v4.
|
||||
- `v3-stable`: Always points to the latest release for v3.
|
||||
@@ -293,7 +306,7 @@ python prowler-cli.py -v
|
||||
|
||||
# 🛡️ GitHub Action
|
||||
|
||||
The official **Prowler GitHub Action** runs Prowler scans in your GitHub workflows using the official [`prowlercloud/prowler`](https://hub.docker.com/r/prowlercloud/prowler) Docker image. Scans run on any [supported provider](https://docs.prowler.com/user-guide/providers/), with optional [`--push-to-cloud`](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings) to send findings to Prowler Cloud and optional SARIF upload so findings show up in the repo's **Security → Code scanning** tab and as inline PR annotations.
|
||||
The official **Prowler GitHub Action** runs Prowler scans in your GitHub workflows using the official [`prowlercloud/prowler`](https://hub.docker.com/r/prowlercloud/prowler) Docker image. Scans run on any [supported provider](https://docs.prowler.com/user-guide/providers/), with optional [`--push-to-cloud`](https://docs.prowler.com/user-guide/tutorials/prowler-import-findings) to send findings to Prowler Cloud and optional SARIF upload so findings show up in the repo's **Security → Code scanning** tab and as inline PR annotations.
|
||||
|
||||
```yaml
|
||||
name: Prowler IaC Scan
|
||||
@@ -338,7 +351,7 @@ Full configuration, per-provider authentication, and SARIF examples: [Prowler Gi
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
**Running Prowler**
|
||||
### Running Prowler
|
||||
|
||||
Prowler can be executed across various environments, offering flexibility to meet your needs. It can be run from:
|
||||
|
||||
|
||||
+2
-2
@@ -22,7 +22,7 @@ inputs:
|
||||
required: false
|
||||
default: json-ocsf
|
||||
push-to-cloud:
|
||||
description: Push scan findings to Prowler Cloud. Requires the PROWLER_CLOUD_API_KEY environment variable. See https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli
|
||||
description: Push scan findings to Prowler Cloud. Requires the PROWLER_CLOUD_API_KEY environment variable. See https://docs.prowler.com/user-guide/tutorials/prowler-import-findings#using-the-cli
|
||||
required: false
|
||||
default: "false"
|
||||
flags:
|
||||
@@ -299,7 +299,7 @@ runs:
|
||||
echo ""
|
||||
echo "**Get started in 3 steps:**"
|
||||
echo "1. Create an account at [cloud.prowler.com](https://cloud.prowler.com)"
|
||||
echo "2. Generate a Prowler Cloud API key ([docs](https://docs.prowler.com/user-guide/tutorials/prowler-app-import-findings#using-the-cli))"
|
||||
echo "2. Generate a Prowler Cloud API key ([docs](https://docs.prowler.com/user-guide/tutorials/prowler-import-findings#using-the-cli))"
|
||||
echo "3. Add \`PROWLER_CLOUD_API_KEY\` to your GitHub secrets and set \`push-to-cloud: true\` on this action"
|
||||
echo ""
|
||||
echo "See [prowler.com/pricing](https://prowler.com/pricing) for plan details."
|
||||
|
||||
+4
-4
@@ -10,7 +10,7 @@
|
||||
> - [`jsonapi`](../skills/jsonapi/SKILL.md) - Strict JSON:API v1.1 spec compliance
|
||||
> - [`pytest`](../skills/pytest/SKILL.md) - Generic pytest patterns
|
||||
|
||||
### Auto-invoke Skills
|
||||
## Auto-invoke Skills
|
||||
|
||||
When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
|
||||
@@ -81,7 +81,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
## DECISION TREES
|
||||
|
||||
### Serializer Selection
|
||||
```
|
||||
```text
|
||||
Read → <Model>Serializer
|
||||
Create → <Model>CreateSerializer
|
||||
Update → <Model>UpdateSerializer
|
||||
@@ -89,7 +89,7 @@ Nested read → <Model>IncludeSerializer
|
||||
```
|
||||
|
||||
### Task vs View
|
||||
```
|
||||
```text
|
||||
< 100ms → View
|
||||
> 100ms or external API → Celery task
|
||||
Needs retry → Celery task
|
||||
@@ -105,7 +105,7 @@ Django 5.1.x | DRF 3.15.x | djangorestframework-jsonapi 7.x | Celery 5.4.x | Pos
|
||||
|
||||
## PROJECT STRUCTURE
|
||||
|
||||
```
|
||||
```text
|
||||
api/src/backend/
|
||||
├── api/ # Main Django app
|
||||
│ ├── v1/ # API version 1 (views, serializers, urls)
|
||||
|
||||
+65
-1
@@ -2,6 +2,71 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
## [1.31.0] (Prowler UNRELEASED)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Opt-in automatic recovery of allowlisted idempotent background tasks whose worker died during a deploy or crash: when enabled via `DJANGO_TASK_RECOVERY_ENABLED` (off by default), stuck summary and deletion tasks are detected and re-run instead of staying pending forever (scan and Jira tasks are excluded), with a `reconcile_orphan_tasks` management command for on-demand recovery [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
- DORA compliance framework support [(#11131)](https://github.com/prowler-cloud/prowler/pull/11131)
|
||||
- Label Postgres connections with `application_name="<component>:<alias>"` (component injected per process via `DJANGO_APP_COMPONENT`) so connections are attributable by component in `pg_stat_activity` [(#11494)](https://github.com/prowler-cloud/prowler/pull/11494)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Allowlisted idempotent background tasks are no longer lost when a worker is stopped or crashes mid-task; tasks with external side effects are marked terminal instead of blindly re-running [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Workers now shut down gracefully on deploy or restart, finishing or re-queueing in-flight tasks instead of being force-killed and leaving them stuck [(#11416)](https://github.com/prowler-cloud/prowler/pull/11416)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- `dulwich` from 0.23.0 to 1.2.5 and `pyjwt` from 2.12.1 to 2.13.0, patching `GHSA-897w-fcg9-f6xj` (arbitrary file write) and `PYSEC-2026-179` (HMAC/JWK key confusion) flagged by osv-scanner in `api/uv.lock` [(#11499)](https://github.com/prowler-cloud/prowler/pull/11499)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.3] (Prowler v5.29.3)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- API startup no longer crashes when Neo4j is unreachable, as the Neo4j driver now connects lazily on first use rather than during app initialization [(#11491)](https://github.com/prowler-cloud/prowler/pull/11491)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.1] (Prowler v5.29.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `GET /api/v1/findings` N+1 query loading `resources__tags` when listing findings [(#11420)](https://github.com/prowler-cloud/prowler/pull/11420)
|
||||
- Clean up the scan tmp output directory when `scan-report` fails so partial files do not accumulate and fill the worker disk (`No space left on device`) [(#11421)](https://github.com/prowler-cloud/prowler/pull/11421)
|
||||
|
||||
---
|
||||
|
||||
## [1.30.0] (Prowler v5.29.0)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Scan finding ingestion: bulk-resolve `Resource`/`ResourceTag` rows, replace per-mapping `SELECT FOR UPDATE` with deferred `ResourceTagMapping.bulk_create(ignore_conflicts=True)`, wrap each micro-batch in a single `rls_transaction`, and raise `SCAN_DB_BATCH_SIZE` to 1000 [(#11249)](https://github.com/prowler-cloud/prowler/pull/11249)
|
||||
- Faster `GET /api/v1/finding-groups/latest` aggregation on tenants where one recent scan holds most findings [(#11380)](https://github.com/prowler-cloud/prowler/pull/11380)
|
||||
|
||||
---
|
||||
|
||||
## [1.29.1] (Prowler v5.28.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `finding-groups` slow response with finding-level filters such as `region`; check title and description are now read from the daily summaries, which drops sorting by `check_title` [(#11326)](https://github.com/prowler-cloud/prowler/pull/11326)
|
||||
|
||||
---
|
||||
|
||||
## [1.29.0] (Prowler v5.28.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- `okta` provider support [(#11184)](https://github.com/prowler-cloud/prowler/pull/11184)
|
||||
- `resource.metadata` attribute included in `/api/v1/findings?include=resources` [(#11187)](https://github.com/prowler-cloud/prowler/pull/11187)
|
||||
|
||||
---
|
||||
|
||||
## [1.28.0] (Prowler v5.27.0)
|
||||
|
||||
### 🚀 Added
|
||||
@@ -20,7 +85,6 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- `perform_scan_task` and `perform_scheduled_scan_task` now short-circuit with a warning and `return None` when the target provider no longer exists, instead of letting `handle_provider_deletion` raise `ProviderDeletedException`. `perform_scheduled_scan_task` also removes any orphan `PeriodicTask` it finds so beat stops re-firing scans for deleted providers. Prevents queued messages for deleted providers from being recorded as `FAILURE` [(#11185)](https://github.com/prowler-cloud/prowler/pull/11185)
|
||||
- Attack Paths: `BEDROCK-001` and `BEDROCK-002` now target roles trusting `bedrock-agentcore.amazonaws.com` instead of `bedrock.amazonaws.com`, eliminating false positives against regular Bedrock service roles (Agents, Knowledge Bases, model invocation) [(#11141)](https://github.com/prowler-cloud/prowler/pull/11141)
|
||||
|
||||
|
||||
---
|
||||
|
||||
## [1.27.1] (Prowler v5.26.1)
|
||||
|
||||
+4
-4
@@ -89,7 +89,7 @@ WORKDIR /home/prowler
|
||||
# Ensure output directory exists
|
||||
RUN mkdir -p /tmp/prowler_api_output
|
||||
|
||||
COPY pyproject.toml uv.lock ./
|
||||
COPY --chown=prowler:prowler pyproject.toml uv.lock ./
|
||||
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir uv==0.11.14
|
||||
@@ -97,13 +97,13 @@ RUN pip install --no-cache-dir --upgrade pip && \
|
||||
ENV PATH="/home/prowler/.local/bin:$PATH"
|
||||
|
||||
# Add `--no-install-project` to avoid installing the current project as a package
|
||||
RUN uv sync --no-install-project && \
|
||||
RUN uv sync --locked --no-install-project && \
|
||||
rm -rf ~/.cache/uv
|
||||
|
||||
RUN .venv/bin/python .venv/lib/python3.12/site-packages/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
COPY src/backend/ ./backend/
|
||||
COPY docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
COPY --chown=prowler:prowler src/backend/ ./backend/
|
||||
COPY --chown=prowler:prowler docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
|
||||
WORKDIR /home/prowler/backend
|
||||
|
||||
|
||||
+29
-29
@@ -2,7 +2,7 @@
|
||||
|
||||
This repository contains the JSON API and Task Runner components for Prowler, which facilitate a complete backend that interacts with the Prowler SDK and is used by the Prowler UI.
|
||||
|
||||
# Components
|
||||
## Components
|
||||
The Prowler API is composed of the following components:
|
||||
|
||||
- The JSON API, which is an API built with Django Rest Framework.
|
||||
@@ -10,13 +10,13 @@ The Prowler API is composed of the following components:
|
||||
- The PostgreSQL database, which is used to store the data.
|
||||
- The Valkey database, which is an in-memory database which is used as a message broker for the Celery workers.
|
||||
|
||||
## Note about Valkey
|
||||
### Note about Valkey
|
||||
|
||||
[Valkey](https://valkey.io/) is an open source (BSD) high performance key/value datastore.
|
||||
|
||||
Valkey exposes a Redis 7.2 compliant API. Any service that exposes the Redis API can be used with Prowler API.
|
||||
|
||||
# Modify environment variables
|
||||
## Modify environment variables
|
||||
|
||||
Under the root path of the project, you can find a file called `.env`. This file shows all the environment variables that the project uses. You should review it and set the values for the variables you want to change.
|
||||
|
||||
@@ -24,7 +24,7 @@ If you don’t set `DJANGO_TOKEN_SIGNING_KEY` or `DJANGO_TOKEN_VERIFYING_KEY`, t
|
||||
|
||||
**Important note**: Every Prowler version (or repository branches and tags) could have different variables set in its `.env` file. Please use the `.env` file that corresponds with each version.
|
||||
|
||||
## Local deployment
|
||||
### Local deployment
|
||||
Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the virtual environment, not before. Otherwise, variables will not be loaded properly.
|
||||
|
||||
To do this, you can run:
|
||||
@@ -34,12 +34,12 @@ set -a
|
||||
source .env
|
||||
```
|
||||
|
||||
# 🚀 Production deployment
|
||||
## Docker deployment
|
||||
## 🚀 Production deployment
|
||||
### Docker deployment
|
||||
|
||||
This method requires `docker` and `docker compose`.
|
||||
|
||||
### Clone the repository
|
||||
#### Clone the repository
|
||||
|
||||
```console
|
||||
# HTTPS
|
||||
@@ -50,13 +50,13 @@ git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
|
||||
### Build the base image
|
||||
#### Build the base image
|
||||
|
||||
```console
|
||||
docker compose --profile prod build
|
||||
```
|
||||
|
||||
### Run the production service
|
||||
#### Run the production service
|
||||
|
||||
This command will start the Django production server and the Celery worker and also the Valkey and PostgreSQL databases.
|
||||
|
||||
@@ -68,7 +68,7 @@ You can access the server in `http://localhost:8080`.
|
||||
|
||||
> **NOTE:** notice how the port is different. When developing using docker, the port will be `8080` to prevent conflicts.
|
||||
|
||||
### View the Production Server Logs
|
||||
#### View the Production Server Logs
|
||||
|
||||
To view the logs for any component (e.g., Django, Celery worker), you can use the following command with a wildcard. This command will follow logs for any container that matches the specified pattern:
|
||||
|
||||
@@ -133,13 +133,13 @@ gunicorn -c config/guniconf.py config.wsgi:application
|
||||
|
||||
> By default, the Gunicorn server will try to use as many workers as your machine can handle. You can manually change that in the `src/backend/config/guniconf.py` file.
|
||||
|
||||
# 🧪 Development guide
|
||||
## 🧪 Development guide
|
||||
|
||||
## Local deployment
|
||||
### Local deployment
|
||||
|
||||
To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `uv` and `docker compose` are installed.
|
||||
|
||||
### Clone the repository
|
||||
#### Clone the repository
|
||||
|
||||
```console
|
||||
# HTTPS
|
||||
@@ -150,7 +150,7 @@ git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
|
||||
### Start the PostgreSQL Database and Valkey
|
||||
#### Start the PostgreSQL Database and Valkey
|
||||
|
||||
The PostgreSQL database (version 16.3) and Valkey (version 7) are required for the development environment. To make development easier, we have provided a `docker-compose` file that will start these components for you.
|
||||
|
||||
@@ -161,7 +161,7 @@ The PostgreSQL database (version 16.3) and Valkey (version 7) are required for t
|
||||
docker compose up postgres valkey -d
|
||||
```
|
||||
|
||||
### Install the Python dependencies
|
||||
#### Install the Python dependencies
|
||||
|
||||
> You must have uv installed
|
||||
|
||||
@@ -169,7 +169,7 @@ docker compose up postgres valkey -d
|
||||
uv sync
|
||||
```
|
||||
|
||||
### Apply migrations
|
||||
#### Apply migrations
|
||||
|
||||
For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run:
|
||||
|
||||
@@ -178,7 +178,7 @@ cd src/backend
|
||||
python manage.py migrate --database admin
|
||||
```
|
||||
|
||||
### Run the Django development server
|
||||
#### Run the Django development server
|
||||
|
||||
```console
|
||||
cd src/backend
|
||||
@@ -188,7 +188,7 @@ python manage.py runserver
|
||||
You can access the server in `http://localhost:8000`.
|
||||
All changes in the code will be automatically reloaded in the server.
|
||||
|
||||
### Run the Celery worker
|
||||
#### Run the Celery worker
|
||||
|
||||
```console
|
||||
python -m celery -A config.celery worker -l info -E
|
||||
@@ -196,11 +196,11 @@ python -m celery -A config.celery worker -l info -E
|
||||
|
||||
The Celery worker does not detect and reload changes in the code, so you need to restart it manually when you make changes.
|
||||
|
||||
## Docker deployment
|
||||
### Docker deployment
|
||||
|
||||
This method requires `docker` and `docker compose`.
|
||||
|
||||
### Clone the repository
|
||||
#### Clone the repository
|
||||
|
||||
```console
|
||||
# HTTPS
|
||||
@@ -211,13 +211,13 @@ git clone git@github.com:prowler-cloud/api.git
|
||||
|
||||
```
|
||||
|
||||
### Build the base image
|
||||
#### Build the base image
|
||||
|
||||
```console
|
||||
docker compose --profile dev build
|
||||
```
|
||||
|
||||
### Run the development service
|
||||
#### Run the development service
|
||||
|
||||
This command will start the Django development server and the Celery worker and also the Valkey and PostgreSQL databases.
|
||||
|
||||
@@ -230,7 +230,7 @@ All changes in the code will be automatically reloaded in the server.
|
||||
|
||||
> **NOTE:** notice how the port is different. When developing using docker, the port will be `8080` to prevent conflicts.
|
||||
|
||||
### View the development server logs
|
||||
#### View the development server logs
|
||||
|
||||
To view the logs for any component (e.g., Django, Celery worker), you can use the following command with a wildcard. This command will follow logs for any container that matches the specified pattern:
|
||||
|
||||
@@ -238,7 +238,7 @@ To view the logs for any component (e.g., Django, Celery worker), you can use th
|
||||
docker logs -f $(docker ps --format "{{.Names}}" | grep 'api-')
|
||||
```
|
||||
|
||||
## Applying migrations
|
||||
### Applying migrations
|
||||
|
||||
For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run:
|
||||
|
||||
@@ -247,7 +247,7 @@ cd src/backend
|
||||
uv run python manage.py migrate --database admin
|
||||
```
|
||||
|
||||
## Apply fixtures
|
||||
### Apply fixtures
|
||||
|
||||
Fixtures are used to populate the database with initial development data.
|
||||
|
||||
@@ -258,7 +258,7 @@ uv run python manage.py loaddata api/fixtures/0_dev_users.json --database admin
|
||||
|
||||
> The default credentials are `dev@prowler.com:Thisisapassword123@` or `dev2@prowler.com:Thisisapassword123@`
|
||||
|
||||
## Run tests
|
||||
### Run tests
|
||||
|
||||
Note that the tests will fail if you use the same `.env` file as the development environment.
|
||||
|
||||
@@ -269,7 +269,7 @@ cd src/backend
|
||||
uv run pytest
|
||||
```
|
||||
|
||||
# Custom commands
|
||||
## Custom commands
|
||||
|
||||
Django provides a way to create custom commands that can be run from the command line.
|
||||
|
||||
@@ -281,7 +281,7 @@ To run a custom command, you need to be in the `prowler/api/src/backend` directo
|
||||
uv run python manage.py <command_name>
|
||||
```
|
||||
|
||||
## Generate dummy data
|
||||
### Generate dummy data
|
||||
|
||||
```console
|
||||
python manage.py findings --tenant
|
||||
@@ -298,7 +298,7 @@ This command creates, for a given tenant, a provider, scan and a set of findings
|
||||
>
|
||||
> The last step is required to access the findings details, since the UI needs that to print all the information.
|
||||
|
||||
### Example
|
||||
#### Example
|
||||
|
||||
```console
|
||||
~/backend $ uv run python manage.py findings --tenant
|
||||
|
||||
@@ -22,12 +22,12 @@ apply_fixtures() {
|
||||
|
||||
start_dev_server() {
|
||||
echo "Starting the development server..."
|
||||
uv run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
exec uv run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
|
||||
}
|
||||
|
||||
start_prod_server() {
|
||||
echo "Starting the Gunicorn server..."
|
||||
uv run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
exec uv run gunicorn -c config/guniconf.py config.wsgi:application
|
||||
}
|
||||
|
||||
resolve_worker_hostname() {
|
||||
@@ -47,7 +47,7 @@ resolve_worker_hostname() {
|
||||
|
||||
start_worker() {
|
||||
echo "Starting the worker..."
|
||||
uv run python -m celery -A config.celery worker \
|
||||
exec uv run python -m celery -A config.celery worker \
|
||||
-n "$(resolve_worker_hostname)" \
|
||||
-l "${DJANGO_LOGGING_LEVEL:-info}" \
|
||||
-Q celery,scans,scan-reports,deletion,backfill,overview,integrations,compliance,attack-paths-scans \
|
||||
@@ -56,7 +56,7 @@ start_worker() {
|
||||
|
||||
start_worker_beat() {
|
||||
echo "Starting the worker-beat..."
|
||||
uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
exec uv run python -m celery -A config.celery beat -l "${DJANGO_LOGGING_LEVEL:-info}" --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
}
|
||||
|
||||
manage_db_partitions() {
|
||||
@@ -68,6 +68,15 @@ manage_db_partitions() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Identify this process to Postgres (application_name=<component>:<alias>) so
|
||||
# connections are attributable by component in pg_stat_activity. Web tiers
|
||||
# report "api"; everything else uses the launch subcommand.
|
||||
case "$1" in
|
||||
prod|dev) DJANGO_APP_COMPONENT="api" ;;
|
||||
*) DJANGO_APP_COMPONENT="$1" ;;
|
||||
esac
|
||||
export DJANGO_APP_COMPONENT
|
||||
|
||||
case "$1" in
|
||||
dev)
|
||||
apply_migrations
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# Orphan Celery task recovery
|
||||
|
||||
When a worker is terminated mid-task (a deploy, an OOM kill, a node eviction), the
|
||||
task it was running can be left non-terminal forever: the `TaskResult` stays
|
||||
`STARTED` and nothing re-runs it. This page describes the mechanisms that detect and
|
||||
recover allowlisted idempotent orphans so pending-task alerts do not fire. Scan tasks
|
||||
are not auto-recovered (re-running a scan is not safe to do automatically); the
|
||||
watchdog covers the summary/aggregation and deletion tasks.
|
||||
|
||||
## How recovery works
|
||||
|
||||
1. **Durable delivery.** The broker is configured so a task message is acknowledged
|
||||
only after the task finishes (`task_acks_late`), one task is reserved at a time
|
||||
(`worker_prefetch_multiplier = 1`), and an abruptly-lost worker re-queues its task
|
||||
(`task_reject_on_worker_lost`). On `SIGTERM` the worker is given a soft-shutdown
|
||||
window (`worker_soft_shutdown_timeout`) to finish or re-queue in-flight work
|
||||
before it is force-killed. `scan-perform`, `scan-perform-scheduled` and
|
||||
`integration-jira` opt out of redelivery with `acks_late=False`, so a crash drops
|
||||
them rather than re-running and duplicating findings or Jira issues. Other
|
||||
non-recovered side-effect tasks keep `acks_late=True`, so the broker can still
|
||||
re-deliver them after a worker loss: the S3 upload rebuilds from worker-local files
|
||||
that did not survive the crash and so no-ops, but Security Hub re-reads findings from
|
||||
the DB and re-sends them to AWS.
|
||||
|
||||
2. **Periodic watchdog.** A Beat task, `reconcile-orphan-tasks`, runs every couple of
|
||||
minutes (a `django_celery_beat` periodic task created by migration). For each
|
||||
in-flight task result with an allowlisted idempotent task name, it pings the
|
||||
worker recorded on the task's `TaskResult`:
|
||||
- worker responds -> the task is still running, leave it alone;
|
||||
- worker is gone (and the task started before a short grace window) -> it is a
|
||||
real orphan: the stale task is revoked and marked terminal (clearing the
|
||||
pending/started alert), and the task is re-enqueued from its stored name and
|
||||
kwargs.
|
||||
|
||||
The re-run is safe because only tasks with proven idempotency are allowlisted: the
|
||||
summary/aggregation tasks clear and re-write their own rows, and deletions are
|
||||
idempotent. Scan tasks and external side effects are excluded: re-running a scan is
|
||||
not safe to do automatically, Jira sends would create duplicate issues, the S3
|
||||
upload rebuilds from worker-local files that do not survive a crash, and
|
||||
report/Security Hub recovery is out of scope.
|
||||
|
||||
3. **Recovery cap.** A per-task Valkey counter limits how often the same task is
|
||||
re-enqueued. After `--max-attempts` recoveries (default 3) the orphan is marked
|
||||
terminal instead of re-enqueued, so a task that repeatedly kills its worker cannot
|
||||
loop forever.
|
||||
|
||||
A Postgres advisory lock ensures that, even with multiple API/worker replicas, only
|
||||
one reconciliation runs at a time; the others no-op.
|
||||
|
||||
## On-demand command
|
||||
|
||||
The same logic is available as a management command, useful right after a deploy or
|
||||
for manual intervention:
|
||||
|
||||
```bash
|
||||
python manage.py reconcile_orphan_tasks # recover now
|
||||
python manage.py reconcile_orphan_tasks --dry-run # report orphans, change nothing
|
||||
python manage.py reconcile_orphan_tasks --grace-minutes 5 --max-attempts 3
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
All settings have safe defaults; override via environment variables.
|
||||
|
||||
| Env var | Default | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `DJANGO_CELERY_WORKER_PREFETCH_MULTIPLIER` | `1` | Tasks reserved per worker process. |
|
||||
| `DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT` | `60` | Seconds the worker drains/re-queues on `SIGTERM` before force-kill. |
|
||||
| `DJANGO_CELERY_TASK_TIME_LIMIT` | `21600` (6h) | Hard limit for most tasks; connection checks are capped at 120s. |
|
||||
| `DJANGO_CELERY_TASK_SOFT_TIME_LIMIT` | hard - 600 | Soft limit; raises `SoftTimeLimitExceeded` for cleanup. |
|
||||
| `DJANGO_CELERY_LONG_TASK_TIME_LIMIT` | `172800` (48h) | Hard limit for scans and provider/tenant deletions, which can legitimately run for more than a day. |
|
||||
| `DJANGO_CELERY_LONG_TASK_SOFT_TIME_LIMIT` | long hard - 600 | Soft limit for the long-running tasks above. |
|
||||
| `DJANGO_TASK_RECOVERY_ENABLED` | `false` | Master switch for orphan-task recovery, disabled by default (opt-in); set to `true` to enable. When off, no orphan is detected, marked terminal, or re-enqueued (attack-paths stale cleanup still runs). |
|
||||
| `DJANGO_TASK_RECOVERY_SUMMARIES_ENABLED` | `true` | Auto re-enqueue orphaned scan summary/aggregation tasks. |
|
||||
| `DJANGO_TASK_RECOVERY_DELETIONS_ENABLED` | `true` | Auto re-enqueue orphaned provider/tenant deletion tasks. |
|
||||
|
||||
Recovery is opt-in: with the master flag off (the default) the sweep does nothing.
|
||||
Once enabled, the per-group flags default to on, so every group recovers unless you
|
||||
turn one off; a task whose group flag is off is marked terminal instead of
|
||||
re-enqueued.
|
||||
|
||||
Turning recovery off only disables this watchdog sweep; it does not change Celery's
|
||||
broker-level redelivery (`task_acks_late`/`task_reject_on_worker_lost`), which still
|
||||
re-delivers tasks that keep `acks_late=True` on worker loss, independently of this flag.
|
||||
|
||||
`task_acks_late` and `task_reject_on_worker_lost` are enabled in `config/celery.py`.
|
||||
|
||||
## Deployment requirement
|
||||
|
||||
Two conditions must both hold for the soft shutdown to actually drain work:
|
||||
|
||||
1. **The worker must receive `SIGTERM`.** The container entrypoint `exec`s the
|
||||
Celery process so it runs as PID 1; otherwise `SIGTERM` from `docker stop`/ECS
|
||||
hits the entrypoint shell, never reaches Celery, and the worker is hard-killed
|
||||
(SIGKILL) at the grace deadline without draining. Custom entrypoints must
|
||||
preserve the `exec`.
|
||||
2. **The orchestrator must give the worker enough time** before force-killing it.
|
||||
Set the stop grace period to exceed `DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT`
|
||||
plus a margin:
|
||||
- **docker-compose:** `stop_grace_period` on the worker services (set to `120s`).
|
||||
- **AWS ECS:** the worker container `stopTimeout` (configured in the deployment
|
||||
repository).
|
||||
|
||||
If either condition is missing, long tasks are still recovered by the watchdog,
|
||||
but they are cut mid-run on every deploy instead of draining.
|
||||
+14
-4
@@ -68,7 +68,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.28.0"
|
||||
version = "1.31.0"
|
||||
|
||||
[tool.uv]
|
||||
# Transitive pins matching master to avoid silent drift; bump deliberately.
|
||||
@@ -226,7 +226,7 @@ constraint-dependencies = [
|
||||
"drf-simple-apikey==2.2.1",
|
||||
"drf-spectacular==0.27.2",
|
||||
"drf-spectacular-jsonapi==0.5.1",
|
||||
"dulwich==0.23.0",
|
||||
"dulwich==1.2.5",
|
||||
"duo-client==5.5.0",
|
||||
"durationpy==0.10",
|
||||
"email-validator==2.2.0",
|
||||
@@ -354,7 +354,7 @@ constraint-dependencies = [
|
||||
"pydantic-core==2.41.5",
|
||||
"pygithub==2.8.0",
|
||||
"pygments==2.20.0",
|
||||
"pyjwt==2.12.1",
|
||||
"pyjwt==2.13.0",
|
||||
"pylint==3.2.5",
|
||||
"pymsalruntime==0.18.1",
|
||||
"pynacl==1.6.2",
|
||||
@@ -443,7 +443,17 @@ constraint-dependencies = [
|
||||
# The microsoft-kiota-http security bump to 1.9.9 (GHSA-7j59-v9qr-6fq9) requires
|
||||
# microsoft-kiota-abstractions>=1.9.9, which a constraint cannot satisfy against the
|
||||
# SDK's hard pin; override it to the patched, kiota-aligned version.
|
||||
#
|
||||
# prowler@master hard-pins dulwich==0.23.0 and pyjwt==2.12.1 in [project.dependencies].
|
||||
# dulwich 1.2.5 patches GHSA-897w-fcg9-f6xj (arbitrary file write) and pyjwt 2.13.0
|
||||
# patches PYSEC-2026-179 (HMAC/JWK key-confusion); a constraint cannot satisfy these
|
||||
# against the SDK's hard pins, so override them to the patched versions until the SDK
|
||||
# bump propagates to the pinned master rev. pyjwt keeps the [crypto] extra because an
|
||||
# override replaces the whole requirement; bare pyjwt would drop it from the consumers
|
||||
# that request pyjwt[crypto] and leave cryptography (needed for RS256) only transitive.
|
||||
override-dependencies = [
|
||||
"okta==3.4.2",
|
||||
"microsoft-kiota-abstractions==1.9.9"
|
||||
"microsoft-kiota-abstractions==1.9.9",
|
||||
"dulwich==1.2.5",
|
||||
"pyjwt[crypto]==2.13.0"
|
||||
]
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
@@ -30,7 +32,6 @@ class ApiConfig(AppConfig):
|
||||
def ready(self):
|
||||
from api import schema_extensions # noqa: F401
|
||||
from api import signals # noqa: F401
|
||||
from api.attack_paths import database as graph_database
|
||||
|
||||
# Generate required cryptographic keys if not present, but only if:
|
||||
# `"manage.py" not in sys.argv[0]`: If an external server (e.g., Gunicorn) is running the app
|
||||
@@ -41,37 +42,8 @@ class ApiConfig(AppConfig):
|
||||
):
|
||||
self._ensure_crypto_keys()
|
||||
|
||||
# Commands that don't need Neo4j
|
||||
SKIP_NEO4J_DJANGO_COMMANDS = [
|
||||
"makemigrations",
|
||||
"migrate",
|
||||
"pgpartition",
|
||||
"check",
|
||||
"help",
|
||||
"showmigrations",
|
||||
"check_and_fix_socialaccount_sites_migration",
|
||||
]
|
||||
|
||||
# Skip eager Neo4j init for tests, some Django commands, and Celery (prefork pool: driver must stay lazy, no post_fork hook)
|
||||
if getattr(settings, "TESTING", False) or (
|
||||
len(sys.argv) > 1
|
||||
and (
|
||||
(
|
||||
"manage.py" in sys.argv[0]
|
||||
and sys.argv[1] in SKIP_NEO4J_DJANGO_COMMANDS
|
||||
)
|
||||
or "celery" in sys.argv[0]
|
||||
)
|
||||
):
|
||||
logger.info(
|
||||
"Skipping eager Neo4j init: tests, some Django commands, or Celery prefork pool (driver stays lazy)"
|
||||
)
|
||||
|
||||
else:
|
||||
graph_database.init_driver()
|
||||
|
||||
# Neo4j driver is initialized at API startup (see api.attack_paths.database)
|
||||
# It remains lazy for Celery workers and selected Django commands
|
||||
# Neo4j driver is created lazily on first use (see api.attack_paths.database).
|
||||
# App init never contacts Neo4j, so a Neo4j outage cannot block API startup.
|
||||
|
||||
def _ensure_crypto_keys(self):
|
||||
"""
|
||||
|
||||
@@ -1,22 +1,24 @@
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Iterator
|
||||
from uuid import UUID
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
|
||||
# Without this Celery goes crazy with Neo4j logging
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
@@ -28,6 +30,9 @@ READ_QUERY_TIMEOUT_SECONDS = env.int(
|
||||
"ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30
|
||||
)
|
||||
MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250)
|
||||
# Shorter than CONN_ACQUISITION_TIMEOUT — the driver requires acquisition to be
|
||||
# the longer of the two (it may include opening a new connection).
|
||||
CONNECTION_TIMEOUT = env.int("NEO4J_CONNECTION_TIMEOUT", default=5)
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15)
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
@@ -58,15 +63,24 @@ def init_driver() -> neo4j.Driver:
|
||||
uri = get_uri()
|
||||
config = settings.DATABASES["neo4j"]
|
||||
|
||||
_driver = neo4j.GraphDatabase.driver(
|
||||
driver = neo4j.GraphDatabase.driver(
|
||||
uri,
|
||||
auth=(config["USER"], config["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=7200,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=50,
|
||||
)
|
||||
_driver.verify_connectivity()
|
||||
# Publish the singleton only after connectivity is verified so a
|
||||
# failed probe does not leave an unverified driver behind. Close the
|
||||
# driver on failure so a repeatedly-probed outage cannot leak pools.
|
||||
try:
|
||||
driver.verify_connectivity()
|
||||
except Exception:
|
||||
driver.close()
|
||||
raise
|
||||
_driver = driver
|
||||
|
||||
# Register cleanup handler (only runs once since we're inside the _driver is None block)
|
||||
atexit.register(close_driver)
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
from api.models import Provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.check.models import CheckMetadata
|
||||
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS = {}
|
||||
@@ -94,25 +96,22 @@ PROWLER_CHECKS = LazyChecksMapping()
|
||||
|
||||
|
||||
def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]:
|
||||
"""List compliance frameworks the API can load for `provider_type`.
|
||||
"""List compliance framework identifiers available for `provider_type`.
|
||||
|
||||
The list is sourced from `Compliance.get_bulk` so that the names
|
||||
returned here are guaranteed to be loadable by the bulk loader. This
|
||||
prevents downstream key mismatches (e.g. CSV report generation iterating
|
||||
framework names and looking them up in the bulk dict).
|
||||
Includes both per-provider frameworks and universal top-level frameworks
|
||||
(e.g. ``dora``, ``csa_ccm_4.0``).
|
||||
|
||||
Args:
|
||||
provider_type (Provider.ProviderChoices): The cloud provider type for which to retrieve
|
||||
available compliance frameworks (e.g., "aws", "azure", "gcp", "m365").
|
||||
provider_type (Provider.ProviderChoices): The cloud provider type
|
||||
(e.g., "aws", "azure", "gcp", "m365").
|
||||
|
||||
Returns:
|
||||
list[str]: A list of framework identifiers (e.g., "cis_1.4_aws", "mitre_attack_azure") available
|
||||
for the given provider.
|
||||
list[str]: Framework identifiers (e.g., "cis_1.4_aws", "dora").
|
||||
"""
|
||||
global AVAILABLE_COMPLIANCE_FRAMEWORKS
|
||||
if provider_type not in AVAILABLE_COMPLIANCE_FRAMEWORKS:
|
||||
AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type] = list(
|
||||
Compliance.get_bulk(provider_type).keys()
|
||||
get_bulk_compliance_frameworks_universal(provider_type).keys()
|
||||
)
|
||||
|
||||
return AVAILABLE_COMPLIANCE_FRAMEWORKS[provider_type]
|
||||
@@ -139,18 +138,14 @@ def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) ->
|
||||
"""
|
||||
Retrieve the Prowler compliance data for a specified provider type.
|
||||
|
||||
This function fetches the compliance frameworks and their associated
|
||||
requirements for the given cloud provider.
|
||||
|
||||
Args:
|
||||
provider_type (Provider.ProviderChoices): The provider type
|
||||
(e.g., 'aws', 'azure') for which to retrieve compliance data.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary mapping compliance framework names to their respective
|
||||
Compliance objects for the specified provider.
|
||||
dict: Mapping of framework name to `ComplianceFramework` for the provider.
|
||||
"""
|
||||
return Compliance.get_bulk(provider_type)
|
||||
return get_bulk_compliance_frameworks_universal(provider_type)
|
||||
|
||||
|
||||
def _load_provider_assets(provider_type: Provider.ProviderChoices) -> tuple[dict, dict]:
|
||||
@@ -209,8 +204,8 @@ def load_prowler_checks(
|
||||
for compliance_name, compliance_data in prowler_compliance.get(
|
||||
provider_type, {}
|
||||
).items():
|
||||
for requirement in compliance_data.Requirements:
|
||||
for check in requirement.Checks:
|
||||
for requirement in compliance_data.requirements:
|
||||
for check in requirement.checks.get(provider_type, []):
|
||||
try:
|
||||
checks[provider_type][check].add(compliance_name)
|
||||
except KeyError:
|
||||
@@ -290,24 +285,40 @@ def generate_compliance_overview_template(
|
||||
requirements_status = {"passed": 0, "failed": 0, "manual": 0}
|
||||
total_requirements = 0
|
||||
|
||||
for requirement in compliance_data.Requirements:
|
||||
for requirement in compliance_data.requirements:
|
||||
total_requirements += 1
|
||||
total_checks = len(requirement.Checks)
|
||||
checks_dict = {check: None for check in requirement.Checks}
|
||||
provider_check_list = list(requirement.checks.get(provider_type, []))
|
||||
total_checks = len(provider_check_list)
|
||||
checks_dict = {check: None for check in provider_check_list}
|
||||
|
||||
req_status_val = "MANUAL" if total_checks == 0 else "PASS"
|
||||
|
||||
# MITRE attrs are wrapped under `_raw_attributes` by the
|
||||
# universal adapter — unwrap so consumers see the flat list.
|
||||
requirement_attributes = requirement.attributes
|
||||
if (
|
||||
isinstance(requirement_attributes, dict)
|
||||
and "_raw_attributes" in requirement_attributes
|
||||
):
|
||||
attributes_payload = list(requirement_attributes["_raw_attributes"])
|
||||
elif isinstance(requirement_attributes, dict):
|
||||
attributes_payload = (
|
||||
[dict(requirement_attributes)] if requirement_attributes else []
|
||||
)
|
||||
else:
|
||||
attributes_payload = [
|
||||
dict(attribute) for attribute in requirement_attributes
|
||||
]
|
||||
|
||||
# Build requirement dictionary
|
||||
requirement_dict = {
|
||||
"name": requirement.Name or requirement.Id,
|
||||
"description": requirement.Description,
|
||||
"tactics": getattr(requirement, "Tactics", []),
|
||||
"subtechniques": getattr(requirement, "SubTechniques", []),
|
||||
"platforms": getattr(requirement, "Platforms", []),
|
||||
"technique_url": getattr(requirement, "TechniqueURL", ""),
|
||||
"attributes": [
|
||||
dict(attribute) for attribute in requirement.Attributes
|
||||
],
|
||||
"name": requirement.name or requirement.id,
|
||||
"description": requirement.description,
|
||||
"tactics": requirement.tactics or [],
|
||||
"subtechniques": requirement.sub_techniques or [],
|
||||
"platforms": requirement.platforms or [],
|
||||
"technique_url": requirement.technique_url or "",
|
||||
"attributes": attributes_payload,
|
||||
"checks": checks_dict,
|
||||
"checks_status": {
|
||||
"pass": 0,
|
||||
@@ -325,15 +336,15 @@ def generate_compliance_overview_template(
|
||||
requirements_status["passed"] += 1
|
||||
|
||||
# Add requirement to compliance requirements
|
||||
compliance_requirements[requirement.Id] = requirement_dict
|
||||
compliance_requirements[requirement.id] = requirement_dict
|
||||
|
||||
# Build compliance dictionary
|
||||
compliance_dict = {
|
||||
"framework": compliance_data.Framework,
|
||||
"name": compliance_data.Name,
|
||||
"version": compliance_data.Version,
|
||||
"framework": compliance_data.framework,
|
||||
"name": compliance_data.name,
|
||||
"version": compliance_data.version,
|
||||
"provider": provider_type,
|
||||
"description": compliance_data.Description,
|
||||
"description": compliance_data.description,
|
||||
"requirements": compliance_requirements,
|
||||
"requirements_status": requirements_status,
|
||||
"total_requirements": total_requirements,
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from tasks.jobs.orphan_recovery import reconcile_orphans
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = (
|
||||
"Recover orphaned allowlisted Celery tasks whose worker is gone and mark "
|
||||
"other stale task results terminal. Single-flight via a Postgres advisory lock."
|
||||
)
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--grace-minutes",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Skip tasks started within this window (worker may still register).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-attempts",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Give up re-running a task after this many recovery attempts; it is then left terminal instead of re-enqueued.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Detect and report orphans without revoking or re-enqueuing.",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
result = reconcile_orphans(
|
||||
grace_minutes=options["grace_minutes"],
|
||||
max_attempts=options["max_attempts"],
|
||||
dry_run=options["dry_run"],
|
||||
)
|
||||
|
||||
if not result.get("acquired"):
|
||||
self.stdout.write("Reconcile skipped: another run holds the lock.")
|
||||
return
|
||||
|
||||
if result.get("enabled") is False:
|
||||
message = (
|
||||
"Task recovery is disabled (DJANGO_TASK_RECOVERY_ENABLED is off); "
|
||||
"no orphans were recovered."
|
||||
)
|
||||
if result.get("attack_paths") is not None:
|
||||
message += " Attack-paths stale cleanup still ran."
|
||||
self.stdout.write(message)
|
||||
return
|
||||
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
"Orphan reconcile complete: "
|
||||
f"recovered={len(result.get('recovered', []))} "
|
||||
f"failed={len(result.get('failed', []))} "
|
||||
f"skipped(in-flight)={len(result.get('skipped', []))}"
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
from django.db import migrations
|
||||
|
||||
import api.db_utils
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0092_findings_arrays_gin_index_parent"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="provider",
|
||||
name="provider",
|
||||
field=api.db_utils.ProviderEnumField(
|
||||
choices=[
|
||||
("aws", "AWS"),
|
||||
("azure", "Azure"),
|
||||
("gcp", "GCP"),
|
||||
("kubernetes", "Kubernetes"),
|
||||
("m365", "M365"),
|
||||
("github", "GitHub"),
|
||||
("mongodbatlas", "MongoDB Atlas"),
|
||||
("iac", "IaC"),
|
||||
("oraclecloud", "Oracle Cloud Infrastructure"),
|
||||
("alibabacloud", "Alibaba Cloud"),
|
||||
("cloudflare", "Cloudflare"),
|
||||
("openstack", "OpenStack"),
|
||||
("image", "Image"),
|
||||
("googleworkspace", "Google Workspace"),
|
||||
("vercel", "Vercel"),
|
||||
("okta", "Okta"),
|
||||
],
|
||||
default="aws",
|
||||
),
|
||||
),
|
||||
migrations.RunSQL(
|
||||
"ALTER TYPE provider ADD VALUE IF NOT EXISTS 'okta';",
|
||||
reverse_sql=migrations.RunSQL.noop,
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,49 @@
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
TASK_NAME = "reconcile-orphan-tasks"
|
||||
INTERVAL_MINUTES = 2
|
||||
|
||||
|
||||
def create_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
schedule, _ = IntervalSchedule.objects.get_or_create(
|
||||
every=INTERVAL_MINUTES,
|
||||
period="minutes",
|
||||
)
|
||||
|
||||
PeriodicTask.objects.update_or_create(
|
||||
name=TASK_NAME,
|
||||
defaults={
|
||||
"task": TASK_NAME,
|
||||
"interval": schedule,
|
||||
"enabled": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def delete_periodic_task(apps, schema_editor):
|
||||
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
|
||||
PeriodicTask.objects.filter(name=TASK_NAME).delete()
|
||||
|
||||
# Clean up the schedule if no other task references it
|
||||
IntervalSchedule.objects.filter(
|
||||
every=INTERVAL_MINUTES,
|
||||
period="minutes",
|
||||
periodictask__isnull=True,
|
||||
).delete()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0093_okta_provider"),
|
||||
("django_celery_beat", "0019_alter_periodictasks_options"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(create_periodic_task, delete_periodic_task),
|
||||
]
|
||||
@@ -296,6 +296,7 @@ class Provider(RowLevelSecurityProtectedModel):
|
||||
IMAGE = "image", _("Image")
|
||||
GOOGLEWORKSPACE = "googleworkspace", _("Google Workspace")
|
||||
VERCEL = "vercel", _("Vercel")
|
||||
OKTA = "okta", _("Okta")
|
||||
|
||||
@staticmethod
|
||||
def validate_aws_uid(value):
|
||||
@@ -354,6 +355,26 @@ class Provider(RowLevelSecurityProtectedModel):
|
||||
pointer="/data/attributes/uid",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def validate_okta_uid(value):
|
||||
if not re.match(
|
||||
r"^[a-z0-9][a-z0-9-]*\.("
|
||||
r"okta\.com|oktapreview\.com|okta-emea\.com|"
|
||||
r"okta-gov\.com|okta\.mil|okta-miltest\.com|trex-govcloud\.com"
|
||||
r")$",
|
||||
value,
|
||||
):
|
||||
raise ModelValidationError(
|
||||
detail=(
|
||||
"Okta provider ID must be a valid Okta-managed org domain "
|
||||
"(e.g., acme.okta.com, also .oktapreview.com / .okta-emea.com "
|
||||
"/ .okta-gov.com / .okta.mil / .okta-miltest.com / "
|
||||
".trex-govcloud.com), without scheme or path."
|
||||
),
|
||||
code="okta-uid",
|
||||
pointer="/data/attributes/uid",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def validate_kubernetes_uid(value):
|
||||
if not re.match(
|
||||
@@ -480,6 +501,12 @@ class Provider(RowLevelSecurityProtectedModel):
|
||||
|
||||
def clean(self):
|
||||
super().clean()
|
||||
if self.provider == self.ProviderChoices.OKTA and self.uid:
|
||||
# Mirror the SDK, which lowercases the org domain before connecting.
|
||||
# Without this the API would reject Acme.okta.com even though the
|
||||
# SDK would accept it, and stored uids could disagree with the
|
||||
# authenticated org domain.
|
||||
self.uid = self.uid.strip().lower()
|
||||
getattr(self, f"validate_{self.provider}_uid")(self.uid)
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.28.0
|
||||
version: 1.31.0
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
@@ -373,6 +373,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -389,6 +390,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -412,6 +414,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -430,6 +433,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -1453,6 +1457,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -1469,6 +1474,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -1491,6 +1497,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -1509,6 +1516,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -1997,6 +2005,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -2013,6 +2022,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -2035,6 +2045,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -2053,6 +2064,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -2584,6 +2596,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -2600,6 +2613,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -2622,6 +2636,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -2640,6 +2655,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -3134,6 +3150,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -3150,6 +3167,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -3173,6 +3191,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -3191,6 +3210,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -3740,6 +3760,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -3756,6 +3777,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -3779,6 +3801,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -3797,6 +3820,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -4254,6 +4278,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -4270,6 +4295,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -4293,6 +4319,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -4311,6 +4338,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -4766,6 +4794,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -4782,6 +4811,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -4805,6 +4835,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -4823,6 +4854,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -5266,6 +5298,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -5282,6 +5315,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -5305,6 +5339,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -5323,6 +5358,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -7156,6 +7192,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -7172,6 +7209,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -7195,6 +7233,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -7213,6 +7252,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- name: filter[search]
|
||||
@@ -7335,6 +7375,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -7351,6 +7392,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -7374,6 +7416,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -7392,6 +7435,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- name: filter[search]
|
||||
@@ -7503,6 +7547,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -7519,6 +7564,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -7541,6 +7587,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -7559,6 +7606,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- name: filter[search]
|
||||
@@ -7702,6 +7750,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -7718,6 +7767,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -7741,6 +7791,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -7759,6 +7810,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -7915,6 +7967,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -7931,6 +7984,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -7954,6 +8008,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -7972,6 +8027,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -8122,6 +8178,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -8138,6 +8195,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -8160,6 +8218,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -8178,6 +8237,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- name: filter[search]
|
||||
@@ -8370,6 +8430,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -8386,6 +8447,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -8409,6 +8471,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -8427,6 +8490,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -8548,6 +8612,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -8564,6 +8629,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -8587,6 +8653,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -8605,6 +8672,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -8750,6 +8818,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -8766,6 +8835,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -8789,6 +8859,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -8807,6 +8878,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -9593,6 +9665,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -9609,6 +9682,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider__in]
|
||||
schema:
|
||||
@@ -9632,6 +9706,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -9650,6 +9725,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -9673,6 +9749,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -9689,6 +9766,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -9712,6 +9790,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -9730,6 +9809,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- name: filter[search]
|
||||
@@ -10400,6 +10480,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -10416,6 +10497,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -10439,6 +10521,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -10457,6 +10540,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -10951,6 +11035,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -10967,6 +11052,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -10990,6 +11076,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -11008,6 +11095,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -11315,6 +11403,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -11331,6 +11420,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -11354,6 +11444,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -11372,6 +11463,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -11685,6 +11777,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -11701,6 +11794,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -11724,6 +11818,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -11742,6 +11837,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -12580,6 +12676,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
* `azure` - Azure
|
||||
@@ -12596,6 +12693,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
- in: query
|
||||
name: filter[provider_type__in]
|
||||
schema:
|
||||
@@ -12619,6 +12717,7 @@ paths:
|
||||
- openstack
|
||||
- oraclecloud
|
||||
- vercel
|
||||
- okta
|
||||
description: |-
|
||||
Multiple values may be separated by commas.
|
||||
|
||||
@@ -12637,6 +12736,7 @@ paths:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
@@ -13037,8 +13137,59 @@ paths:
|
||||
responses:
|
||||
'200':
|
||||
description: CSV file containing the compliance report
|
||||
'202':
|
||||
description: The task is in progress
|
||||
'403':
|
||||
description: There is a problem with credentials
|
||||
'404':
|
||||
description: Compliance report not found
|
||||
description: Compliance report not found, or the scan has no reports yet
|
||||
/api/v1/scans/{id}/compliance/{name}/ocsf:
|
||||
get:
|
||||
operationId: scans_compliance_ocsf_retrieve
|
||||
description: Download a specific compliance report as an OCSF JSON file. Only
|
||||
universal frameworks that declare an output configuration produce this artifact
|
||||
(currently 'dora' and 'csa_ccm_4.0'); any other framework returns 404.
|
||||
summary: Retrieve compliance report as OCSF JSON
|
||||
parameters:
|
||||
- in: query
|
||||
name: fields[scan-reports]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum:
|
||||
- id
|
||||
- name
|
||||
description: endpoint return only specific fields in the response on a per-type
|
||||
basis by including a fields[TYPE] query parameter.
|
||||
explode: false
|
||||
- in: path
|
||||
name: id
|
||||
schema:
|
||||
type: string
|
||||
format: uuid
|
||||
description: A UUID string identifying this scan.
|
||||
required: true
|
||||
- in: path
|
||||
name: name
|
||||
schema:
|
||||
type: string
|
||||
description: The compliance report name, like 'dora'
|
||||
required: true
|
||||
tags:
|
||||
- Scan
|
||||
security:
|
||||
- JWT or API Key: []
|
||||
responses:
|
||||
'200':
|
||||
description: OCSF JSON file containing the compliance report
|
||||
'202':
|
||||
description: The task is in progress
|
||||
'403':
|
||||
description: There is a problem with credentials
|
||||
'404':
|
||||
description: Compliance report not found, the framework does not provide
|
||||
an OCSF export, or the scan has no reports yet
|
||||
/api/v1/scans/{id}/csa:
|
||||
get:
|
||||
operationId: scans_csa_retrieve
|
||||
@@ -20115,6 +20266,23 @@ components:
|
||||
required:
|
||||
- clouds_yaml_content
|
||||
- clouds_yaml_cloud
|
||||
- type: object
|
||||
title: Okta OAuth Credentials
|
||||
properties:
|
||||
okta_client_id:
|
||||
type: string
|
||||
description: Client ID of the Okta API Services app used for OAuth 2.0 private-key JWT authentication.
|
||||
okta_private_key:
|
||||
type: string
|
||||
description: PEM-encoded private key whose matching public key (JWK) is registered on the Okta service app.
|
||||
okta_scopes:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: OAuth scopes to request. Optional; defaults to the minimum set required to run the currently enabled Okta checks.
|
||||
required:
|
||||
- okta_client_id
|
||||
- okta_private_key
|
||||
- type: object
|
||||
title: Vercel API Token
|
||||
properties:
|
||||
@@ -21127,6 +21295,7 @@ components:
|
||||
- image
|
||||
- googleworkspace
|
||||
- vercel
|
||||
- okta
|
||||
type: string
|
||||
description: |-
|
||||
* `aws` - AWS
|
||||
@@ -21144,6 +21313,7 @@ components:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
x-spec-enum-id: 91f917e0c3ab97e8
|
||||
uid:
|
||||
type: string
|
||||
@@ -21265,6 +21435,7 @@ components:
|
||||
- image
|
||||
- googleworkspace
|
||||
- vercel
|
||||
- okta
|
||||
type: string
|
||||
x-spec-enum-id: 91f917e0c3ab97e8
|
||||
description: |-
|
||||
@@ -21285,6 +21456,7 @@ components:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
uid:
|
||||
type: string
|
||||
title: Unique identifier for the provider, set by the provider
|
||||
@@ -21337,6 +21509,7 @@ components:
|
||||
- image
|
||||
- googleworkspace
|
||||
- vercel
|
||||
- okta
|
||||
type: string
|
||||
x-spec-enum-id: 91f917e0c3ab97e8
|
||||
description: |-
|
||||
@@ -21357,6 +21530,7 @@ components:
|
||||
* `image` - Image
|
||||
* `googleworkspace` - Google Workspace
|
||||
* `vercel` - Vercel
|
||||
* `okta` - Okta
|
||||
uid:
|
||||
type: string
|
||||
minLength: 3
|
||||
@@ -22206,6 +22380,23 @@ components:
|
||||
required:
|
||||
- clouds_yaml_content
|
||||
- clouds_yaml_cloud
|
||||
- type: object
|
||||
title: Okta OAuth Credentials
|
||||
properties:
|
||||
okta_client_id:
|
||||
type: string
|
||||
description: Client ID of the Okta API Services app used for OAuth 2.0 private-key JWT authentication.
|
||||
okta_private_key:
|
||||
type: string
|
||||
description: PEM-encoded private key whose matching public key (JWK) is registered on the Okta service app.
|
||||
okta_scopes:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: OAuth scopes to request. Optional; defaults to the minimum set required to run the currently enabled Okta checks.
|
||||
required:
|
||||
- okta_client_id
|
||||
- okta_private_key
|
||||
- type: object
|
||||
title: Vercel API Token
|
||||
properties:
|
||||
@@ -22631,6 +22822,23 @@ components:
|
||||
required:
|
||||
- clouds_yaml_content
|
||||
- clouds_yaml_cloud
|
||||
- type: object
|
||||
title: Okta OAuth Credentials
|
||||
properties:
|
||||
okta_client_id:
|
||||
type: string
|
||||
description: Client ID of the Okta API Services app used for OAuth 2.0 private-key JWT authentication.
|
||||
okta_private_key:
|
||||
type: string
|
||||
description: PEM-encoded private key whose matching public key (JWK) is registered on the Okta service app.
|
||||
okta_scopes:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: OAuth scopes to request. Optional; defaults to the minimum set required to run the currently enabled Okta checks.
|
||||
required:
|
||||
- okta_client_id
|
||||
- okta_private_key
|
||||
- type: object
|
||||
title: Vercel API Token
|
||||
properties:
|
||||
@@ -23066,6 +23274,23 @@ components:
|
||||
required:
|
||||
- clouds_yaml_content
|
||||
- clouds_yaml_cloud
|
||||
- type: object
|
||||
title: Okta OAuth Credentials
|
||||
properties:
|
||||
okta_client_id:
|
||||
type: string
|
||||
description: Client ID of the Okta API Services app used for OAuth 2.0 private-key JWT authentication.
|
||||
okta_private_key:
|
||||
type: string
|
||||
description: PEM-encoded private key whose matching public key (JWK) is registered on the Okta service app.
|
||||
okta_scopes:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: OAuth scopes to request. Optional; defaults to the minimum set required to run the currently enabled Okta checks.
|
||||
required:
|
||||
- okta_client_id
|
||||
- okta_private_key
|
||||
- type: object
|
||||
title: Vercel API Token
|
||||
properties:
|
||||
|
||||
@@ -182,23 +182,19 @@ def _make_app():
|
||||
return ApiConfig("api", api)
|
||||
|
||||
|
||||
def test_ready_initializes_driver_for_api_process(monkeypatch):
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[
|
||||
["gunicorn"],
|
||||
["celery", "-A", "api"],
|
||||
["manage.py", "migrate"],
|
||||
],
|
||||
ids=["api", "celery", "manage_py"],
|
||||
)
|
||||
def test_ready_never_eagerly_initializes_neo4j_driver(monkeypatch, argv):
|
||||
"""ready() must never contact Neo4j; the driver is created lazily on first use."""
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with (
|
||||
patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None),
|
||||
patch("api.attack_paths.database.init_driver") as init_driver,
|
||||
):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_called_once()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_celery(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["celery", "-A", "api"])
|
||||
_set_argv(monkeypatch, argv)
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with (
|
||||
@@ -208,31 +204,3 @@ def test_ready_skips_driver_for_celery(monkeypatch):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_for_manage_py_skip_command(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["manage.py", "migrate"])
|
||||
_set_testing(monkeypatch, False)
|
||||
|
||||
with (
|
||||
patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None),
|
||||
patch("api.attack_paths.database.init_driver") as init_driver,
|
||||
):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
|
||||
def test_ready_skips_driver_when_testing(monkeypatch):
|
||||
config = _make_app()
|
||||
_set_argv(monkeypatch, ["gunicorn"])
|
||||
_set_testing(monkeypatch, True)
|
||||
|
||||
with (
|
||||
patch.object(ApiConfig, "_ensure_crypto_keys", return_value=None),
|
||||
patch("api.attack_paths.database.init_driver") as init_driver,
|
||||
):
|
||||
config.ready()
|
||||
|
||||
init_driver.assert_not_called()
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
"""
|
||||
Tests for Neo4j database lazy initialization.
|
||||
|
||||
The Neo4j driver connects on first use by default. API processes may
|
||||
eagerly initialize the driver during app startup, while Celery workers
|
||||
remain lazy. These tests validate the database module behavior itself.
|
||||
The Neo4j driver is created on first use for every process type; app startup
|
||||
never contacts Neo4j. These tests validate the database module behavior itself.
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
import pytest
|
||||
|
||||
import api.attack_paths.database as db_module
|
||||
@@ -59,6 +60,32 @@ class TestLazyInitialization:
|
||||
assert result is mock_driver
|
||||
assert db_module._driver is mock_driver
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_leaves_driver_none_when_verify_fails(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""A failed verify_connectivity() must not publish or leak the driver."""
|
||||
mock_driver = MagicMock()
|
||||
mock_driver.verify_connectivity.side_effect = (
|
||||
neo4j.exceptions.ServiceUnavailable("down")
|
||||
)
|
||||
mock_driver_factory.return_value = mock_driver
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
"HOST": "localhost",
|
||||
"PORT": 7687,
|
||||
"USER": "neo4j",
|
||||
"PASSWORD": "password",
|
||||
}
|
||||
}
|
||||
|
||||
with pytest.raises(neo4j.exceptions.ServiceUnavailable):
|
||||
db_module.init_driver()
|
||||
|
||||
assert db_module._driver is None
|
||||
mock_driver.close.assert_called_once()
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_init_driver_returns_cached_driver_on_subsequent_calls(
|
||||
@@ -116,21 +143,23 @@ class TestConnectionAcquisitionTimeout:
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_module_state(self):
|
||||
original_driver = db_module._driver
|
||||
original_timeout = db_module.CONN_ACQUISITION_TIMEOUT
|
||||
original_acq_timeout = db_module.CONN_ACQUISITION_TIMEOUT
|
||||
original_conn_timeout = db_module.CONNECTION_TIMEOUT
|
||||
|
||||
db_module._driver = None
|
||||
|
||||
yield
|
||||
|
||||
db_module._driver = original_driver
|
||||
db_module.CONN_ACQUISITION_TIMEOUT = original_timeout
|
||||
db_module.CONN_ACQUISITION_TIMEOUT = original_acq_timeout
|
||||
db_module.CONNECTION_TIMEOUT = original_conn_timeout
|
||||
|
||||
@patch("api.attack_paths.database.settings")
|
||||
@patch("api.attack_paths.database.neo4j.GraphDatabase.driver")
|
||||
def test_driver_receives_configured_timeout(
|
||||
self, mock_driver_factory, mock_settings
|
||||
):
|
||||
"""init_driver() should pass CONN_ACQUISITION_TIMEOUT to the neo4j driver."""
|
||||
"""init_driver() should pass the configured timeouts to the neo4j driver."""
|
||||
mock_driver_factory.return_value = MagicMock()
|
||||
mock_settings.DATABASES = {
|
||||
"neo4j": {
|
||||
@@ -141,11 +170,13 @@ class TestConnectionAcquisitionTimeout:
|
||||
}
|
||||
}
|
||||
db_module.CONN_ACQUISITION_TIMEOUT = 42
|
||||
db_module.CONNECTION_TIMEOUT = 7
|
||||
|
||||
db_module.init_driver()
|
||||
|
||||
_, kwargs = mock_driver_factory.call_args
|
||||
assert kwargs["connection_acquisition_timeout"] == 42
|
||||
assert kwargs["connection_timeout"] == 7
|
||||
|
||||
|
||||
class TestAtexitRegistration:
|
||||
|
||||
@@ -12,7 +12,9 @@ from api.compliance import (
|
||||
load_prowler_checks,
|
||||
)
|
||||
from api.models import Provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
|
||||
|
||||
class TestCompliance:
|
||||
@@ -28,16 +30,16 @@ class TestCompliance:
|
||||
assert set(checks) == {"check1", "check2", "check3"}
|
||||
mock_check_metadata.get_bulk.assert_called_once_with(provider_type)
|
||||
|
||||
@patch("api.compliance.Compliance")
|
||||
def test_get_prowler_provider_compliance(self, mock_compliance):
|
||||
@patch("api.compliance.get_bulk_compliance_frameworks_universal")
|
||||
def test_get_prowler_provider_compliance(self, mock_get_bulk):
|
||||
provider_type = Provider.ProviderChoices.AWS
|
||||
mock_compliance.get_bulk.return_value = {
|
||||
mock_get_bulk.return_value = {
|
||||
"compliance1": MagicMock(),
|
||||
"compliance2": MagicMock(),
|
||||
}
|
||||
compliance_data = get_prowler_provider_compliance(provider_type)
|
||||
assert compliance_data == mock_compliance.get_bulk.return_value
|
||||
mock_compliance.get_bulk.assert_called_once_with(provider_type)
|
||||
assert compliance_data == mock_get_bulk.return_value
|
||||
mock_get_bulk.assert_called_once_with(provider_type)
|
||||
|
||||
@patch("api.compliance.get_prowler_provider_checks")
|
||||
@patch("api.models.Provider.ProviderChoices")
|
||||
@@ -51,9 +53,9 @@ class TestCompliance:
|
||||
prowler_compliance = {
|
||||
"aws": {
|
||||
"compliance1": MagicMock(
|
||||
Requirements=[
|
||||
requirements=[
|
||||
MagicMock(
|
||||
Checks=["check1", "check2"],
|
||||
checks={"aws": ["check1", "check2"]},
|
||||
),
|
||||
],
|
||||
),
|
||||
@@ -167,35 +169,38 @@ class TestCompliance:
|
||||
def test_generate_compliance_overview_template(self, mock_provider_choices):
|
||||
mock_provider_choices.values = ["aws"]
|
||||
|
||||
# ``name`` is a reserved MagicMock kwarg (it labels the mock for repr,
|
||||
# it does NOT set a ``.name`` attribute), so it must be assigned
|
||||
# explicitly after construction.
|
||||
requirement1 = MagicMock(
|
||||
Id="requirement1",
|
||||
Name="Requirement 1",
|
||||
Description="Description of requirement 1",
|
||||
Attributes=[],
|
||||
Checks=["check1", "check2"],
|
||||
Tactics=["tactic1"],
|
||||
SubTechniques=["subtechnique1"],
|
||||
Platforms=["platform1"],
|
||||
TechniqueURL="https://example.com",
|
||||
id="requirement1",
|
||||
description="Description of requirement 1",
|
||||
attributes=[],
|
||||
checks={"aws": ["check1", "check2"]},
|
||||
tactics=["tactic1"],
|
||||
sub_techniques=["subtechnique1"],
|
||||
platforms=["platform1"],
|
||||
technique_url="https://example.com",
|
||||
)
|
||||
requirement1.name = "Requirement 1"
|
||||
requirement2 = MagicMock(
|
||||
Id="requirement2",
|
||||
Name="Requirement 2",
|
||||
Description="Description of requirement 2",
|
||||
Attributes=[],
|
||||
Checks=[],
|
||||
Tactics=[],
|
||||
SubTechniques=[],
|
||||
Platforms=[],
|
||||
TechniqueURL="",
|
||||
id="requirement2",
|
||||
description="Description of requirement 2",
|
||||
attributes=[],
|
||||
checks={"aws": []},
|
||||
tactics=[],
|
||||
sub_techniques=[],
|
||||
platforms=[],
|
||||
technique_url="",
|
||||
)
|
||||
requirement2.name = "Requirement 2"
|
||||
compliance1 = MagicMock(
|
||||
Requirements=[requirement1, requirement2],
|
||||
Framework="Framework 1",
|
||||
Version="1.0",
|
||||
Description="Description of compliance1",
|
||||
Name="Compliance 1",
|
||||
requirements=[requirement1, requirement2],
|
||||
framework="Framework 1",
|
||||
version="1.0",
|
||||
description="Description of compliance1",
|
||||
)
|
||||
compliance1.name = "Compliance 1"
|
||||
prowler_compliance = {"aws": {"compliance1": compliance1}}
|
||||
|
||||
template = generate_compliance_overview_template(prowler_compliance)
|
||||
@@ -271,24 +276,28 @@ def reset_compliance_cache():
|
||||
|
||||
class TestGetComplianceFrameworks:
|
||||
def test_returns_keys_from_compliance_get_bulk(self, reset_compliance_cache):
|
||||
with patch("api.compliance.Compliance") as mock_compliance:
|
||||
mock_compliance.get_bulk.return_value = {
|
||||
with patch(
|
||||
"api.compliance.get_bulk_compliance_frameworks_universal"
|
||||
) as mock_get_bulk:
|
||||
mock_get_bulk.return_value = {
|
||||
"cis_1.4_aws": MagicMock(),
|
||||
"mitre_attack_aws": MagicMock(),
|
||||
}
|
||||
result = get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
|
||||
assert sorted(result) == ["cis_1.4_aws", "mitre_attack_aws"]
|
||||
mock_compliance.get_bulk.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
mock_get_bulk.assert_called_once_with(Provider.ProviderChoices.AWS)
|
||||
|
||||
def test_caches_result_per_provider(self, reset_compliance_cache):
|
||||
with patch("api.compliance.Compliance") as mock_compliance:
|
||||
mock_compliance.get_bulk.return_value = {"cis_1.4_aws": MagicMock()}
|
||||
with patch(
|
||||
"api.compliance.get_bulk_compliance_frameworks_universal"
|
||||
) as mock_get_bulk:
|
||||
mock_get_bulk.return_value = {"cis_1.4_aws": MagicMock()}
|
||||
get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
get_compliance_frameworks(Provider.ProviderChoices.AWS)
|
||||
|
||||
# Cached after first call.
|
||||
assert mock_compliance.get_bulk.call_count == 1
|
||||
assert mock_get_bulk.call_count == 1
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_type",
|
||||
@@ -296,17 +305,19 @@ class TestGetComplianceFrameworks:
|
||||
)
|
||||
def test_listing_is_subset_of_bulk(self, reset_compliance_cache, provider_type):
|
||||
"""Regression for CLOUD-API-40S: every name returned by
|
||||
``get_compliance_frameworks`` must be loadable via ``Compliance.get_bulk``.
|
||||
``get_compliance_frameworks`` must be loadable via
|
||||
``get_bulk_compliance_frameworks_universal``.
|
||||
|
||||
A divergence here is what produced ``KeyError: 'csa_ccm_4.0'`` in
|
||||
``generate_outputs_task`` after universal/multi-provider compliance
|
||||
JSONs were introduced at the top-level ``prowler/compliance/`` path.
|
||||
"""
|
||||
bulk_keys = set(Compliance.get_bulk(provider_type).keys())
|
||||
bulk_keys = set(get_bulk_compliance_frameworks_universal(provider_type).keys())
|
||||
listed = set(get_compliance_frameworks(provider_type))
|
||||
|
||||
missing = listed - bulk_keys
|
||||
assert not missing, (
|
||||
f"get_compliance_frameworks({provider_type!r}) returned names not "
|
||||
f"loadable by Compliance.get_bulk: {sorted(missing)}"
|
||||
f"loadable by get_bulk_compliance_frameworks_universal: "
|
||||
f"{sorted(missing)}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
from config.django.base import label_postgres_connections
|
||||
|
||||
|
||||
class TestLabelPostgresConnections:
|
||||
def test_labels_postgres_and_skips_neo4j(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "scan")
|
||||
databases = {
|
||||
"default": {"ENGINE": "psqlextra.backend"},
|
||||
"neo4j": {"HOST": "neo4j", "PORT": "7687"},
|
||||
}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["default"]["OPTIONS"]["application_name"] == "scan:default"
|
||||
assert "OPTIONS" not in databases["neo4j"]
|
||||
|
||||
def test_labels_plain_postgresql_backend(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "api")
|
||||
databases = {"saas": {"ENGINE": "django.db.backends.postgresql"}}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["saas"]["OPTIONS"]["application_name"] == "api:saas"
|
||||
|
||||
def test_defaults_component_to_api_when_unset(self, monkeypatch):
|
||||
monkeypatch.delenv("DJANGO_APP_COMPONENT", raising=False)
|
||||
databases = {"default": {"ENGINE": "psqlextra.backend"}}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["default"]["OPTIONS"]["application_name"] == "api:default"
|
||||
|
||||
def test_preserves_existing_options(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "worker")
|
||||
databases = {
|
||||
"replica": {
|
||||
"ENGINE": "psqlextra.backend",
|
||||
"OPTIONS": {"sslmode": "require"},
|
||||
}
|
||||
}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert databases["replica"]["OPTIONS"] == {
|
||||
"sslmode": "require",
|
||||
"application_name": "worker:replica",
|
||||
}
|
||||
|
||||
def test_truncates_application_name_to_63_bytes(self, monkeypatch):
|
||||
monkeypatch.setenv("DJANGO_APP_COMPONENT", "c" * 80)
|
||||
databases = {"default": {"ENGINE": "psqlextra.backend"}}
|
||||
|
||||
label_postgres_connections(databases)
|
||||
|
||||
assert len(databases["default"]["OPTIONS"]["application_name"]) == 63
|
||||
@@ -31,6 +31,7 @@ from prowler.providers.image.image_provider import ImageProvider
|
||||
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
|
||||
from prowler.providers.m365.m365_provider import M365Provider
|
||||
from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider
|
||||
from prowler.providers.okta.okta_provider import OktaProvider
|
||||
from prowler.providers.openstack.openstack_provider import OpenstackProvider
|
||||
from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
|
||||
from prowler.providers.vercel.vercel_provider import VercelProvider
|
||||
@@ -130,6 +131,7 @@ class TestReturnProwlerProvider:
|
||||
(Provider.ProviderChoices.OPENSTACK.value, OpenstackProvider),
|
||||
(Provider.ProviderChoices.IMAGE.value, ImageProvider),
|
||||
(Provider.ProviderChoices.VERCEL.value, VercelProvider),
|
||||
(Provider.ProviderChoices.OKTA.value, OktaProvider),
|
||||
],
|
||||
)
|
||||
def test_return_prowler_provider(self, provider_type, expected_provider):
|
||||
@@ -238,6 +240,31 @@ class TestProwlerProviderConnectionTest:
|
||||
raise_on_exception=False,
|
||||
)
|
||||
|
||||
@patch("api.utils.return_prowler_provider")
|
||||
def test_prowler_provider_connection_test_okta_provider(
|
||||
self, mock_return_prowler_provider
|
||||
):
|
||||
"""Test connection test for Okta provider passes org domain and provider_id."""
|
||||
provider = MagicMock()
|
||||
provider.uid = "acme.okta.com"
|
||||
provider.provider = Provider.ProviderChoices.OKTA.value
|
||||
provider.secret.secret = {
|
||||
"okta_client_id": "0oa123456789abcdef",
|
||||
"okta_private_key": "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----",
|
||||
"okta_scopes": ["okta.policies.read"],
|
||||
}
|
||||
mock_return_prowler_provider.return_value = MagicMock()
|
||||
|
||||
prowler_provider_connection_test(provider)
|
||||
mock_return_prowler_provider.return_value.test_connection.assert_called_once_with(
|
||||
okta_client_id="0oa123456789abcdef",
|
||||
okta_private_key="-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----",
|
||||
okta_scopes=["okta.policies.read"],
|
||||
okta_org_domain="acme.okta.com",
|
||||
provider_id="acme.okta.com",
|
||||
raise_on_exception=False,
|
||||
)
|
||||
|
||||
@patch("api.utils.return_prowler_provider")
|
||||
def test_prowler_provider_connection_test_image_provider_no_creds(
|
||||
self, mock_return_prowler_provider
|
||||
@@ -308,6 +335,10 @@ class TestGetProwlerProviderKwargs:
|
||||
Provider.ProviderChoices.VERCEL.value,
|
||||
{"team_id": "provider_uid"},
|
||||
),
|
||||
(
|
||||
Provider.ProviderChoices.OKTA.value,
|
||||
{"okta_org_domain": "provider_uid"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_prowler_provider_kwargs(self, provider_type, expected_extra_kwargs):
|
||||
|
||||
@@ -24,9 +24,11 @@ from conftest import (
|
||||
today_after_n_days,
|
||||
)
|
||||
from django.conf import settings
|
||||
from django.db import connection
|
||||
from django.db.models import Count
|
||||
from django.http import JsonResponse
|
||||
from django.test import RequestFactory
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from django.urls import reverse
|
||||
from django_celery_results.models import TaskResult
|
||||
from rest_framework import status
|
||||
@@ -64,6 +66,7 @@ from api.models import (
|
||||
ProviderSecret,
|
||||
Resource,
|
||||
ResourceFindingMapping,
|
||||
ResourceTag,
|
||||
Role,
|
||||
RoleProviderGroupRelationship,
|
||||
SAMLConfiguration,
|
||||
@@ -1625,6 +1628,21 @@ class TestProviderViewSet:
|
||||
"uid": "C12",
|
||||
"alias": "Google Workspace Minimum Length",
|
||||
},
|
||||
{
|
||||
"provider": "okta",
|
||||
"uid": "acme.okta.com",
|
||||
"alias": "Okta Org",
|
||||
},
|
||||
{
|
||||
"provider": "okta",
|
||||
"uid": "agency.okta-gov.com",
|
||||
"alias": "Okta Gov Org",
|
||||
},
|
||||
{
|
||||
"provider": "okta",
|
||||
"uid": "agency.okta.mil",
|
||||
"alias": "Okta Mil Org",
|
||||
},
|
||||
]
|
||||
),
|
||||
)
|
||||
@@ -2143,6 +2161,24 @@ class TestProviderViewSet:
|
||||
"googleworkspace-uid",
|
||||
"uid",
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "okta",
|
||||
"uid": "https://acme.okta.com",
|
||||
"alias": "test",
|
||||
},
|
||||
"okta-uid",
|
||||
"uid",
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "okta",
|
||||
"uid": "acme.example.com",
|
||||
"alias": "test",
|
||||
},
|
||||
"okta-uid",
|
||||
"uid",
|
||||
),
|
||||
]
|
||||
),
|
||||
)
|
||||
@@ -2163,6 +2199,25 @@ class TestProviderViewSet:
|
||||
== f"/data/attributes/{error_pointer}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_uid,stored_uid",
|
||||
[
|
||||
("Acme.okta.com", "acme.okta.com"),
|
||||
(" ACME.OKTA.COM ", "acme.okta.com"),
|
||||
("Agency.Okta-Gov.com", "agency.okta-gov.com"),
|
||||
],
|
||||
)
|
||||
def test_providers_create_okta_uid_normalized(
|
||||
self, authenticated_client, input_uid, stored_uid
|
||||
):
|
||||
response = authenticated_client.post(
|
||||
reverse("provider-list"),
|
||||
data={"provider": "okta", "uid": input_uid, "alias": "Okta"},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
assert Provider.objects.get().uid == stored_uid
|
||||
|
||||
def test_providers_partial_update(self, authenticated_client, providers_fixture):
|
||||
provider1, *_ = providers_fixture
|
||||
new_alias = "This is the new name"
|
||||
@@ -2320,17 +2375,17 @@ class TestProviderViewSet:
|
||||
),
|
||||
("alias", "aws_testing_1", 1),
|
||||
("alias.icontains", "aws", 2),
|
||||
("inserted_at", TODAY, 13),
|
||||
("inserted_at", TODAY, 14),
|
||||
(
|
||||
"inserted_at.gte",
|
||||
"2024-01-01",
|
||||
13,
|
||||
14,
|
||||
),
|
||||
("inserted_at.lte", "2024-01-01", 0),
|
||||
(
|
||||
"updated_at.gte",
|
||||
"2024-01-01",
|
||||
13,
|
||||
14,
|
||||
),
|
||||
("updated_at.lte", "2024-01-01", 0),
|
||||
]
|
||||
@@ -2963,6 +3018,19 @@ class TestProviderSecretViewSet:
|
||||
"api_token": "fake-vercel-api-token-for-testing",
|
||||
},
|
||||
),
|
||||
# Okta with inline private key credentials
|
||||
(
|
||||
Provider.ProviderChoices.OKTA.value,
|
||||
ProviderSecret.TypeChoices.STATIC,
|
||||
{
|
||||
"okta_client_id": "0oa123456789abcdef",
|
||||
"okta_private_key": "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----",
|
||||
"okta_scopes": [
|
||||
"okta.policies.read",
|
||||
"okta.groups.read",
|
||||
],
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_provider_secrets_create_valid(
|
||||
@@ -3075,6 +3143,46 @@ class TestProviderSecretViewSet:
|
||||
== f"/data/attributes/{error_pointer}"
|
||||
)
|
||||
|
||||
def test_provider_secrets_invalid_create_okta_missing_private_key(
|
||||
self,
|
||||
providers_fixture,
|
||||
authenticated_client,
|
||||
):
|
||||
okta_provider = next(
|
||||
provider
|
||||
for provider in providers_fixture
|
||||
if provider.provider == Provider.ProviderChoices.OKTA.value
|
||||
)
|
||||
data = {
|
||||
"data": {
|
||||
"type": "provider-secrets",
|
||||
"attributes": {
|
||||
"name": "Okta Secret",
|
||||
"secret_type": ProviderSecret.TypeChoices.STATIC,
|
||||
"secret": {
|
||||
"okta_client_id": "0oa123456789abcdef",
|
||||
},
|
||||
},
|
||||
"relationships": {
|
||||
"provider": {
|
||||
"data": {"type": "providers", "id": str(okta_provider.id)}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("providersecret-list"),
|
||||
data=json.dumps(data),
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert response.json()["errors"][0]["code"] == "required"
|
||||
assert response.json()["errors"][0]["source"]["pointer"] == (
|
||||
"/data/attributes/secret/okta_private_key"
|
||||
)
|
||||
|
||||
def test_provider_secrets_partial_update(
|
||||
self, authenticated_client, provider_secret_fixture
|
||||
):
|
||||
@@ -3751,16 +3859,20 @@ class TestScanViewSet:
|
||||
scan.output_location = "dummy"
|
||||
scan.save()
|
||||
|
||||
dummy_task = Task.objects.create(tenant_id=scan.tenant_id)
|
||||
dummy_task.id = "dummy-task-id"
|
||||
dummy_task_data = {"id": dummy_task.id, "state": StateChoices.EXECUTING}
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
)
|
||||
task = Task.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
task_runner_task=task_result,
|
||||
)
|
||||
dummy_task_data = {"id": str(task.id), "state": StateChoices.EXECUTING}
|
||||
|
||||
with (
|
||||
patch("api.v1.views.Task.objects.get", return_value=dummy_task),
|
||||
patch(
|
||||
"api.v1.views.TaskSerializer",
|
||||
return_value=type("DummySerializer", (), {"data": dummy_task_data}),
|
||||
),
|
||||
with patch(
|
||||
"api.v1.views.TaskSerializer",
|
||||
return_value=type("DummySerializer", (), {"data": dummy_task_data}),
|
||||
):
|
||||
url = reverse("scan-report", kwargs={"pk": scan.id})
|
||||
response = authenticated_client.get(url)
|
||||
@@ -4081,6 +4193,88 @@ class TestScanViewSet:
|
||||
assert resp.status_code == status.HTTP_302_FOUND
|
||||
assert resp["Location"] == presigned_url
|
||||
|
||||
def test_compliance_s3_returns_latest_match(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
):
|
||||
"""When several files match, the most recently modified one is served."""
|
||||
scan = scans_fixture[0]
|
||||
bucket = "bucket"
|
||||
scan.output_location = f"s3://{bucket}/path/scan.zip"
|
||||
scan.state = StateChoices.COMPLETED
|
||||
scan.save()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
old_key = "path/compliance/prowler-output-aws-20240101000000_cis_1.4_aws.csv"
|
||||
latest_key = "path/compliance/prowler-output-aws-20240202000000_cis_1.4_aws.csv"
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
return {
|
||||
"Contents": [
|
||||
{
|
||||
"Key": old_key,
|
||||
"LastModified": datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
},
|
||||
{
|
||||
"Key": latest_key,
|
||||
"LastModified": datetime(2024, 2, 2, tzinfo=timezone.utc),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
def generate_presigned_url(self, ClientMethod, Params, ExpiresIn):
|
||||
assert Params["Key"] == latest_key
|
||||
return "https://test-bucket.s3.amazonaws.com/latest"
|
||||
|
||||
monkeypatch.setattr("api.v1.views.get_s3_client", lambda: FakeS3Client())
|
||||
|
||||
url = reverse("scan-compliance", kwargs={"pk": scan.id, "name": "cis_1.4_aws"})
|
||||
resp = authenticated_client.get(url)
|
||||
assert resp.status_code == status.HTTP_302_FOUND
|
||||
assert resp["Location"].endswith("/latest")
|
||||
|
||||
def test_compliance_local_returns_latest_match(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
):
|
||||
"""The local branch serves the most recently modified matching file."""
|
||||
scan = scans_fixture[0]
|
||||
scan.state = StateChoices.COMPLETED
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
comp_dir = Path(tmp) / "reports" / "compliance"
|
||||
comp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
old_file = comp_dir / "prowler-output-aws-20240101000000_cis_1.4_aws.csv"
|
||||
old_file.write_bytes(b"old")
|
||||
latest_file = comp_dir / "prowler-output-aws-20240202000000_cis_1.4_aws.csv"
|
||||
latest_file.write_bytes(b"latest")
|
||||
# Make `latest_file` newer regardless of creation order.
|
||||
os.utime(old_file, (1_700_000_000, 1_700_000_000))
|
||||
os.utime(latest_file, (1_700_000_100, 1_700_000_100))
|
||||
|
||||
scan.output_location = str(Path(tmp) / "reports" / "scan.zip")
|
||||
scan.save()
|
||||
|
||||
monkeypatch.setattr(
|
||||
glob,
|
||||
"glob",
|
||||
lambda p: [str(old_file), str(latest_file)],
|
||||
)
|
||||
|
||||
url = reverse(
|
||||
"scan-compliance", kwargs={"pk": scan.id, "name": "cis_1.4_aws"}
|
||||
)
|
||||
resp = authenticated_client.get(url)
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
assert resp.content == b"latest"
|
||||
assert resp["Content-Disposition"].endswith(
|
||||
f'filename="{latest_file.name}"'
|
||||
)
|
||||
|
||||
def test_compliance_s3_not_found(
|
||||
self, authenticated_client, scans_fixture, monkeypatch
|
||||
):
|
||||
@@ -4189,18 +4383,24 @@ class TestScanViewSet:
|
||||
assert cd.startswith('attachment; filename="')
|
||||
assert cd.endswith(f'filename="{fname.name}"')
|
||||
|
||||
@patch("api.v1.views.Task.objects.get")
|
||||
@patch("api.v1.views.TaskSerializer")
|
||||
def test__get_task_status_returns_none_if_task_not_executing(
|
||||
self, mock_task_serializer, mock_task_get, authenticated_client, scans_fixture
|
||||
self, mock_task_serializer, authenticated_client, scans_fixture
|
||||
):
|
||||
scan = scans_fixture[0]
|
||||
scan.state = StateChoices.COMPLETED
|
||||
scan.output_location = "dummy"
|
||||
scan.save()
|
||||
|
||||
task = Task.objects.create(tenant_id=scan.tenant_id)
|
||||
mock_task_get.return_value = task
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
)
|
||||
task = Task.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
task_runner_task=task_result,
|
||||
)
|
||||
mock_task_serializer.return_value.data = {
|
||||
"id": str(task.id),
|
||||
"state": StateChoices.COMPLETED,
|
||||
@@ -4221,6 +4421,7 @@ class TestScanViewSet:
|
||||
scan.save()
|
||||
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
)
|
||||
@@ -4241,6 +4442,51 @@ class TestScanViewSet:
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
assert response.data["id"] == str(task.id)
|
||||
|
||||
@patch("api.v1.views.TaskSerializer")
|
||||
def test__get_task_status_returns_latest_task(
|
||||
self, mock_task_serializer, authenticated_client, scans_fixture
|
||||
):
|
||||
"""With several scan-report tasks for the scan, the most recent is used."""
|
||||
scan = scans_fixture[0]
|
||||
scan.state = StateChoices.COMPLETED
|
||||
scan.output_location = "dummy"
|
||||
scan.save()
|
||||
|
||||
old_task = Task.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
task_runner_task=TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
),
|
||||
)
|
||||
new_task = Task.objects.create(
|
||||
tenant_id=scan.tenant_id,
|
||||
task_runner_task=TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-report",
|
||||
task_kwargs={"scan_id": str(scan.id)},
|
||||
),
|
||||
)
|
||||
# `inserted_at` is `auto_now_add`, and within the test transaction the DB
|
||||
# `now()` is constant, so force distinct timestamps to make order_by stable.
|
||||
base = datetime(2024, 1, 1, tzinfo=timezone.utc)
|
||||
Task.objects.filter(pk=old_task.pk).update(inserted_at=base)
|
||||
Task.objects.filter(pk=new_task.pk).update(
|
||||
inserted_at=base + timedelta(hours=1)
|
||||
)
|
||||
|
||||
mock_task_serializer.side_effect = lambda instance, *a, **k: SimpleNamespace(
|
||||
data={"id": str(instance.id), "state": StateChoices.EXECUTING}
|
||||
)
|
||||
|
||||
url = reverse("scan-report", kwargs={"pk": scan.id})
|
||||
response = authenticated_client.get(url)
|
||||
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
assert str(new_task.id) in response["Content-Location"]
|
||||
assert str(old_task.id) not in response["Content-Location"]
|
||||
|
||||
@patch("api.v1.views.get_s3_client")
|
||||
@patch("api.v1.views.sentry_sdk.capture_exception")
|
||||
def test_compliance_list_objects_client_error(
|
||||
@@ -6811,6 +7057,80 @@ class TestFindingViewSet:
|
||||
== findings_fixture[0].status
|
||||
)
|
||||
|
||||
def test_findings_list_resource_tags_no_n_plus_one(
|
||||
self, authenticated_client, findings_fixture
|
||||
):
|
||||
"""Listing findings must load every resource's tags in a constant
|
||||
number of queries, no matter how many findings/resources are returned.
|
||||
|
||||
This guards ``FindingViewSet._optimize_tags_loading`` against
|
||||
regressions that would reintroduce one extra query per resource (the
|
||||
N+1 the prefetch was added to remove).
|
||||
"""
|
||||
scan = findings_fixture[0].scan
|
||||
tenant_id = findings_fixture[0].tenant_id
|
||||
provider = scan.provider
|
||||
|
||||
def _create_finding_with_tagged_resource(index):
|
||||
resource = Resource.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
provider=provider,
|
||||
uid=f"arn:aws:ec2:us-east-1:123456789012:instance/n-plus-one-{index}",
|
||||
name=f"N+1 Instance {index}",
|
||||
region="us-east-1",
|
||||
service="ec2",
|
||||
type="prowler-test",
|
||||
)
|
||||
resource.upsert_or_delete_tags(
|
||||
[
|
||||
ResourceTag.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
key=f"key-{index}",
|
||||
value=f"value-{index}",
|
||||
)
|
||||
]
|
||||
)
|
||||
finding = Finding.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
uid=f"n_plus_one_finding_{index}",
|
||||
scan=scan,
|
||||
status=Status.FAIL,
|
||||
status_extended="n+1 status",
|
||||
impact=Severity.medium,
|
||||
severity=Severity.medium,
|
||||
check_id="test_check_id",
|
||||
check_metadata={"CheckId": "test_check_id", "servicename": "ec2"},
|
||||
first_seen_at="2024-01-02T00:00:00Z",
|
||||
)
|
||||
finding.add_resources([resource])
|
||||
return finding
|
||||
|
||||
params = {"filter[inserted_at]": TODAY, "include": "resources"}
|
||||
|
||||
# Baseline: the two findings provided by the fixture.
|
||||
with CaptureQueriesContext(connection) as baseline:
|
||||
response = authenticated_client.get(reverse("finding-list"), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
|
||||
# Add more findings, each with its own resource carrying tags.
|
||||
extra_findings = 5
|
||||
for index in range(extra_findings):
|
||||
_create_finding_with_tagged_resource(index)
|
||||
|
||||
with CaptureQueriesContext(connection) as scaled:
|
||||
response = authenticated_client.get(reverse("finding-list"), params)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert len(response.json()["data"]) == len(findings_fixture) + extra_findings
|
||||
|
||||
# The query count must not grow with the number of findings/resources.
|
||||
assert len(scaled.captured_queries) == len(baseline.captured_queries), (
|
||||
"Resource tags are not being prefetched: "
|
||||
f"{len(baseline.captured_queries)} queries for {len(findings_fixture)} "
|
||||
f"findings vs {len(scaled.captured_queries)} for "
|
||||
f"{len(findings_fixture) + extra_findings}. Likely an N+1 regression "
|
||||
"in FindingViewSet._optimize_tags_loading."
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"include_values, expected_resources",
|
||||
[
|
||||
@@ -7053,6 +7373,32 @@ class TestFindingViewSet:
|
||||
"id"
|
||||
] == str(finding_1.resources.first().id)
|
||||
|
||||
def test_findings_retrieve_include_resource_metadata(
|
||||
self, authenticated_client, findings_fixture
|
||||
):
|
||||
finding_1, *_ = findings_fixture
|
||||
resource = finding_1.resources.first()
|
||||
resource.metadata = '{"VulnerabilityID": "CVE-2026-0001"}'
|
||||
resource.details = "Python 3.12 base image"
|
||||
resource.save()
|
||||
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-detail", kwargs={"pk": finding_1.id}),
|
||||
{"include": "resources"},
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
|
||||
included_resource = next(
|
||||
item
|
||||
for item in response.json()["included"]
|
||||
if item["type"] == "resources" and item["id"] == str(resource.id)
|
||||
)
|
||||
assert (
|
||||
included_resource["attributes"]["metadata"]
|
||||
== '{"VulnerabilityID": "CVE-2026-0001"}'
|
||||
)
|
||||
assert included_resource["attributes"]["details"] == "Python 3.12 base image"
|
||||
|
||||
def test_findings_invalid_retrieve(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-detail", kwargs={"pk": "random_id"}),
|
||||
@@ -9214,6 +9560,16 @@ class TestComplianceOverviewViewSet:
|
||||
assert "platforms" in attributes["attributes"]["technique_details"]
|
||||
assert "technique_url" in attributes["attributes"]["technique_details"]
|
||||
|
||||
# Guard against the `_raw_attributes` wrapper leaking through —
|
||||
# the UI reads metadata[i].Category / .AWSService directly.
|
||||
metadata = attributes["attributes"]["metadata"]
|
||||
assert isinstance(metadata, list) and len(metadata) > 0
|
||||
first_attr = metadata[0]
|
||||
assert isinstance(first_attr, dict)
|
||||
assert "_raw_attributes" not in first_attr
|
||||
assert "Category" in first_attr
|
||||
assert "AWSService" in first_attr
|
||||
|
||||
def test_compliance_overview_attributes_missing_compliance_id(
|
||||
self, authenticated_client
|
||||
):
|
||||
@@ -15790,6 +16146,12 @@ class TestFindingGroupViewSet:
|
||||
assert attrs["fail_count"] == 0
|
||||
assert attrs["resources_total"] == 1
|
||||
assert attrs["resources_fail"] == 0
|
||||
# check_title / check_description are resolved post-pagination from the
|
||||
# summary table, not from the finding's check_metadata.
|
||||
assert attrs["check_title"] == "Ensure EC2 instances do not have public IPs"
|
||||
assert (
|
||||
attrs["check_description"] == "EC2 instances should use private IPs only."
|
||||
)
|
||||
|
||||
def test_finding_groups_status_pass_when_no_fail(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
@@ -17031,6 +17393,12 @@ class TestFindingGroupViewSet:
|
||||
assert attrs["fail_count"] == 0
|
||||
assert attrs["resources_total"] == 1
|
||||
assert attrs["resources_fail"] == 0
|
||||
# check_title / check_description are resolved post-pagination from the
|
||||
# summary table, not from the finding's check_metadata.
|
||||
assert attrs["check_title"] == "Ensure EC2 instances do not have public IPs"
|
||||
assert (
|
||||
attrs["check_description"] == "EC2 instances should use private IPs only."
|
||||
)
|
||||
|
||||
def test_finding_groups_latest_status_in_filter(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
@@ -17288,18 +17656,20 @@ class TestFindingGroupViewSet:
|
||||
check_ids = [item["id"] for item in data]
|
||||
assert check_ids == sorted(check_ids)
|
||||
|
||||
def test_finding_groups_latest_sort_by_check_title(
|
||||
def test_finding_groups_latest_sort_by_check_title_not_supported(
|
||||
self, authenticated_client, finding_groups_fixture
|
||||
):
|
||||
"""Test /latest supports sorting by check_title."""
|
||||
"""check_title is not a sortable field for finding groups.
|
||||
|
||||
Titles live in the TOASTed check_metadata blob and are resolved after
|
||||
pagination from the summary table, so they cannot drive DB-level
|
||||
ordering. Requesting that sort is rejected.
|
||||
"""
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-group-latest"),
|
||||
{"sort": "check_title"},
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
check_titles = [item["attributes"]["check_title"] for item in data]
|
||||
assert check_titles == sorted(check_titles)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name", ["finding-group-list", "finding-group-latest"]
|
||||
|
||||
@@ -37,6 +37,7 @@ if TYPE_CHECKING:
|
||||
from prowler.providers.mongodbatlas.mongodbatlas_provider import (
|
||||
MongodbatlasProvider,
|
||||
)
|
||||
from prowler.providers.okta.okta_provider import OktaProvider
|
||||
from prowler.providers.openstack.openstack_provider import OpenstackProvider
|
||||
from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider
|
||||
from prowler.providers.vercel.vercel_provider import VercelProvider
|
||||
@@ -93,6 +94,7 @@ def return_prowler_provider(
|
||||
| KubernetesProvider
|
||||
| M365Provider
|
||||
| MongodbatlasProvider
|
||||
| OktaProvider
|
||||
| OpenstackProvider
|
||||
| OraclecloudProvider
|
||||
| VercelProvider
|
||||
@@ -181,6 +183,10 @@ def return_prowler_provider(
|
||||
from prowler.providers.vercel.vercel_provider import VercelProvider
|
||||
|
||||
prowler_provider = VercelProvider
|
||||
case Provider.ProviderChoices.OKTA.value:
|
||||
from prowler.providers.okta.okta_provider import OktaProvider
|
||||
|
||||
prowler_provider = OktaProvider
|
||||
case _:
|
||||
raise ValueError(f"Provider type {provider.provider} not supported")
|
||||
return prowler_provider
|
||||
@@ -246,6 +252,11 @@ def get_prowler_provider_kwargs(
|
||||
**prowler_provider_kwargs,
|
||||
"team_id": provider.uid,
|
||||
}
|
||||
elif provider.provider == Provider.ProviderChoices.OKTA.value:
|
||||
prowler_provider_kwargs = {
|
||||
**prowler_provider_kwargs,
|
||||
"okta_org_domain": provider.uid,
|
||||
}
|
||||
elif provider.provider == Provider.ProviderChoices.IMAGE.value:
|
||||
# Detect whether uid is a registry URL (e.g. "docker.io/andoniaf") or
|
||||
# a concrete image reference (e.g. "docker.io/andoniaf/myimage:latest").
|
||||
@@ -290,6 +301,7 @@ def initialize_prowler_provider(
|
||||
| KubernetesProvider
|
||||
| M365Provider
|
||||
| MongodbatlasProvider
|
||||
| OktaProvider
|
||||
| OpenstackProvider
|
||||
| OraclecloudProvider
|
||||
| VercelProvider
|
||||
@@ -351,6 +363,14 @@ def prowler_provider_connection_test(provider: Provider) -> Connection:
|
||||
"raise_on_exception": False,
|
||||
}
|
||||
return prowler_provider.test_connection(**vercel_kwargs)
|
||||
elif provider.provider == Provider.ProviderChoices.OKTA.value:
|
||||
okta_kwargs = {
|
||||
**prowler_provider_kwargs,
|
||||
"okta_org_domain": provider.uid,
|
||||
"provider_id": provider.uid,
|
||||
"raise_on_exception": False,
|
||||
}
|
||||
return prowler_provider.test_connection(**okta_kwargs)
|
||||
elif provider.provider == Provider.ProviderChoices.IMAGE.value:
|
||||
image_kwargs = {
|
||||
"image": provider.uid,
|
||||
|
||||
@@ -404,6 +404,26 @@ from rest_framework_json_api import serializers
|
||||
},
|
||||
"required": ["clouds_yaml_content", "clouds_yaml_cloud"],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"title": "Okta OAuth Credentials",
|
||||
"properties": {
|
||||
"okta_client_id": {
|
||||
"type": "string",
|
||||
"description": "Client ID of the Okta API Services app used for OAuth 2.0 private-key JWT authentication.",
|
||||
},
|
||||
"okta_private_key": {
|
||||
"type": "string",
|
||||
"description": "PEM-encoded private key whose matching public key (JWK) is registered on the Okta service app.",
|
||||
},
|
||||
"okta_scopes": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "OAuth scopes to request. Optional; defaults to the minimum set required to run the currently enabled Okta checks.",
|
||||
},
|
||||
},
|
||||
"required": ["okta_client_id", "okta_private_key"],
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"title": "Vercel API Token",
|
||||
|
||||
@@ -1397,6 +1397,7 @@ class ResourceIncludeSerializer(RLSSerializer):
|
||||
"service",
|
||||
"type_",
|
||||
"tags",
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
]
|
||||
@@ -1404,6 +1405,7 @@ class ResourceIncludeSerializer(RLSSerializer):
|
||||
"id": {"read_only": True},
|
||||
"inserted_at": {"read_only": True},
|
||||
"updated_at": {"read_only": True},
|
||||
"metadata": {"read_only": True},
|
||||
"details": {"read_only": True},
|
||||
"partition": {"read_only": True},
|
||||
}
|
||||
@@ -1543,6 +1545,8 @@ class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
|
||||
serializer = GCPProviderSecret(data=secret)
|
||||
elif provider_type == Provider.ProviderChoices.GOOGLEWORKSPACE.value:
|
||||
serializer = GoogleWorkspaceProviderSecret(data=secret)
|
||||
elif provider_type == Provider.ProviderChoices.OKTA.value:
|
||||
serializer = OktaProviderSecret(data=secret)
|
||||
elif provider_type == Provider.ProviderChoices.GITHUB.value:
|
||||
serializer = GithubProviderSecret(data=secret)
|
||||
elif provider_type == Provider.ProviderChoices.IAC.value:
|
||||
@@ -1688,6 +1692,15 @@ class GoogleWorkspaceProviderSecret(serializers.Serializer):
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
|
||||
class OktaProviderSecret(serializers.Serializer):
|
||||
okta_client_id = serializers.CharField()
|
||||
okta_private_key = serializers.CharField()
|
||||
okta_scopes = serializers.ListField(child=serializers.CharField(), required=False)
|
||||
|
||||
class Meta:
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
|
||||
class MongoDBAtlasProviderSecret(serializers.Serializer):
|
||||
atlas_public_key = serializers.CharField()
|
||||
atlas_private_key = serializers.CharField()
|
||||
|
||||
+182
-68
@@ -116,6 +116,7 @@ from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset
|
||||
from api.compliance import (
|
||||
PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE,
|
||||
get_compliance_frameworks,
|
||||
get_prowler_provider_compliance,
|
||||
)
|
||||
from api.constants import SEVERITY_ORDER
|
||||
from api.db_router import MainRouter
|
||||
@@ -1849,7 +1850,42 @@ class ProviderViewSet(DisablePaginationMixin, BaseRLSViewSet):
|
||||
200: OpenApiResponse(
|
||||
description="CSV file containing the compliance report"
|
||||
),
|
||||
404: OpenApiResponse(description="Compliance report not found"),
|
||||
202: OpenApiResponse(description="The task is in progress"),
|
||||
403: OpenApiResponse(description="There is a problem with credentials"),
|
||||
404: OpenApiResponse(
|
||||
description="Compliance report not found, or the scan has no reports yet"
|
||||
),
|
||||
},
|
||||
request=None,
|
||||
),
|
||||
compliance_ocsf=extend_schema(
|
||||
tags=["Scan"],
|
||||
summary="Retrieve compliance report as OCSF JSON",
|
||||
description=(
|
||||
"Download a specific compliance report as an OCSF JSON file. "
|
||||
"Only universal frameworks that declare an output configuration "
|
||||
"produce this artifact (currently 'dora' and 'csa_ccm_4.0'); any "
|
||||
"other framework returns 404."
|
||||
),
|
||||
parameters=[
|
||||
OpenApiParameter(
|
||||
name="name",
|
||||
type=str,
|
||||
location=OpenApiParameter.PATH,
|
||||
required=True,
|
||||
description="The compliance report name, like 'dora'",
|
||||
),
|
||||
],
|
||||
responses={
|
||||
200: OpenApiResponse(
|
||||
description="OCSF JSON file containing the compliance report"
|
||||
),
|
||||
202: OpenApiResponse(description="The task is in progress"),
|
||||
403: OpenApiResponse(description="There is a problem with credentials"),
|
||||
404: OpenApiResponse(
|
||||
description="Compliance report not found, the framework does "
|
||||
"not provide an OCSF export, or the scan has no reports yet"
|
||||
),
|
||||
},
|
||||
request=None,
|
||||
),
|
||||
@@ -1992,35 +2028,23 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
return queryset.select_related("provider", "task")
|
||||
|
||||
def get_serializer_class(self):
|
||||
if self.action == "create":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
return ScanCreateSerializer
|
||||
elif self.action == "partial_update":
|
||||
if self.action == "partial_update":
|
||||
return ScanUpdateSerializer
|
||||
elif self.action == "report":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
return ScanReportSerializer
|
||||
elif self.action == "compliance":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
return ScanComplianceReportSerializer
|
||||
elif self.action == "threatscore":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "ens":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "nis2":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "csa":
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
elif self.action == "cis":
|
||||
|
||||
action_defaults = {
|
||||
"create": ScanCreateSerializer,
|
||||
"report": ScanReportSerializer,
|
||||
"compliance": ScanComplianceReportSerializer,
|
||||
"compliance_ocsf": ScanComplianceReportSerializer,
|
||||
}
|
||||
response_only_actions = {"threatscore", "ens", "nis2", "csa", "cis"}
|
||||
|
||||
if self.action in action_defaults or self.action in response_only_actions:
|
||||
if hasattr(self, "response_serializer_class"):
|
||||
return self.response_serializer_class
|
||||
if self.action in action_defaults:
|
||||
return action_defaults[self.action]
|
||||
|
||||
return super().get_serializer_class()
|
||||
|
||||
def partial_update(self, request, *args, **kwargs):
|
||||
@@ -2059,12 +2083,17 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
if scan_instance.state == StateChoices.EXECUTING and scan_instance.task:
|
||||
task = scan_instance.task
|
||||
else:
|
||||
try:
|
||||
task = Task.objects.get(
|
||||
# A scan can have several `scan-report` tasks (e.g. re-runs); take the
|
||||
# most recent one. `.first()` also avoids `MultipleObjectsReturned`.
|
||||
task = (
|
||||
Task.objects.filter(
|
||||
task_runner_task__task_name="scan-report",
|
||||
task_runner_task__task_kwargs__contains=str(scan_instance.id),
|
||||
)
|
||||
except Task.DoesNotExist:
|
||||
.order_by("-inserted_at")
|
||||
.first()
|
||||
)
|
||||
if task is None:
|
||||
return None
|
||||
|
||||
self.response_serializer_class = TaskSerializer
|
||||
@@ -2139,27 +2168,32 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
status=status.HTTP_502_BAD_GATEWAY,
|
||||
)
|
||||
contents = resp.get("Contents", [])
|
||||
keys = []
|
||||
matches = []
|
||||
for obj in contents:
|
||||
key = obj["Key"]
|
||||
key_basename = os.path.basename(key)
|
||||
if any(ch in suffix for ch in ("*", "?", "[")):
|
||||
if fnmatch.fnmatch(key_basename, suffix):
|
||||
keys.append(key)
|
||||
matches.append(obj)
|
||||
elif key_basename == suffix:
|
||||
keys.append(key)
|
||||
matches.append(obj)
|
||||
elif key.endswith(suffix):
|
||||
# Backward compatibility if suffix already includes directories
|
||||
keys.append(key)
|
||||
if not keys:
|
||||
matches.append(obj)
|
||||
if not matches:
|
||||
return Response(
|
||||
{
|
||||
"detail": f"No compliance file found for name '{os.path.splitext(suffix)[0]}'."
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
# path_pattern here is prefix, but in compliance we build correct suffix check before
|
||||
key = keys[0]
|
||||
# Return the most recently modified match (latest report) when
|
||||
# several files share the prefix/suffix. `list_objects_v2` always
|
||||
# returns `LastModified`; the fallback keeps ordering deterministic
|
||||
# if it is ever absent.
|
||||
key = max(matches, key=lambda o: (o.get("LastModified", ""), o["Key"]))[
|
||||
"Key"
|
||||
]
|
||||
else:
|
||||
# path_pattern is exact key; HEAD before presigning to preserve the 404 contract.
|
||||
key = path_pattern
|
||||
@@ -2209,7 +2243,9 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
filepath = files[0]
|
||||
# Return the most recently modified match (latest report) when the
|
||||
# pattern resolves to several files.
|
||||
filepath = max(files, key=os.path.getmtime)
|
||||
with open(filepath, "rb") as f:
|
||||
content = f.read()
|
||||
filename = os.path.basename(filepath)
|
||||
@@ -2257,20 +2293,16 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
content, filename = loader
|
||||
return self._serve_file(content, filename, "application/x-zip-compressed")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_path="compliance/(?P<name>[^/]+)",
|
||||
url_name="compliance",
|
||||
)
|
||||
def compliance(self, request, pk=None, name=None):
|
||||
scan = self.get_object()
|
||||
if name not in get_compliance_frameworks(scan.provider.provider):
|
||||
return Response(
|
||||
{"detail": f"Compliance '{name}' not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
def _serve_compliance_artifact(self, scan, name, file_extension, content_type):
|
||||
"""Resolve and serve a per-framework compliance artifact from disk/S3.
|
||||
|
||||
Shared by the CSV and OCSF compliance download actions. Both are
|
||||
path-based (no query params) on purpose: ``get_object`` runs
|
||||
``filter_queryset``, which triggers JSON:API's
|
||||
``QueryParameterValidationFilter`` and 400s on any non-JSON:API
|
||||
query param, so a ``?format=`` / ``?type=`` selector is not viable
|
||||
here — the format is encoded in the route instead.
|
||||
"""
|
||||
running_resp = self._get_task_status(scan)
|
||||
if running_resp:
|
||||
return running_resp
|
||||
@@ -2287,25 +2319,66 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
bucket = env.str("DJANGO_OUTPUT_S3_AWS_OUTPUT_BUCKET", "")
|
||||
key_prefix = scan.output_location.removeprefix(f"s3://{bucket}/")
|
||||
prefix = os.path.join(
|
||||
os.path.dirname(key_prefix), "compliance", f"{name}.csv"
|
||||
os.path.dirname(key_prefix), "compliance", f"{name}.{file_extension}"
|
||||
)
|
||||
loader = self._load_file(
|
||||
prefix,
|
||||
s3=True,
|
||||
bucket=bucket,
|
||||
list_objects=True,
|
||||
content_type="text/csv",
|
||||
content_type=content_type,
|
||||
)
|
||||
else:
|
||||
base = os.path.dirname(scan.output_location)
|
||||
pattern = os.path.join(base, "compliance", f"*_{name}.csv")
|
||||
pattern = os.path.join(base, "compliance", f"*_{name}.{file_extension}")
|
||||
loader = self._load_file(pattern, s3=False)
|
||||
|
||||
if isinstance(loader, HttpResponseBase):
|
||||
return loader
|
||||
|
||||
content, filename = loader
|
||||
return self._serve_file(content, filename, "text/csv")
|
||||
return self._serve_file(content, filename, content_type)
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_path="compliance/(?P<name>[^/]+)",
|
||||
url_name="compliance",
|
||||
)
|
||||
def compliance(self, request, pk=None, name=None):
|
||||
scan = self.get_object()
|
||||
if name not in get_compliance_frameworks(scan.provider.provider):
|
||||
return Response(
|
||||
{"detail": f"Compliance '{name}' not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
return self._serve_compliance_artifact(scan, name, "csv", "text/csv")
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get"],
|
||||
url_path="compliance/(?P<name>[^/]+)/ocsf",
|
||||
url_name="compliance-ocsf",
|
||||
)
|
||||
def compliance_ocsf(self, request, pk=None, name=None):
|
||||
scan = self.get_object()
|
||||
if name not in get_compliance_frameworks(scan.provider.provider):
|
||||
return Response(
|
||||
{"detail": f"Compliance '{name}' not found."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
universal_bulk = get_prowler_provider_compliance(scan.provider.provider)
|
||||
framework_obj = universal_bulk.get(name)
|
||||
if not (framework_obj and getattr(framework_obj, "outputs", None)):
|
||||
return Response(
|
||||
{"detail": f"Compliance '{name}' does not provide an OCSF export."},
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
return self._serve_compliance_artifact(
|
||||
scan, name, "ocsf.json", "application/json"
|
||||
)
|
||||
|
||||
@action(
|
||||
detail=True,
|
||||
@@ -3749,6 +3822,16 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet):
|
||||
return queryset
|
||||
return super().filter_queryset(queryset)
|
||||
|
||||
def _optimize_tags_loading(self, queryset):
|
||||
"""Prefetch resource tags to avoid N+1 queries when serializing findings"""
|
||||
return queryset.prefetch_related(
|
||||
Prefetch(
|
||||
"resources__tags",
|
||||
queryset=ResourceTag.objects.filter(tenant_id=self.request.tenant_id),
|
||||
to_attr="prefetched_tags",
|
||||
)
|
||||
)
|
||||
|
||||
def list(self, request, *args, **kwargs):
|
||||
filtered_queryset = self.filter_queryset(self.get_queryset())
|
||||
return self.paginate_by_pk(
|
||||
@@ -7369,6 +7452,15 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
|
||||
# `check_title` / `check_description` are intentionally NOT resolved
|
||||
# here. They live in the large JSONB `check_metadata` blob (TOASTed),
|
||||
# so reading them per finding row is very expensive, and pulling them
|
||||
# in via a correlated subquery makes Django add the subquery to GROUP
|
||||
# BY, which re-evaluates it once per input row. They are identical for
|
||||
# every finding of a `check_id`, so `_post_process_aggregation` fills
|
||||
# them from the summary table's plain columns in a single batched
|
||||
# lookup scoped to the paginated page.
|
||||
|
||||
# `pass_count`, `fail_count` and `manual_count` only count non-muted
|
||||
# findings. Muted findings are tracked separately via the
|
||||
# `*_muted_count` fields.
|
||||
@@ -7439,15 +7531,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
agg_failing_since=Min(
|
||||
"first_seen_at", filter=Q(status="FAIL", muted=False)
|
||||
),
|
||||
check_title=Coalesce(
|
||||
Max(KeyTextTransform("checktitle", "check_metadata")),
|
||||
Max(KeyTextTransform("CheckTitle", "check_metadata")),
|
||||
Max(KeyTextTransform("Checktitle", "check_metadata")),
|
||||
),
|
||||
check_description=Coalesce(
|
||||
Max(KeyTextTransform("description", "check_metadata")),
|
||||
Max(KeyTextTransform("Description", "check_metadata")),
|
||||
),
|
||||
)
|
||||
.annotate(
|
||||
# Group is muted only if it has zero non-muted findings.
|
||||
@@ -7484,14 +7567,17 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
|
||||
def _get_latest_findings_per_provider(self, filtered_queryset):
|
||||
"""Keep only findings from each provider's most recent completed scan."""
|
||||
latest_scan_ids = (
|
||||
# Materialize to a literal IN list. Left as a subquery, Postgres can't
|
||||
# estimate the match count and picks a serial nested loop on
|
||||
# resource_finding_mappings when one scan dominates findings
|
||||
latest_scan_ids = list(
|
||||
Scan.objects.filter(
|
||||
tenant_id=self.request.tenant_id,
|
||||
state=StateChoices.COMPLETED,
|
||||
)
|
||||
.order_by("provider_id", "-completed_at", "-inserted_at")
|
||||
.distinct("provider_id")
|
||||
.values("id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
return filtered_queryset.filter(scan_id__in=latest_scan_ids)
|
||||
|
||||
@@ -7503,9 +7589,38 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
- Computes aggregated status (FAIL > PASS > MANUAL); the orthogonal
|
||||
``muted`` boolean is already on the row from the SQL aggregation
|
||||
- Converts provider string to list
|
||||
- Fills check_title / check_description for the findings path
|
||||
"""
|
||||
rows = list(aggregated_data)
|
||||
|
||||
# The findings-aggregation path omits check_title / check_description
|
||||
# (they sit in TOASTed JSONB; see _aggregate_findings). Fill them from
|
||||
# the summary table's plain columns in one query scoped to this page.
|
||||
# The summary-aggregation path already carries them, so skip it there.
|
||||
if rows and "check_title" not in rows[0]:
|
||||
check_ids = [row["check_id"] for row in rows]
|
||||
role = get_role(self.request.user, self.request.tenant_id)
|
||||
summaries = FindingGroupDailySummary.objects.filter(
|
||||
tenant_id=self.request.tenant_id,
|
||||
check_id__in=check_ids,
|
||||
)
|
||||
# Scope to the user's providers, mirroring get_queryset(), so titles
|
||||
# are read only from providers the user can see.
|
||||
if not role.unlimited_visibility:
|
||||
summaries = summaries.filter(provider__in=get_providers(role))
|
||||
metadata_by_check = {
|
||||
item["check_id"]: item
|
||||
for item in summaries.order_by("check_id", "-inserted_at")
|
||||
.distinct("check_id")
|
||||
.values("check_id", "check_title", "check_description")
|
||||
}
|
||||
for row in rows:
|
||||
metadata = metadata_by_check.get(row["check_id"], {})
|
||||
row["check_title"] = metadata.get("check_title")
|
||||
row["check_description"] = metadata.get("check_description")
|
||||
|
||||
results = []
|
||||
for row in aggregated_data:
|
||||
for row in rows:
|
||||
# Convert severity order back to string
|
||||
severity_order = row.get("severity_order", 1)
|
||||
row["severity"] = SEVERITY_ORDER_REVERSE.get(
|
||||
@@ -7551,7 +7666,6 @@ class FindingGroupViewSet(BaseRLSViewSet):
|
||||
|
||||
_FINDING_GROUP_SORT_MAP = {
|
||||
"check_id": "check_id",
|
||||
"check_title": "check_title",
|
||||
"severity": "severity_order",
|
||||
"status": "status_order",
|
||||
"muted": "muted",
|
||||
|
||||
@@ -26,6 +26,61 @@ celery_app.conf.result_backend_transport_options = {
|
||||
}
|
||||
celery_app.conf.visibility_timeout = BROKER_VISIBILITY_TIMEOUT
|
||||
|
||||
# Durable delivery: keep the message until the task finishes, so a worker killed
|
||||
# mid-task (deploy/OOM/eviction) does not silently drop it. Reserve one task at a
|
||||
# time so a crash exposes at most one extra reserved message.
|
||||
celery_app.conf.task_acks_late = True
|
||||
celery_app.conf.task_reject_on_worker_lost = True
|
||||
celery_app.conf.worker_prefetch_multiplier = env.int(
|
||||
"DJANGO_CELERY_WORKER_PREFETCH_MULTIPLIER", default=1
|
||||
)
|
||||
# On SIGTERM, give the worker time to finish or re-queue in-flight tasks before
|
||||
# it is forcefully killed (Celery 5.5+ soft shutdown).
|
||||
celery_app.conf.worker_soft_shutdown_timeout = env.int(
|
||||
"DJANGO_CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT", default=60
|
||||
)
|
||||
# Bound execution so a blocked task cannot pin a worker forever. Connection
|
||||
# checks get a tight limit; scans and provider/tenant deletions can legitimately
|
||||
# run for more than a day on large tenants, so they get a much higher cap.
|
||||
# The default for every other task is set as the global limit, not as a "*"
|
||||
# annotation: Celery applies the "*" entry AFTER the per-task one, so a "*" in
|
||||
# task_annotations would silently overwrite every specific limit defined below.
|
||||
_TASK_HARD_LIMIT = env.int("DJANGO_CELERY_TASK_TIME_LIMIT", default=6 * 60 * 60)
|
||||
_TASK_SOFT_LIMIT = env.int(
|
||||
"DJANGO_CELERY_TASK_SOFT_TIME_LIMIT", default=_TASK_HARD_LIMIT - 600
|
||||
)
|
||||
_LONG_TASK_HARD_LIMIT = env.int(
|
||||
"DJANGO_CELERY_LONG_TASK_TIME_LIMIT", default=48 * 60 * 60
|
||||
)
|
||||
_LONG_TASK_SOFT_LIMIT = env.int(
|
||||
"DJANGO_CELERY_LONG_TASK_SOFT_TIME_LIMIT", default=_LONG_TASK_HARD_LIMIT - 600
|
||||
)
|
||||
celery_app.conf.task_time_limit = _TASK_HARD_LIMIT
|
||||
celery_app.conf.task_soft_time_limit = _TASK_SOFT_LIMIT
|
||||
celery_app.conf.task_annotations = {
|
||||
**{
|
||||
name: {"soft_time_limit": 60, "time_limit": 120}
|
||||
for name in (
|
||||
"provider-connection-check",
|
||||
"integration-connection-check",
|
||||
"lighthouse-connection-check",
|
||||
"lighthouse-provider-connection-check",
|
||||
)
|
||||
},
|
||||
**{
|
||||
name: {
|
||||
"soft_time_limit": _LONG_TASK_SOFT_LIMIT,
|
||||
"time_limit": _LONG_TASK_HARD_LIMIT,
|
||||
}
|
||||
for name in (
|
||||
"scan-perform",
|
||||
"scan-perform-scheduled",
|
||||
"provider-deletion",
|
||||
"tenant-deletion",
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
celery_app.autodiscover_tasks(["api"])
|
||||
|
||||
|
||||
|
||||
@@ -306,3 +306,32 @@ SESSION_COOKIE_SECURE = True
|
||||
ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
|
||||
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 2880
|
||||
) # 48h
|
||||
|
||||
# Orphan task recovery feature flags. The master switch is OFF by default, so task
|
||||
# recovery is opt-in; enable it with DJANGO_TASK_RECOVERY_ENABLED=true. The per-group
|
||||
# toggles default to enabled, so once the master is on every group recovers unless a
|
||||
# group is explicitly turned off.
|
||||
TASK_RECOVERY_ENABLED = env.bool("DJANGO_TASK_RECOVERY_ENABLED", False)
|
||||
TASK_RECOVERY_SUMMARIES_ENABLED = env.bool(
|
||||
"DJANGO_TASK_RECOVERY_SUMMARIES_ENABLED", True
|
||||
)
|
||||
TASK_RECOVERY_DELETIONS_ENABLED = env.bool(
|
||||
"DJANGO_TASK_RECOVERY_DELETIONS_ENABLED", True
|
||||
)
|
||||
|
||||
|
||||
def label_postgres_connections(databases):
|
||||
"""Tag each Postgres connection with ``application_name="<component>:<alias>"``
|
||||
so connections are attributable by component in ``pg_stat_activity`` (and any
|
||||
tooling that surfaces ``application_name``). The component (api / worker /
|
||||
scan / ...) is injected per process by the container entrypoint via
|
||||
``DJANGO_APP_COMPONENT``; the alias distinguishes which pool inside the
|
||||
process owns the connection. The neo4j entry is skipped (not a Postgres
|
||||
backend). Postgres truncates ``application_name`` at 63 bytes.
|
||||
"""
|
||||
component = env.str("DJANGO_APP_COMPONENT", default="api")
|
||||
for alias, config in databases.items():
|
||||
engine = config.get("ENGINE", "")
|
||||
if engine.startswith("psqlextra") or "postgresql" in engine:
|
||||
name = f"{component}:{alias}"[:63]
|
||||
config.setdefault("OPTIONS", {})["application_name"] = name
|
||||
|
||||
@@ -54,6 +54,8 @@ DATABASES = {
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
label_postgres_connections(DATABASES) # noqa: F405
|
||||
|
||||
REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] = tuple( # noqa: F405
|
||||
render_class
|
||||
for render_class in REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] # noqa: F405
|
||||
|
||||
@@ -58,3 +58,5 @@ DATABASES = {
|
||||
}
|
||||
|
||||
DATABASES["default"] = DATABASES["prowler_user"]
|
||||
|
||||
label_postgres_connections(DATABASES) # noqa: F405
|
||||
|
||||
@@ -34,3 +34,8 @@ DRF_API_KEY = {
|
||||
# JWT
|
||||
|
||||
SIMPLE_JWT["ALGORITHM"] = "HS256" # noqa: F405
|
||||
# pyjwt >= 2.13.0 rejects an empty HMAC signing key, so HS256 tests need a real
|
||||
# key (>= 32 bytes also avoids the InsecureKeyLengthWarning). Production uses RS256.
|
||||
SIMPLE_JWT["SIGNING_KEY"] = env.str( # noqa: F405
|
||||
"DJANGO_TOKEN_SIGNING_KEY", "insecure-testing-jwt-signing-key-do-not-use-in-prod"
|
||||
)
|
||||
|
||||
@@ -571,6 +571,12 @@ def providers_fixture(tenants_fixture):
|
||||
alias="vercel_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
provider14 = Provider.objects.create(
|
||||
provider="okta",
|
||||
uid="acme.okta.com",
|
||||
alias="okta_testing",
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
|
||||
return (
|
||||
provider1,
|
||||
@@ -586,6 +592,7 @@ def providers_fixture(tenants_fixture):
|
||||
provider11,
|
||||
provider12,
|
||||
provider13,
|
||||
provider14,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from celery import current_app, states
|
||||
from celery import states
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES
|
||||
from tasks.jobs.attack_paths.db_utils import (
|
||||
_mark_scan_finished,
|
||||
recover_graph_data_ready,
|
||||
)
|
||||
from tasks.jobs.orphan_recovery import is_worker_alive as _is_worker_alive
|
||||
from tasks.jobs.orphan_recovery import revoke_task as _revoke_task
|
||||
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.db_router import MainRouter
|
||||
@@ -150,32 +152,6 @@ def _cleanup_stale_scheduled_scans(cutoff: datetime) -> list[str]:
|
||||
return cleaned_up
|
||||
|
||||
|
||||
def _is_worker_alive(worker: str) -> bool:
|
||||
"""Ping a specific Celery worker. Returns `True` if it responds or on error."""
|
||||
try:
|
||||
response = current_app.control.inspect(destination=[worker], timeout=1.0).ping()
|
||||
return response is not None and worker in response
|
||||
except Exception:
|
||||
logger.exception(f"Failed to ping worker {worker}, treating as alive")
|
||||
return True
|
||||
|
||||
|
||||
def _revoke_task(task_result, terminate: bool = True) -> None:
|
||||
"""Revoke a Celery task. Non-fatal on failure.
|
||||
|
||||
`terminate=True` SIGTERMs the worker if the task is mid-execution; use
|
||||
for EXECUTING cleanup. `terminate=False` only marks the task id revoked
|
||||
across workers, so any worker pulling the queued message discards it;
|
||||
use for SCHEDULED cleanup where the task hasn't run yet.
|
||||
"""
|
||||
try:
|
||||
kwargs = {"terminate": True, "signal": "SIGTERM"} if terminate else {}
|
||||
current_app.control.revoke(task_result.task_id, **kwargs)
|
||||
logger.info(f"Revoked task {task_result.task_id}")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to revoke task {task_result.task_id}")
|
||||
|
||||
|
||||
def _cleanup_scan(scan, task_result, reason: str) -> bool:
|
||||
"""
|
||||
Clean up a single stale `AttackPathsScan`:
|
||||
|
||||
@@ -39,11 +39,6 @@ from prowler.lib.outputs.compliance.cis.cis_oraclecloud import OracleCloudCIS
|
||||
from prowler.lib.outputs.compliance.cisa_scuba.cisa_scuba_googleworkspace import (
|
||||
GoogleWorkspaceCISASCuBA,
|
||||
)
|
||||
from prowler.lib.outputs.compliance.csa.csa_alibabacloud import AlibabaCloudCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_aws import AWSCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_azure import AzureCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_gcp import GCPCSA
|
||||
from prowler.lib.outputs.compliance.csa.csa_oraclecloud import OracleCloudCSA
|
||||
from prowler.lib.outputs.compliance.ens.ens_aws import AWSENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_azure import AzureENS
|
||||
from prowler.lib.outputs.compliance.ens.ens_gcp import GCPENS
|
||||
@@ -102,7 +97,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name == "prowler_threatscore_aws", ProwlerThreatScoreAWS),
|
||||
(lambda name: name.startswith("ccc_"), CCC_AWS),
|
||||
(lambda name: name.startswith("c5_"), AWSC5),
|
||||
(lambda name: name.startswith("csa_"), AWSCSA),
|
||||
(lambda name: name == "asd_essential_eight_aws", ASDEssentialEightAWS),
|
||||
],
|
||||
"azure": [
|
||||
@@ -113,7 +107,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name.startswith("ccc_"), CCC_Azure),
|
||||
(lambda name: name == "prowler_threatscore_azure", ProwlerThreatScoreAzure),
|
||||
(lambda name: name == "c5_azure", AzureC5),
|
||||
(lambda name: name.startswith("csa_"), AzureCSA),
|
||||
],
|
||||
"gcp": [
|
||||
(lambda name: name.startswith("cis_"), GCPCIS),
|
||||
@@ -123,7 +116,6 @@ COMPLIANCE_CLASS_MAP = {
|
||||
(lambda name: name == "prowler_threatscore_gcp", ProwlerThreatScoreGCP),
|
||||
(lambda name: name.startswith("ccc_"), CCC_GCP),
|
||||
(lambda name: name == "c5_gcp", GCPC5),
|
||||
(lambda name: name.startswith("csa_"), GCPCSA),
|
||||
],
|
||||
"kubernetes": [
|
||||
(lambda name: name.startswith("cis_"), KubernetesCIS),
|
||||
@@ -152,11 +144,9 @@ COMPLIANCE_CLASS_MAP = {
|
||||
"image": [],
|
||||
"oraclecloud": [
|
||||
(lambda name: name.startswith("cis_"), OracleCloudCIS),
|
||||
(lambda name: name.startswith("csa_"), OracleCloudCSA),
|
||||
],
|
||||
"alibabacloud": [
|
||||
(lambda name: name.startswith("cis_"), AlibabaCloudCIS),
|
||||
(lambda name: name.startswith("csa_"), AlibabaCloudCSA),
|
||||
(
|
||||
lambda name: name == "prowler_threatscore_alibabacloud",
|
||||
ProwlerThreatScoreAlibaba,
|
||||
|
||||
@@ -0,0 +1,341 @@
|
||||
"""Detect and recover orphaned Celery tasks.
|
||||
|
||||
A task is "orphaned" when its result row is non-terminal (STARTED/RECEIVED) but the
|
||||
worker that was running it is gone (deploy, OOM, eviction). We tell a real orphan
|
||||
from a still-running task by pinging the worker recorded on its `TaskResult`:
|
||||
|
||||
- worker responds -> the task is in flight, leave it alone (never double-run);
|
||||
- worker is gone -> real orphan: mark the stale result terminal (so pending/started
|
||||
alerts clear), then re-enqueue the task from its stored name + kwargs.
|
||||
|
||||
This recovers only allowlisted tasks with local, proven idempotency. Celery's
|
||||
`result_extended=True` gives us the stored `task_name`/`task_kwargs`/`worker` once
|
||||
the task starts, but external side-effect tasks are failed instead of blindly
|
||||
re-run. A small recovery cap stops a task that repeatedly kills its worker from
|
||||
looping forever.
|
||||
|
||||
This is the shared engine behind both the periodic Beat watchdog and the
|
||||
`reconcile_orphan_tasks` management command.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from celery import current_app, states
|
||||
from celery.utils.log import get_task_logger
|
||||
from django.db import connections
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# Arbitrary constant key for pg_try_advisory_lock so only one reconciliation
|
||||
# runs at a time across replicas / the watchdog / the command.
|
||||
ORPHAN_RECOVERY_LOCK_KEY = 0x70726F77 # "prow"
|
||||
|
||||
# Non-terminal states that mean "a worker had this and may have died with it".
|
||||
IN_FLIGHT_STATES = (states.STARTED, states.RECEIVED)
|
||||
|
||||
# Tasks with proven idempotency are eligible for auto re-enqueue, grouped so each
|
||||
# group can be toggled independently by a feature flag (see config.django.base).
|
||||
# Summaries clear and rewrite their own rows and deletions are idempotent. Tasks with
|
||||
# external side effects are never eligible: integration-jira would create duplicate
|
||||
# issues, integration-s3 rebuilds its upload from worker-local files that do not
|
||||
# survive a crash, and report/Security Hub recovery is out of scope.
|
||||
RECOVERY_TASK_GROUPS = {
|
||||
"summaries": {
|
||||
"scan-summary",
|
||||
"scan-compliance-overviews",
|
||||
"scan-provider-compliance-scores",
|
||||
"scan-daily-severity",
|
||||
"scan-finding-group-summaries",
|
||||
"scan-reset-ephemeral-resources",
|
||||
},
|
||||
"deletions": {"provider-deletion", "tenant-deletion"},
|
||||
}
|
||||
|
||||
|
||||
def reenqueueable_tasks() -> set[str]:
|
||||
"""Task names eligible for auto re-enqueue, honoring the per-group feature flags.
|
||||
|
||||
A group whose flag is disabled is dropped, so its orphaned tasks are marked
|
||||
terminal instead of re-enqueued.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
group_enabled = {
|
||||
"summaries": settings.TASK_RECOVERY_SUMMARIES_ENABLED,
|
||||
"deletions": settings.TASK_RECOVERY_DELETIONS_ENABLED,
|
||||
}
|
||||
return {
|
||||
task
|
||||
for group, tasks in RECOVERY_TASK_GROUPS.items()
|
||||
if group_enabled[group]
|
||||
for task in tasks
|
||||
}
|
||||
|
||||
|
||||
# Tasks the watchdog ignores entirely (not even marked terminal): scan tasks are not
|
||||
# auto-recovered, since re-running a scan is not safe to do automatically; attack-paths
|
||||
# scans are handled by their own stale-cleanup (which also drops the temp Neo4j db);
|
||||
# and the maintenance tasks must not self-recover (they run again on their own schedule).
|
||||
_SKIP_RECOVERY = {
|
||||
"scan-perform",
|
||||
"scan-perform-scheduled",
|
||||
"attack-paths-scan-perform",
|
||||
"attack-paths-cleanup-stale-scans",
|
||||
"reconcile-orphan-tasks",
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def advisory_lock(key: int = ORPHAN_RECOVERY_LOCK_KEY, using: str = "default"):
|
||||
"""Yield True if this session won a Postgres advisory lock, else False.
|
||||
|
||||
Non-blocking: losers get False and should no-op. The lock is released on
|
||||
exit (and implicitly if the session dies).
|
||||
"""
|
||||
with connections[using].cursor() as cursor:
|
||||
cursor.execute("SELECT pg_try_advisory_lock(%s)", [key])
|
||||
acquired = bool(cursor.fetchone()[0])
|
||||
try:
|
||||
yield acquired
|
||||
finally:
|
||||
if acquired:
|
||||
cursor.execute("SELECT pg_advisory_unlock(%s)", [key])
|
||||
|
||||
|
||||
def is_worker_alive(worker: str, timeout: float = 1.0) -> bool:
|
||||
"""Ping a specific Celery worker. Returns True if it responds, or on error.
|
||||
|
||||
Erring on the side of "alive" means an unreachable control bus never causes
|
||||
a still-running task to be re-enqueued.
|
||||
"""
|
||||
try:
|
||||
response = current_app.control.inspect(
|
||||
destination=[worker], timeout=timeout
|
||||
).ping()
|
||||
return response is not None and worker in response
|
||||
except Exception:
|
||||
logger.exception(f"Failed to ping worker {worker}, treating as alive")
|
||||
return True
|
||||
|
||||
|
||||
def revoke_task(task_result, terminate: bool = True) -> None:
|
||||
"""Revoke a Celery task by its TaskResult. Non-fatal on failure.
|
||||
|
||||
terminate=True SIGTERMs the worker if the task is mid-execution; terminate=False
|
||||
only marks the id revoked so any worker pulling the queued message discards it
|
||||
(use before re-enqueuing, so a later broker redelivery of the stale message is
|
||||
dropped).
|
||||
"""
|
||||
try:
|
||||
kwargs = {"terminate": True, "signal": "SIGTERM"} if terminate else {}
|
||||
current_app.control.revoke(task_result.task_id, **kwargs)
|
||||
logger.info(f"Revoked task {task_result.task_id}")
|
||||
except Exception:
|
||||
logger.exception(f"Failed to revoke task {task_result.task_id}")
|
||||
|
||||
|
||||
def _decode_celery_field(value, default):
|
||||
"""Decode django-celery-results' stored task_args/task_kwargs to a Python object.
|
||||
|
||||
The backend stores them as a (sometimes double-encoded) repr/JSON string. An
|
||||
empty or missing field returns ``default``; a non-empty value that cannot be
|
||||
decoded raises ``ValueError`` so the caller can avoid re-enqueuing a task with
|
||||
the wrong arguments.
|
||||
"""
|
||||
obj = value
|
||||
for _ in range(2): # values can be double-encoded (a string holding a repr)
|
||||
if not isinstance(obj, str):
|
||||
break
|
||||
text = obj.strip()
|
||||
if not text:
|
||||
return default
|
||||
parsed = None
|
||||
for parser in (ast.literal_eval, json.loads):
|
||||
try:
|
||||
parsed = parser(text)
|
||||
break
|
||||
except (ValueError, SyntaxError, TypeError):
|
||||
continue
|
||||
if parsed is None:
|
||||
raise ValueError(f"undecodable celery field: {text[:120]!r}")
|
||||
obj = parsed
|
||||
return default if obj is None else obj
|
||||
|
||||
|
||||
def reconcile_orphans(
|
||||
grace_minutes: int = 2,
|
||||
max_attempts: int = 3,
|
||||
window_hours: int = 6,
|
||||
dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Run the full orphan sweep under a single-flight advisory lock.
|
||||
|
||||
Recovers any orphaned in-flight task and delegates attack-paths scans that
|
||||
never reached a worker to their existing stale-cleanup. Returns a summary;
|
||||
a no-op (lock not won) is reported too.
|
||||
"""
|
||||
with advisory_lock() as acquired:
|
||||
if not acquired:
|
||||
logger.info("Orphan reconcile skipped: another run holds the lock")
|
||||
return {"acquired": False}
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
if settings.TASK_RECOVERY_ENABLED:
|
||||
# Populate the task registry so we can re-enqueue any task by name.
|
||||
import tasks.tasks # noqa: F401
|
||||
|
||||
result = _reconcile_task_results(
|
||||
grace_minutes=grace_minutes,
|
||||
max_attempts=max_attempts,
|
||||
window_hours=window_hours,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
result["enabled"] = True
|
||||
else:
|
||||
logger.info("Orphan task recovery disabled by feature flag")
|
||||
result = {"recovered": [], "failed": [], "skipped": [], "enabled": False}
|
||||
|
||||
if not dry_run:
|
||||
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
|
||||
|
||||
result["attack_paths"] = cleanup_stale_attack_paths_scans()
|
||||
|
||||
return {"acquired": True, **result}
|
||||
|
||||
|
||||
def _reconcile_task_results(
|
||||
grace_minutes: int, max_attempts: int, window_hours: int, dry_run: bool
|
||||
) -> dict:
|
||||
from django_celery_results.models import TaskResult
|
||||
|
||||
cutoff = datetime.now(tz=timezone.utc) - timedelta(minutes=grace_minutes)
|
||||
candidates = list(
|
||||
TaskResult.objects.filter(status__in=IN_FLIGHT_STATES, date_created__lt=cutoff)
|
||||
.exclude(worker__isnull=True)
|
||||
.exclude(worker="")
|
||||
.exclude(task_name__in=_SKIP_RECOVERY)
|
||||
)
|
||||
|
||||
# Ping each distinct worker at most once.
|
||||
worker_alive = {w: is_worker_alive(w) for w in {tr.worker for tr in candidates}}
|
||||
|
||||
recovered, failed, skipped = [], [], []
|
||||
for task_result in candidates:
|
||||
if worker_alive.get(task_result.worker, True):
|
||||
skipped.append(task_result.task_id) # in flight, do not double-run
|
||||
continue
|
||||
if dry_run:
|
||||
recovered.append(task_result.task_id)
|
||||
continue
|
||||
outcome = _recover_task(task_result, max_attempts, window_hours)
|
||||
(recovered if outcome == "recovered" else failed).append(task_result.task_id)
|
||||
|
||||
logger.info(
|
||||
"Orphan reconcile: recovered=%d failed=%d skipped(in-flight)=%d",
|
||||
len(recovered),
|
||||
len(failed),
|
||||
len(skipped),
|
||||
)
|
||||
return {"recovered": recovered, "failed": failed, "skipped": skipped}
|
||||
|
||||
|
||||
def _recovery_attempt_count(name: str, kwargs_repr, window_hours: int) -> int:
|
||||
"""Increment and return the recovery count for this (task, kwargs) within the
|
||||
window. Backed by Valkey so it survives result-row churn (a worker processing
|
||||
the revoke can blank the TaskResult fields). Fail-open if Valkey is down (the
|
||||
broker being unreachable means nothing is running anyway).
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
try:
|
||||
import redis
|
||||
|
||||
client = redis.from_url(settings.CELERY_BROKER_URL)
|
||||
signature = f"{name}|{kwargs_repr}".encode()
|
||||
key = (
|
||||
"orphan-recovery:"
|
||||
+ hashlib.sha1(signature, usedforsecurity=False).hexdigest()
|
||||
)
|
||||
count = client.incr(key)
|
||||
if count == 1:
|
||||
client.expire(key, max(1, window_hours) * 3600)
|
||||
return int(count)
|
||||
except Exception:
|
||||
logger.exception("Recovery-attempt counter unavailable; allowing recovery")
|
||||
return 1
|
||||
|
||||
|
||||
def _recover_task(task_result, max_attempts: int, window_hours: int) -> str:
|
||||
"""Recover one orphaned task. Returns 'recovered' or 'failed'."""
|
||||
# Capture name/args/kwargs now: revoking can let a worker blank the row.
|
||||
name = task_result.task_name
|
||||
args_repr = task_result.task_args
|
||||
kwargs_repr = task_result.task_kwargs
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
|
||||
# Drop any future broker redelivery of the stale message.
|
||||
revoke_task(task_result, terminate=False)
|
||||
|
||||
# Mark the stale result terminal so "pending/started forever" alerts clear.
|
||||
task_result.status = states.REVOKED
|
||||
task_result.date_done = now
|
||||
task_result.save(update_fields=["status", "date_done"])
|
||||
|
||||
if name not in reenqueueable_tasks():
|
||||
logger.warning(
|
||||
"Orphan %s (%s) not re-enqueued: not allowlisted for auto recovery",
|
||||
task_result.task_id,
|
||||
name,
|
||||
)
|
||||
return "failed"
|
||||
|
||||
# Count the attempt only once the task is allowlisted, so a task sitting in a
|
||||
# disabled group does not burn its recovery budget while the flag is off (and is
|
||||
# not already over the cap the moment the group is re-enabled).
|
||||
attempt = _recovery_attempt_count(name, kwargs_repr, window_hours)
|
||||
if attempt > max_attempts:
|
||||
logger.warning(
|
||||
"Orphan %s (%s) not re-enqueued: recovery cap reached (%d/%d)",
|
||||
task_result.task_id,
|
||||
name,
|
||||
attempt,
|
||||
max_attempts,
|
||||
)
|
||||
return "failed"
|
||||
|
||||
task_obj = current_app.tasks.get(name)
|
||||
if task_obj is None:
|
||||
logger.error(
|
||||
"Orphan %s: task %s not registered, cannot re-enqueue",
|
||||
task_result.task_id,
|
||||
name,
|
||||
)
|
||||
return "failed"
|
||||
|
||||
try:
|
||||
args = _decode_celery_field(args_repr, [])
|
||||
kwargs = _decode_celery_field(kwargs_repr, {})
|
||||
except ValueError:
|
||||
logger.error(
|
||||
"Orphan %s (%s): could not decode stored args/kwargs, not re-enqueuing",
|
||||
task_result.task_id,
|
||||
name,
|
||||
)
|
||||
return "failed"
|
||||
new_task_id = str(uuid4())
|
||||
task_obj.apply_async(
|
||||
args=list(args) if isinstance(args, (list, tuple)) else [],
|
||||
kwargs=kwargs if isinstance(kwargs, dict) else {},
|
||||
task_id=new_task_id,
|
||||
)
|
||||
logger.info(
|
||||
"Re-enqueued orphan %s (%s) as %s", task_result.task_id, name, new_task_id
|
||||
)
|
||||
return "recovered"
|
||||
@@ -29,7 +29,10 @@ from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, Scan, ScanSummary, StateChoices, ThreatScoreSnapshot
|
||||
from api.utils import initialize_prowler_provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
Compliance,
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
@@ -571,7 +574,7 @@ def generate_csa_report(
|
||||
Args:
|
||||
tenant_id: The tenant ID for Row-Level Security context.
|
||||
scan_id: ID of the scan executed by Prowler.
|
||||
compliance_id: ID of the compliance framework (e.g., "csa_ccm_4.0_aws").
|
||||
compliance_id: ID of the compliance framework (e.g., "csa_ccm_4.0").
|
||||
output_path: Output PDF file path.
|
||||
provider_id: Provider ID for the scan.
|
||||
only_failed: If True, only include failed requirements in detailed section.
|
||||
@@ -883,9 +886,11 @@ def generate_compliance_reports(
|
||||
frameworks_bulk.get(f"nis2_{provider_type}")
|
||||
)
|
||||
if generate_csa:
|
||||
pending_checks_by_framework["csa"] = _get_compliance_check_ids(
|
||||
frameworks_bulk.get(f"csa_ccm_4.0_{provider_type}")
|
||||
)
|
||||
# csa_ccm_4.0 lives at the top level, not under compliance/{provider}/.
|
||||
csa_framework = frameworks_bulk.get(
|
||||
"csa_ccm_4.0"
|
||||
) or get_bulk_compliance_frameworks_universal(provider_type).get("csa_ccm_4.0")
|
||||
pending_checks_by_framework["csa"] = _get_compliance_check_ids(csa_framework)
|
||||
if generate_cis and latest_cis:
|
||||
pending_checks_by_framework["cis"] = _get_compliance_check_ids(
|
||||
frameworks_bulk.get(latest_cis)
|
||||
@@ -1183,7 +1188,7 @@ def generate_compliance_reports(
|
||||
if generate_csa:
|
||||
generated_report_keys.append("csa")
|
||||
csa_path = output_paths["csa"]
|
||||
compliance_id_csa = f"csa_ccm_4.0_{provider_type}"
|
||||
compliance_id_csa = "csa_ccm_4.0"
|
||||
pdf_path_csa = f"{csa_path}_csa_report.pdf"
|
||||
logger.info("Generating CSA CCM report with compliance %s", compliance_id_csa)
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import time
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
@@ -26,7 +27,10 @@ from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, StatusChoices
|
||||
from api.utils import initialize_prowler_provider
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.check.compliance_models import (
|
||||
Compliance,
|
||||
get_bulk_compliance_frameworks_universal,
|
||||
)
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
|
||||
from .components import (
|
||||
@@ -222,6 +226,46 @@ def get_requirement_metadata(
|
||||
return None
|
||||
|
||||
|
||||
def _universal_attributes_to_list(attributes) -> list:
|
||||
"""Flatten a universal requirement's ``attributes`` into a list of objects
|
||||
with attribute access. MITRE wraps its list under ``_raw_attributes``."""
|
||||
if isinstance(attributes, dict) and "_raw_attributes" in attributes:
|
||||
entries = attributes.get("_raw_attributes") or []
|
||||
return [
|
||||
SimpleNamespace(**entry) for entry in entries if isinstance(entry, dict)
|
||||
]
|
||||
if isinstance(attributes, dict):
|
||||
return [SimpleNamespace(**attributes)] if attributes else []
|
||||
return list(attributes or [])
|
||||
|
||||
|
||||
def _adapt_universal_to_legacy(framework, provider_type: str) -> SimpleNamespace:
|
||||
"""Expose a universal ``ComplianceFramework`` under the legacy ``Compliance``
|
||||
attribute names used by the PDF pipeline."""
|
||||
provider_key = (provider_type or "").lower()
|
||||
requirements = []
|
||||
for requirement in framework.requirements:
|
||||
checks_by_provider = (
|
||||
requirement.checks if isinstance(requirement.checks, dict) else {}
|
||||
)
|
||||
requirements.append(
|
||||
SimpleNamespace(
|
||||
Id=requirement.id,
|
||||
Description=requirement.description or "",
|
||||
Checks=list(checks_by_provider.get(provider_key, [])),
|
||||
Attributes=_universal_attributes_to_list(requirement.attributes),
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(
|
||||
Framework=framework.framework,
|
||||
Name=framework.name,
|
||||
Version=framework.version or "",
|
||||
Description=framework.description or "",
|
||||
Provider=framework.provider or provider_type,
|
||||
Requirements=requirements,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# PDF Styles Cache
|
||||
# =============================================================================
|
||||
@@ -869,9 +913,18 @@ class BaseComplianceReportGenerator(ABC):
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
provider_type = provider_obj.provider
|
||||
|
||||
# Load compliance framework
|
||||
frameworks_bulk = Compliance.get_bulk(provider_type)
|
||||
compliance_obj = frameworks_bulk.get(compliance_id)
|
||||
# Load compliance framework — fall back to the universal loader
|
||||
# for top-level JSONs (e.g. csa_ccm_4.0) that Compliance.get_bulk
|
||||
# does not scan.
|
||||
compliance_obj = Compliance.get_bulk(provider_type).get(compliance_id)
|
||||
if not compliance_obj:
|
||||
universal_framework = get_bulk_compliance_frameworks_universal(
|
||||
provider_type
|
||||
).get(compliance_id)
|
||||
if universal_framework:
|
||||
compliance_obj = _adapt_universal_to_legacy(
|
||||
universal_framework, provider_type
|
||||
)
|
||||
|
||||
if not compliance_obj:
|
||||
raise ValueError(f"Compliance framework not found: {compliance_id}")
|
||||
|
||||
+465
-275
@@ -42,7 +42,6 @@ from api.db_utils import (
|
||||
SET_CONFIG_QUERY,
|
||||
psycopg_connection,
|
||||
rls_transaction,
|
||||
update_objects_in_batches,
|
||||
)
|
||||
from api.exceptions import ProviderConnectionError
|
||||
from api.models import (
|
||||
@@ -59,6 +58,7 @@ from api.models import (
|
||||
ResourceFindingMapping,
|
||||
ResourceScanSummary,
|
||||
ResourceTag,
|
||||
ResourceTagMapping,
|
||||
Scan,
|
||||
ScanCategorySummary,
|
||||
ScanGroupSummary,
|
||||
@@ -97,8 +97,16 @@ COMPLIANCE_REQUIREMENT_COPY_COLUMNS = (
|
||||
)
|
||||
# Controls how many findings we process per micro-batch before flushing to DB writes
|
||||
FINDINGS_MICRO_BATCH_SIZE = env.int("DJANGO_FINDINGS_MICRO_BATCH_SIZE", default=3000)
|
||||
# Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres
|
||||
SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=500)
|
||||
# Controls how many rows each ORM bulk_create/bulk_update call sends to Postgres.
|
||||
SCAN_DB_BATCH_SIZE = env.int("DJANGO_SCAN_DB_BATCH_SIZE", default=1000)
|
||||
# Throttle scan progress persistence: minimum progress delta (fraction 0-1)
|
||||
# between two persisted progress updates.
|
||||
PROGRESS_THROTTLE_DELTA = env.float("DJANGO_SCAN_PROGRESS_THROTTLE_DELTA", default=0.01)
|
||||
# Throttle scan progress persistence: maximum seconds without persisting progress
|
||||
# regardless of delta (so slow checks still show progress in the UI).
|
||||
PROGRESS_THROTTLE_SECONDS = env.float(
|
||||
"DJANGO_SCAN_PROGRESS_THROTTLE_SECONDS", default=10.0
|
||||
)
|
||||
|
||||
ATTACK_SURFACE_PROVIDER_COMPATIBILITY = {
|
||||
"internet-exposed": None, # Compatible with all providers
|
||||
@@ -468,9 +476,12 @@ def _create_compliance_summaries(
|
||||
)
|
||||
)
|
||||
|
||||
# Bulk insert summaries
|
||||
if summary_objects:
|
||||
with rls_transaction(tenant_id):
|
||||
# Idempotent re-run: clear this scan's prior summaries before re-inserting, so a
|
||||
# recovered scan-compliance-overviews run reflects its own re-derived rows instead
|
||||
# of keeping a stale one (bulk_create ignore_conflicts alone would keep the old).
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceOverviewSummary.objects.filter(scan_id=scan_id).delete()
|
||||
if summary_objects:
|
||||
ComplianceOverviewSummary.objects.bulk_create(
|
||||
summary_objects, batch_size=500, ignore_conflicts=True
|
||||
)
|
||||
@@ -528,16 +539,26 @@ def _process_finding_micro_batch(
|
||||
"""
|
||||
# Accumulate objects for bulk operations
|
||||
findings_to_create = []
|
||||
mappings_to_create = []
|
||||
dirty_resources = {}
|
||||
resources_with_new_tag_mappings: set[str] = set()
|
||||
resource_denormalized_data = [] # (finding_instance, resource_instance) pairs
|
||||
tag_mappings_to_create: list[ResourceTagMapping] = []
|
||||
skipped_findings_count = 0 # Track findings skipped due to UID length
|
||||
|
||||
# Prefetch last statuses for all findings in this batch
|
||||
# TEMPORARY WORKAROUND: Filter out UIDs > 300 chars to avoid query errors
|
||||
finding_uids = [
|
||||
f.uid for f in findings_batch if f is not None and len(f.uid) <= 300
|
||||
]
|
||||
# Separate findings into those persistable (uid <= 300) and over-limit.
|
||||
# Resources/tags ARE still resolved for over-limit findings to preserve the
|
||||
# original behavior (resources are persisted even when their finding is dropped).
|
||||
non_null_findings = [f for f in findings_batch if f is not None]
|
||||
persistable_findings = [f for f in non_null_findings if len(f.uid) <= 300]
|
||||
skipped_findings_count = len(non_null_findings) - len(persistable_findings)
|
||||
none_count = len(findings_batch) - len(non_null_findings)
|
||||
if none_count:
|
||||
logger.error(
|
||||
f"{none_count} None finding(s) detected on scan {scan_instance.id}."
|
||||
)
|
||||
|
||||
# Prefetch last statuses for all persistable findings in this batch (read replica)
|
||||
finding_uids = [f.uid for f in persistable_findings]
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
last_statuses = {
|
||||
item["uid"]: (item["status"], item["first_seen_at"])
|
||||
@@ -548,281 +569,411 @@ def _process_finding_micro_batch(
|
||||
.order_by("uid", "-inserted_at")
|
||||
.distinct("uid")
|
||||
}
|
||||
# Update cache
|
||||
for uid, data in last_statuses.items():
|
||||
if uid not in last_status_cache:
|
||||
last_status_cache[uid] = data
|
||||
|
||||
# Process each finding in the batch
|
||||
for finding in findings_batch:
|
||||
if finding is None:
|
||||
logger.error(f"None finding detected on scan {scan_instance.id}.")
|
||||
continue
|
||||
# All DB writes for this micro-batch run inside ONE rls_transaction,
|
||||
# with deadlock-retry at micro-batch granularity instead of per-finding.
|
||||
for attempt in range(CELERY_DEADLOCK_ATTEMPTS):
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
# 1) Pre-resolve Resources in bulk
|
||||
# Collect all uids referenced by this batch that are not in cache yet.
|
||||
# NOTE: we intentionally include empty-string uids here. The SDK
|
||||
# explicitly emits findings with `resource_uid=""` for some flows
|
||||
# (IaC scans, some Azure/GCP/K8s checks). The original
|
||||
# `get_or_create` behavior was to create/share a Resource with
|
||||
# uid="" for these findings rather than dropping them. Preserve
|
||||
# that behavior; do NOT filter by truthiness.
|
||||
batch_resource_uids: set[str] = set()
|
||||
for f in non_null_findings:
|
||||
if f.resource_uid not in resource_cache:
|
||||
batch_resource_uids.add(f.resource_uid)
|
||||
|
||||
# Process resource with deadlock retry
|
||||
for attempt in range(CELERY_DEADLOCK_ATTEMPTS):
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
resource_uid = finding.resource_uid
|
||||
if resource_uid not in resource_cache:
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
resource_instance, _ = Resource.objects.get_or_create(
|
||||
if batch_resource_uids:
|
||||
existing_resources = {
|
||||
r.uid: r
|
||||
for r in Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider=provider_instance,
|
||||
uid=resource_uid,
|
||||
defaults={
|
||||
"region": finding.region,
|
||||
"service": finding.service_name,
|
||||
"type": finding.resource_type,
|
||||
"name": finding.resource_name,
|
||||
"groups": [group] if group else None,
|
||||
},
|
||||
provider_id=provider_instance.id,
|
||||
uid__in=batch_resource_uids,
|
||||
)
|
||||
resource_cache[resource_uid] = resource_instance
|
||||
resource_failed_findings_cache[resource_uid] = 0
|
||||
else:
|
||||
resource_instance = resource_cache[resource_uid]
|
||||
break
|
||||
except (OperationalError, IntegrityError) as db_err:
|
||||
if attempt < CELERY_DEADLOCK_ATTEMPTS - 1:
|
||||
logger.warning(
|
||||
f"{'Deadlock error' if isinstance(db_err, OperationalError) else 'Integrity error'} "
|
||||
f"detected when processing resource {resource_uid} on scan {scan_instance.id}. Retrying..."
|
||||
}
|
||||
missing_uids = batch_resource_uids - existing_resources.keys()
|
||||
if missing_uids:
|
||||
# Build defaults from the first finding referencing each uid.
|
||||
first_finding_per_uid: dict[str, ProwlerFinding] = {}
|
||||
for f in non_null_findings:
|
||||
if f.resource_uid in missing_uids:
|
||||
first_finding_per_uid.setdefault(f.resource_uid, f)
|
||||
resources_to_create = []
|
||||
for uid in missing_uids:
|
||||
f = first_finding_per_uid[uid]
|
||||
check_metadata = f.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
resources_to_create.append(
|
||||
Resource(
|
||||
tenant_id=tenant_id,
|
||||
provider=provider_instance,
|
||||
uid=uid,
|
||||
region=f.region,
|
||||
service=f.service_name,
|
||||
type=f.resource_type,
|
||||
name=f.resource_name,
|
||||
groups=[group] if group else None,
|
||||
)
|
||||
)
|
||||
Resource.objects.bulk_create(
|
||||
resources_to_create,
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "provider_id", "uid"],
|
||||
)
|
||||
# Re-fetch to obtain instances we just created AND any
|
||||
# created concurrently by another scan against the same provider.
|
||||
existing_resources.update(
|
||||
{
|
||||
r.uid: r
|
||||
for r in Resource.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_instance.id,
|
||||
uid__in=missing_uids,
|
||||
)
|
||||
}
|
||||
)
|
||||
for uid, r in existing_resources.items():
|
||||
resource_cache[uid] = r
|
||||
resource_failed_findings_cache.setdefault(uid, 0)
|
||||
|
||||
# 2) Pre-resolve ResourceTags in bulk
|
||||
batch_tag_kv: set[tuple[str, str]] = set()
|
||||
for f in non_null_findings:
|
||||
for k, v in f.resource_tags.items():
|
||||
if (k, v) not in tag_cache:
|
||||
batch_tag_kv.add((k, v))
|
||||
|
||||
if batch_tag_kv:
|
||||
keys_to_query = {k for k, _ in batch_tag_kv}
|
||||
existing_tags = {
|
||||
(t.key, t.value): t
|
||||
for t in ResourceTag.objects.filter(
|
||||
tenant_id=tenant_id, key__in=keys_to_query
|
||||
)
|
||||
if (t.key, t.value) in batch_tag_kv
|
||||
}
|
||||
missing_kv = batch_tag_kv - existing_tags.keys()
|
||||
if missing_kv:
|
||||
ResourceTag.objects.bulk_create(
|
||||
[
|
||||
ResourceTag(tenant_id=tenant_id, key=k, value=v)
|
||||
for k, v in missing_kv
|
||||
],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "key", "value"],
|
||||
)
|
||||
existing_tags.update(
|
||||
{
|
||||
(t.key, t.value): t
|
||||
for t in ResourceTag.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
key__in={k for k, _ in missing_kv},
|
||||
)
|
||||
if (t.key, t.value) in missing_kv
|
||||
}
|
||||
)
|
||||
tag_cache.update(existing_tags)
|
||||
|
||||
# 3) Per-finding in-memory processing
|
||||
for finding in non_null_findings:
|
||||
resource_uid = finding.resource_uid
|
||||
resource_instance = resource_cache.get(resource_uid)
|
||||
if resource_instance is None:
|
||||
# Should be unreachable after the pre-resolve step. Defensive log.
|
||||
logger.error(
|
||||
f"Resource {resource_uid} missing from cache after pre-resolve "
|
||||
f"on scan {scan_instance.id}; skipping finding."
|
||||
)
|
||||
continue
|
||||
|
||||
# Detect resource field changes (defer save until end-of-batch bulk_update).
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
updated = False
|
||||
if finding.region and resource_instance.region != finding.region:
|
||||
resource_instance.region = finding.region
|
||||
updated = True
|
||||
if resource_instance.service != finding.service_name:
|
||||
resource_instance.service = finding.service_name
|
||||
updated = True
|
||||
if resource_instance.type != finding.resource_type:
|
||||
resource_instance.type = finding.resource_type
|
||||
updated = True
|
||||
if resource_instance.metadata != finding.resource_metadata:
|
||||
resource_instance.metadata = json.dumps(
|
||||
finding.resource_metadata, cls=CustomEncoder
|
||||
)
|
||||
updated = True
|
||||
if resource_instance.details != finding.resource_details:
|
||||
resource_instance.details = finding.resource_details
|
||||
updated = True
|
||||
if resource_instance.partition != finding.partition:
|
||||
resource_instance.partition = finding.partition
|
||||
updated = True
|
||||
if group and (
|
||||
not resource_instance.groups
|
||||
or group not in resource_instance.groups
|
||||
):
|
||||
resource_instance.groups = (resource_instance.groups or []) + [
|
||||
group
|
||||
]
|
||||
updated = True
|
||||
|
||||
if updated:
|
||||
dirty_resources[resource_uid] = resource_instance
|
||||
|
||||
# Accumulate ResourceTagMapping rows; bulk_create at end of block.
|
||||
for k, v in finding.resource_tags.items():
|
||||
tag_instance = tag_cache.get((k, v))
|
||||
if tag_instance is None:
|
||||
# Should not happen after pre-resolve; skip defensively.
|
||||
continue
|
||||
tag_mappings_to_create.append(
|
||||
ResourceTagMapping(
|
||||
tenant_id=tenant_id,
|
||||
resource=resource_instance,
|
||||
tag=tag_instance,
|
||||
)
|
||||
)
|
||||
|
||||
unique_resources.add(
|
||||
(resource_instance.uid, resource_instance.region)
|
||||
)
|
||||
time.sleep(0.1 * (2**attempt))
|
||||
continue
|
||||
else:
|
||||
raise db_err
|
||||
|
||||
# Track resource field changes (defer save)
|
||||
updated = False
|
||||
check_metadata = finding.get_metadata()
|
||||
group = check_metadata.get("resourcegroup") or None
|
||||
if finding.region and resource_instance.region != finding.region:
|
||||
resource_instance.region = finding.region
|
||||
updated = True
|
||||
if resource_instance.service != finding.service_name:
|
||||
resource_instance.service = finding.service_name
|
||||
updated = True
|
||||
if resource_instance.type != finding.resource_type:
|
||||
resource_instance.type = finding.resource_type
|
||||
updated = True
|
||||
if resource_instance.metadata != finding.resource_metadata:
|
||||
resource_instance.metadata = json.dumps(
|
||||
finding.resource_metadata, cls=CustomEncoder
|
||||
)
|
||||
updated = True
|
||||
if resource_instance.details != finding.resource_details:
|
||||
resource_instance.details = finding.resource_details
|
||||
updated = True
|
||||
if resource_instance.partition != finding.partition:
|
||||
resource_instance.partition = finding.partition
|
||||
updated = True
|
||||
if group and (
|
||||
not resource_instance.groups or group not in resource_instance.groups
|
||||
):
|
||||
resource_instance.groups = (resource_instance.groups or []) + [group]
|
||||
updated = True
|
||||
# TEMPORARY WORKAROUND: Skip findings with UID > 300 chars
|
||||
# TODO: Remove this after implementing text field migration for finding.uid
|
||||
if len(finding.uid) > 300:
|
||||
logger.warning(
|
||||
f"Skipping finding with UID exceeding 300 characters. "
|
||||
f"Length: {len(finding.uid)}, "
|
||||
f"Check: {finding.check_id}, "
|
||||
f"Resource: {finding.resource_name}, "
|
||||
f"UID: {finding.uid}"
|
||||
)
|
||||
continue
|
||||
|
||||
if updated:
|
||||
dirty_resources[resource_uid] = resource_instance
|
||||
|
||||
# Process tags
|
||||
tags = []
|
||||
with rls_transaction(tenant_id):
|
||||
for key, value in finding.resource_tags.items():
|
||||
tag_key = (key, value)
|
||||
if tag_key not in tag_cache:
|
||||
tag_instance, _ = ResourceTag.objects.get_or_create(
|
||||
tenant_id=tenant_id, key=key, value=value
|
||||
finding_uid = finding.uid
|
||||
last_status, last_first_seen_at = last_status_cache.get(
|
||||
finding_uid, (None, None)
|
||||
)
|
||||
tag_cache[tag_key] = tag_instance
|
||||
else:
|
||||
tag_instance = tag_cache[tag_key]
|
||||
tags.append(tag_instance)
|
||||
resource_instance.upsert_or_delete_tags(tags=tags)
|
||||
|
||||
unique_resources.add((resource_instance.uid, resource_instance.region))
|
||||
status = FindingStatus[finding.status]
|
||||
delta = _create_finding_delta(last_status, status)
|
||||
|
||||
# Prepare finding data
|
||||
finding_uid = finding.uid
|
||||
if not last_first_seen_at:
|
||||
last_first_seen_at = datetime.now(tz=timezone.utc)
|
||||
|
||||
# TEMPORARY WORKAROUND: Skip findings with UID > 300 chars
|
||||
# TODO: Remove this after implementing text field migration for finding.uid
|
||||
if len(finding_uid) > 300:
|
||||
skipped_findings_count += 1
|
||||
logger.warning(
|
||||
f"Skipping finding with UID exceeding 300 characters. "
|
||||
f"Length: {len(finding_uid)}, "
|
||||
f"Check: {finding.check_id}, "
|
||||
f"Resource: {finding.resource_name}, "
|
||||
f"UID: {finding_uid}"
|
||||
)
|
||||
continue
|
||||
# Determine if finding should be muted and why
|
||||
# Priority: mutelist processor (highest) > manual mute rules
|
||||
is_muted = False
|
||||
muted_reason = None
|
||||
if finding.muted:
|
||||
is_muted = True
|
||||
muted_reason = "Muted by mutelist"
|
||||
elif finding_uid in mute_rules_cache:
|
||||
is_muted = True
|
||||
muted_reason = mute_rules_cache[finding_uid]
|
||||
|
||||
last_status, last_first_seen_at = last_status_cache.get(
|
||||
finding_uid, (None, None)
|
||||
)
|
||||
if status == FindingStatus.FAIL and not is_muted:
|
||||
resource_failed_findings_cache[resource_uid] += 1
|
||||
|
||||
status = FindingStatus[finding.status]
|
||||
delta = _create_finding_delta(last_status, status)
|
||||
check_metadata["compliance"] = finding.compliance
|
||||
finding_instance = Finding(
|
||||
tenant_id=tenant_id,
|
||||
uid=finding_uid,
|
||||
delta=delta,
|
||||
check_metadata=check_metadata,
|
||||
status=status,
|
||||
status_extended=finding.status_extended,
|
||||
severity=finding.severity,
|
||||
impact=finding.severity,
|
||||
raw_result=finding.raw,
|
||||
check_id=finding.check_id,
|
||||
scan=scan_instance,
|
||||
first_seen_at=last_first_seen_at,
|
||||
muted=is_muted,
|
||||
muted_at=datetime.now(tz=timezone.utc) if is_muted else None,
|
||||
muted_reason=muted_reason,
|
||||
compliance=finding.compliance,
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
resource_groups=check_metadata.get("resourcegroup") or None,
|
||||
# Denormalized resource arrays populated directly on insert
|
||||
# (was previously a separate bulk_update; saves a CASE WHEN
|
||||
# over thousands of rows per micro-batch).
|
||||
resource_regions=[resource_instance.region]
|
||||
if resource_instance.region
|
||||
else [],
|
||||
resource_services=[resource_instance.service]
|
||||
if resource_instance.service
|
||||
else [],
|
||||
resource_types=[resource_instance.type]
|
||||
if resource_instance.type
|
||||
else [],
|
||||
)
|
||||
findings_to_create.append(finding_instance)
|
||||
resource_denormalized_data.append(
|
||||
(finding_instance, resource_instance)
|
||||
)
|
||||
|
||||
if not last_first_seen_at:
|
||||
last_first_seen_at = datetime.now(tz=timezone.utc)
|
||||
scan_resource_cache.add(
|
||||
(
|
||||
str(resource_instance.id),
|
||||
resource_instance.service,
|
||||
resource_instance.region,
|
||||
resource_instance.type,
|
||||
)
|
||||
)
|
||||
|
||||
# Determine if finding should be muted and why
|
||||
# Priority: mutelist processor (highest) > manual mute rules
|
||||
is_muted = False
|
||||
muted_reason = None
|
||||
aggregate_category_counts(
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
cache=scan_categories_cache,
|
||||
)
|
||||
|
||||
# Check mutelist processor first (highest priority)
|
||||
if finding.muted:
|
||||
is_muted = True
|
||||
muted_reason = "Muted by mutelist"
|
||||
# If not muted by mutelist, check manual mute rules
|
||||
elif finding_uid in mute_rules_cache:
|
||||
is_muted = True
|
||||
muted_reason = mute_rules_cache[finding_uid]
|
||||
aggregate_resource_group_counts(
|
||||
resource_group=check_metadata.get("resourcegroup") or None,
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
resource_uid=resource_instance.uid if resource_instance else "",
|
||||
cache=scan_resource_groups_cache,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Increment failed_findings_count cache if needed
|
||||
if status == FindingStatus.FAIL and not is_muted:
|
||||
resource_failed_findings_cache[resource_uid] += 1
|
||||
# 4) Bulk create ResourceTagMappings
|
||||
# Replaces the original per-resource `upsert_or_delete_tags`
|
||||
# (which did one `update_or_create` + SELECT FOR UPDATE per mapping).
|
||||
if tag_mappings_to_create:
|
||||
# Pre-SELECT existing pairs: `bulk_create(ignore_conflicts=True)`
|
||||
# does not populate `pk`, so we cannot tell new vs existing from
|
||||
# the result; we need that to bump `updated_at` only on resources
|
||||
# that actually gain a mapping.
|
||||
candidate_resource_ids = {
|
||||
m.resource_id for m in tag_mappings_to_create
|
||||
}
|
||||
candidate_tag_ids = {m.tag_id for m in tag_mappings_to_create}
|
||||
existing_pairs = set(
|
||||
ResourceTagMapping.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
resource_id__in=candidate_resource_ids,
|
||||
tag_id__in=candidate_tag_ids,
|
||||
).values_list("resource_id", "tag_id")
|
||||
)
|
||||
resource_uid_by_id = {
|
||||
str(r.id): uid for uid, r in resource_cache.items()
|
||||
}
|
||||
for m in tag_mappings_to_create:
|
||||
if (m.resource_id, m.tag_id) not in existing_pairs:
|
||||
uid = resource_uid_by_id.get(str(m.resource_id))
|
||||
if uid is not None:
|
||||
resources_with_new_tag_mappings.add(uid)
|
||||
|
||||
# Create finding object (don't save yet)
|
||||
check_metadata = finding.get_metadata()
|
||||
check_metadata["compliance"] = finding.compliance
|
||||
finding_instance = Finding(
|
||||
tenant_id=tenant_id,
|
||||
uid=finding_uid,
|
||||
delta=delta,
|
||||
check_metadata=check_metadata,
|
||||
status=status,
|
||||
status_extended=finding.status_extended,
|
||||
severity=finding.severity,
|
||||
impact=finding.severity,
|
||||
raw_result=finding.raw,
|
||||
check_id=finding.check_id,
|
||||
scan=scan_instance,
|
||||
first_seen_at=last_first_seen_at,
|
||||
muted=is_muted,
|
||||
muted_at=datetime.now(tz=timezone.utc) if is_muted else None,
|
||||
muted_reason=muted_reason,
|
||||
compliance=finding.compliance,
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
resource_groups=check_metadata.get("resourcegroup") or None,
|
||||
)
|
||||
findings_to_create.append(finding_instance)
|
||||
resource_denormalized_data.append((finding_instance, resource_instance))
|
||||
ResourceTagMapping.objects.bulk_create(
|
||||
tag_mappings_to_create,
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "resource_id", "tag_id"],
|
||||
)
|
||||
|
||||
# Track for scan summary
|
||||
scan_resource_cache.add(
|
||||
(
|
||||
str(resource_instance.id),
|
||||
resource_instance.service,
|
||||
resource_instance.region,
|
||||
resource_instance.type,
|
||||
)
|
||||
)
|
||||
# 5) Bulk create Findings
|
||||
if findings_to_create:
|
||||
Finding.objects.bulk_create(
|
||||
findings_to_create, batch_size=SCAN_DB_BATCH_SIZE
|
||||
)
|
||||
|
||||
# Track categories with counts for ScanCategorySummary by (category, severity)
|
||||
aggregate_category_counts(
|
||||
categories=check_metadata.get("categories", []) or [],
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
cache=scan_categories_cache,
|
||||
)
|
||||
# 6) Bulk create ResourceFindingMapping rows
|
||||
mappings_to_create = [
|
||||
ResourceFindingMapping(
|
||||
tenant_id=tenant_id,
|
||||
resource=resource_instance,
|
||||
finding=finding_instance,
|
||||
)
|
||||
for finding_instance, resource_instance in resource_denormalized_data
|
||||
]
|
||||
if mappings_to_create:
|
||||
created_mappings = ResourceFindingMapping.objects.bulk_create(
|
||||
mappings_to_create,
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "resource_id", "finding_id"],
|
||||
)
|
||||
inserted = sum(1 for m in created_mappings if m.pk)
|
||||
if inserted != len(mappings_to_create):
|
||||
logger.error(
|
||||
f"scan {scan_instance.id}: expected "
|
||||
f"{len(mappings_to_create)} ResourceFindingMapping rows, "
|
||||
f"inserted {inserted}. Rolling back micro-batch."
|
||||
)
|
||||
|
||||
# Track resource groups with counts for ScanGroupSummary
|
||||
aggregate_resource_group_counts(
|
||||
resource_group=check_metadata.get("resourcegroup") or None,
|
||||
severity=finding.severity.value,
|
||||
status=status.value,
|
||||
delta=delta.value if delta else None,
|
||||
muted=is_muted,
|
||||
resource_uid=resource_instance.uid if resource_instance else "",
|
||||
cache=scan_resource_groups_cache,
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Bulk operations within single transaction
|
||||
with rls_transaction(tenant_id):
|
||||
# Bulk create findings
|
||||
if findings_to_create:
|
||||
Finding.objects.bulk_create(
|
||||
findings_to_create, batch_size=SCAN_DB_BATCH_SIZE
|
||||
)
|
||||
|
||||
# Bulk create resource-finding mappings
|
||||
for finding_instance, resource_instance in resource_denormalized_data:
|
||||
mappings_to_create.append(
|
||||
ResourceFindingMapping(
|
||||
tenant_id=tenant_id,
|
||||
resource=resource_instance,
|
||||
finding=finding_instance,
|
||||
# 7) Bulk update Resources
|
||||
# Union of:
|
||||
# - resources whose fields changed (dirty_resources)
|
||||
# - resources that got new tag mappings (need updated_at bump,
|
||||
# preserving the original `self.save(update_fields=["updated_at"])`
|
||||
# behavior of `upsert_or_delete_tags`)
|
||||
all_resource_uids_to_touch = (
|
||||
set(dirty_resources.keys()) | resources_with_new_tag_mappings
|
||||
)
|
||||
)
|
||||
|
||||
if mappings_to_create:
|
||||
created_mappings = ResourceFindingMapping.objects.bulk_create(
|
||||
mappings_to_create,
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
ignore_conflicts=True,
|
||||
unique_fields=["tenant_id", "resource_id", "finding_id"],
|
||||
)
|
||||
inserted = sum(1 for m in created_mappings if m.pk)
|
||||
if inserted != len(mappings_to_create):
|
||||
logger.error(
|
||||
f"scan {scan_instance.id}: expected "
|
||||
f"{len(mappings_to_create)} ResourceFindingMapping rows, "
|
||||
f"inserted {inserted}. Rolling back micro-batch."
|
||||
if all_resource_uids_to_touch:
|
||||
now_utc = datetime.now(tz=timezone.utc)
|
||||
resources_to_bulk_update = []
|
||||
for uid in all_resource_uids_to_touch:
|
||||
# Use the instance from dirty_resources if present (has mutated
|
||||
# fields), otherwise the cached one (for updated_at bump only).
|
||||
r = dirty_resources.get(uid) or resource_cache.get(uid)
|
||||
if r is None:
|
||||
continue
|
||||
# Manually bump updated_at since bulk_update bypasses auto_now.
|
||||
r.updated_at = now_utc
|
||||
resources_to_bulk_update.append(r)
|
||||
if resources_to_bulk_update:
|
||||
Resource.objects.bulk_update(
|
||||
resources_to_bulk_update,
|
||||
[
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
"region",
|
||||
"service",
|
||||
"type",
|
||||
"groups",
|
||||
"updated_at",
|
||||
],
|
||||
batch_size=1000,
|
||||
)
|
||||
# Successful execution: leave deadlock retry loop.
|
||||
break
|
||||
except (OperationalError, IntegrityError) as db_err:
|
||||
if attempt < CELERY_DEADLOCK_ATTEMPTS - 1:
|
||||
logger.warning(
|
||||
f"{'Deadlock error' if isinstance(db_err, OperationalError) else 'Integrity error'} "
|
||||
f"on micro-batch for scan {scan_instance.id}. Retrying (attempt {attempt + 1})..."
|
||||
)
|
||||
|
||||
# Update finding denormalized arrays
|
||||
findings_to_update = []
|
||||
for finding_instance, resource_instance in resource_denormalized_data:
|
||||
if not finding_instance.resource_regions:
|
||||
finding_instance.resource_regions = []
|
||||
if not finding_instance.resource_services:
|
||||
finding_instance.resource_services = []
|
||||
if not finding_instance.resource_types:
|
||||
finding_instance.resource_types = []
|
||||
|
||||
if resource_instance.region not in finding_instance.resource_regions:
|
||||
finding_instance.resource_regions.append(resource_instance.region)
|
||||
if resource_instance.service not in finding_instance.resource_services:
|
||||
finding_instance.resource_services.append(resource_instance.service)
|
||||
if resource_instance.type not in finding_instance.resource_types:
|
||||
finding_instance.resource_types.append(resource_instance.type)
|
||||
|
||||
findings_to_update.append(finding_instance)
|
||||
|
||||
if findings_to_update:
|
||||
Finding.objects.bulk_update(
|
||||
findings_to_update,
|
||||
["resource_regions", "resource_services", "resource_types"],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
)
|
||||
|
||||
# Bulk update dirty resources
|
||||
if dirty_resources:
|
||||
update_objects_in_batches(
|
||||
tenant_id=tenant_id,
|
||||
model=Resource,
|
||||
objects=list(dirty_resources.values()),
|
||||
fields=[
|
||||
"metadata",
|
||||
"details",
|
||||
"partition",
|
||||
"region",
|
||||
"service",
|
||||
"type",
|
||||
"groups",
|
||||
],
|
||||
batch_size=1000,
|
||||
)
|
||||
time.sleep(0.1 * (2**attempt))
|
||||
# Clear accumulators that we appended to inside the failed transaction
|
||||
# so the retry produces consistent results.
|
||||
findings_to_create.clear()
|
||||
resource_denormalized_data.clear()
|
||||
tag_mappings_to_create.clear()
|
||||
dirty_resources.clear()
|
||||
resources_with_new_tag_mappings.clear()
|
||||
continue
|
||||
raise
|
||||
|
||||
# Log skipped findings summary
|
||||
if skipped_findings_count > 0:
|
||||
@@ -873,7 +1024,7 @@ def perform_prowler_scan(
|
||||
scan_instance = Scan.objects.get(pk=scan_id)
|
||||
scan_instance.state = StateChoices.EXECUTING
|
||||
scan_instance.started_at = datetime.now(tz=timezone.utc)
|
||||
scan_instance.save()
|
||||
scan_instance.save(update_fields=["state", "started_at", "updated_at"])
|
||||
|
||||
# Find the mutelist processor if it exists
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
@@ -918,7 +1069,13 @@ def perform_prowler_scan(
|
||||
provider_instance.connection_last_checked_at = datetime.now(
|
||||
tz=timezone.utc
|
||||
)
|
||||
provider_instance.save()
|
||||
provider_instance.save(
|
||||
update_fields=[
|
||||
"connected",
|
||||
"connection_last_checked_at",
|
||||
"updated_at",
|
||||
]
|
||||
)
|
||||
|
||||
# If the provider is not connected, raise an exception outside the transaction.
|
||||
# If raised within the transaction, the transaction will be rolled back and the provider will not be marked
|
||||
@@ -933,6 +1090,13 @@ def perform_prowler_scan(
|
||||
last_status_cache = {}
|
||||
resource_failed_findings_cache = defaultdict(int)
|
||||
|
||||
# Throttle scan_instance progress writes to avoid hammering the writer:
|
||||
# only persist when progress moves by at least `PROGRESS_THROTTLE_DELTA`
|
||||
# OR `PROGRESS_THROTTLE_SECONDS` have elapsed. The final progress (1.0)
|
||||
# always persists in the `finally` block below.
|
||||
last_persisted_progress = -1.0
|
||||
last_persisted_progress_at = 0.0
|
||||
|
||||
for progress, findings in prowler_scan.scan():
|
||||
# Process findings in micro-batches
|
||||
findings_list = list(findings)
|
||||
@@ -959,10 +1123,20 @@ def perform_prowler_scan(
|
||||
group_resources_cache=group_resources_cache,
|
||||
)
|
||||
|
||||
# Update scan progress
|
||||
with rls_transaction(tenant_id):
|
||||
scan_instance.progress = progress
|
||||
scan_instance.save()
|
||||
# Throttled progress save (the final save in the `finally` block
|
||||
# below always runs regardless of throttle).
|
||||
now = time.time()
|
||||
progress_delta = progress - last_persisted_progress
|
||||
elapsed = now - last_persisted_progress_at
|
||||
if (
|
||||
progress_delta >= PROGRESS_THROTTLE_DELTA
|
||||
or elapsed >= PROGRESS_THROTTLE_SECONDS
|
||||
):
|
||||
with rls_transaction(tenant_id):
|
||||
scan_instance.progress = progress
|
||||
scan_instance.save(update_fields=["progress", "updated_at"])
|
||||
last_persisted_progress = progress
|
||||
last_persisted_progress_at = now
|
||||
|
||||
scan_instance.state = StateChoices.COMPLETED
|
||||
|
||||
@@ -976,13 +1150,16 @@ def perform_prowler_scan(
|
||||
resources_to_update.append(resource_instance)
|
||||
|
||||
if resources_to_update:
|
||||
update_objects_in_batches(
|
||||
tenant_id=tenant_id,
|
||||
model=Resource,
|
||||
objects=resources_to_update,
|
||||
fields=["failed_findings_count"],
|
||||
batch_size=1000,
|
||||
)
|
||||
# Single rls_transaction wrapping the bulk_update (previously
|
||||
# `update_objects_in_batches` opened one rls_transaction per
|
||||
# chunk; for tenants with many resources this collapsed N
|
||||
# BEGINs/COMMITs into 1).
|
||||
with rls_transaction(tenant_id):
|
||||
Resource.objects.bulk_update(
|
||||
resources_to_update,
|
||||
["failed_findings_count"],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error performing scan {scan_id}: {e}")
|
||||
@@ -994,7 +1171,16 @@ def perform_prowler_scan(
|
||||
scan_instance.duration = time.time() - start_time
|
||||
scan_instance.completed_at = datetime.now(tz=timezone.utc)
|
||||
scan_instance.unique_resource_count = len(unique_resources)
|
||||
scan_instance.save()
|
||||
scan_instance.save(
|
||||
update_fields=[
|
||||
"state",
|
||||
"duration",
|
||||
"completed_at",
|
||||
"unique_resource_count",
|
||||
"progress",
|
||||
"updated_at",
|
||||
]
|
||||
)
|
||||
|
||||
if exception is not None:
|
||||
raise exception
|
||||
@@ -1468,6 +1654,10 @@ def create_compliance_requirements(tenant_id: str, scan_id: str):
|
||||
elif requirement_status == "PASS":
|
||||
requirement_statuses[key]["pass_count"] += 1
|
||||
|
||||
# Idempotent re-run: COPY can't ON CONFLICT, so clear this scan's rows first.
|
||||
with rls_transaction(tenant_id):
|
||||
ComplianceRequirementOverview.objects.filter(scan_id=scan_id).delete()
|
||||
|
||||
# Bulk create requirement records using PostgreSQL COPY
|
||||
_persist_compliance_requirement_rows(tenant_id, compliance_requirement_rows)
|
||||
|
||||
|
||||
@@ -359,35 +359,40 @@ def _load_findings_for_requirement_checks(
|
||||
def _get_compliance_check_ids(compliance_obj) -> set[str]:
|
||||
"""Return the union of all check_ids referenced by a compliance framework.
|
||||
|
||||
Used by the master report orchestrator to know which checks each
|
||||
framework consumes from the shared ``findings_cache``, so that once a
|
||||
framework finishes the entries no other pending framework needs can be
|
||||
evicted from the cache (PROWLER-1733).
|
||||
Used by the master report orchestrator to evict entries from
|
||||
``findings_cache`` once no pending framework needs them (PROWLER-1733).
|
||||
|
||||
Args:
|
||||
compliance_obj: A loaded Compliance framework object exposing a
|
||||
``Requirements`` iterable, each requirement carrying ``Checks``.
|
||||
``None`` is treated as "no checks" rather than raising, so the
|
||||
caller can pass ``frameworks_bulk.get(...)`` directly without
|
||||
an extra existence check.
|
||||
|
||||
Returns:
|
||||
Set of check_id strings (empty if ``compliance_obj`` is ``None``).
|
||||
Accepts the legacy ``Compliance`` shape (``Requirements`` / ``Checks``
|
||||
lists) and the universal ``ComplianceFramework`` shape (``requirements``
|
||||
/ ``checks`` dict keyed by provider). ``None`` returns an empty set so
|
||||
callers can pass ``frameworks_bulk.get(...)`` directly.
|
||||
"""
|
||||
if compliance_obj is None:
|
||||
return set()
|
||||
checks: set[str] = set()
|
||||
requirements = getattr(compliance_obj, "Requirements", None) or []
|
||||
|
||||
requirements = getattr(compliance_obj, "Requirements", None) or getattr(
|
||||
compliance_obj, "requirements", None
|
||||
)
|
||||
if not requirements:
|
||||
return set()
|
||||
|
||||
check_ids: set[str] = set()
|
||||
try:
|
||||
# Defensive: Mock objects (used in unit tests) return another Mock
|
||||
# for any attribute access, which is truthy but not iterable. Treat
|
||||
# any non-iterable Requirements value as "no checks".
|
||||
for req in requirements:
|
||||
req_checks = getattr(req, "Checks", None) or []
|
||||
# Mock objects in unit tests return another Mock for any attribute
|
||||
# access — truthy but not iterable. Treat that as "no checks".
|
||||
for requirement in requirements:
|
||||
requirement_checks = getattr(requirement, "Checks", None)
|
||||
if requirement_checks is None:
|
||||
checks_by_provider = getattr(requirement, "checks", None) or {}
|
||||
requirement_checks = [
|
||||
check_id
|
||||
for check_ids_list in checks_by_provider.values()
|
||||
for check_id in check_ids_list
|
||||
]
|
||||
try:
|
||||
checks.update(req_checks)
|
||||
check_ids.update(requirement_checks)
|
||||
except TypeError:
|
||||
continue
|
||||
except TypeError:
|
||||
return set()
|
||||
return checks
|
||||
return check_ids
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user