Compare commits

..
2581 changed files with 24778 additions and 140933 deletions
+8 -9
View File
@@ -72,8 +72,8 @@ NEO4J_APOC_IMPORT_FILE_ENABLED=false
NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG=true
NEO4J_APOC_TRIGGER_ENABLED=false
NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS=0.0.0.0:7687
# Attack Paths graph settings
ATTACK_PATHS_GRAPH_MUTATION_BATCH_SIZE=1000
# Neo4j Prowler settings
ATTACK_PATHS_BATCH_SIZE=1000
ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES=3
ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS=30
ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES=250
@@ -145,20 +145,19 @@ DJANGO_BROKER_VISIBILITY_TIMEOUT=86400
DJANGO_SENTRY_DSN=
DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute
# Sentry for the web app (server + browser). The UI_SENTRY_* values load only
# when UI_SENTRY_ENABLED="true"; without it they are ignored (default off, zero
# egress). The deprecated NEXT_PUBLIC_SENTRY_DSN still activates Sentry without
# the flag. SENTRY_RELEASE (unprefixed) feeds the web app's server/edge SDKs.
# Sentry for the web app (server + browser). Empty/unset UI_SENTRY_DSN ⇒
# Sentry disabled, zero egress. SENTRY_RELEASE (unprefixed) feeds the web app's
# server/edge SDKs.
UI_SENTRY_DSN=
UI_SENTRY_ENVIRONMENT=local
SENTRY_RELEASE=local
# Reserved runtime public config (registered now; no UI consumer yet)
# UI_POSTHOG_KEY=
# UI_POSTHOG_HOST=
# POSTHOG_KEY=
# POSTHOG_HOST=
# REO_DEV_CLIENT_ID=
#### Prowler release version ####
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.37.0
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.32.0
# Social login credentials
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
+2 -2
View File
@@ -54,7 +54,7 @@ runs:
trivy-db-${{ runner.os }}-
- name: Run Trivy vulnerability scan (JSON)
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@e368e328979b113139d6f9068e03accaed98a518 # 0.34.1
with:
image-ref: ${{ inputs.image-name }}:${{ inputs.image-tag }}
format: 'json'
@@ -67,7 +67,7 @@ runs:
- name: Run Trivy vulnerability scan (SARIF)
if: inputs.upload-sarif == 'true' && github.event_name == 'push'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@e368e328979b113139d6f9068e03accaed98a518 # 0.34.1
with:
image-ref: ${{ inputs.image-name }}:${{ inputs.image-tag }}
format: 'sarif'
-10
View File
@@ -5,20 +5,10 @@
"version": "v8",
"sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd"
},
"github/gh-aw-actions/setup@v0.81.6": {
"repo": "github/gh-aw-actions/setup",
"version": "v0.81.6",
"sha": "ba6380cc6e5be5d21677bebe04d52fb48e3abec7"
},
"github/gh-aw/actions/setup@v0.43.23": {
"repo": "github/gh-aw/actions/setup",
"version": "v0.43.23",
"sha": "9382be3ca9ac18917e111a99d4e6bbff58d0dccc"
},
"step-security/harden-runner@v2.20.0": {
"repo": "step-security/harden-runner",
"version": "v2.20.0",
"sha": "bf7454d06d71f1098171f2acdf0cd4708d7b5920"
}
}
}
+138
View File
@@ -0,0 +1,138 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
# v5
# - package-ecosystem: "pip"
# directory: "/"
# schedule:
# interval: "monthly"
# open-pull-requests-limit: 25
# target-branch: master
# labels:
# - "dependencies"
# - "pip"
# cooldown:
# default-days: 7
# Dependabot Updates are temporary disabled - 2025/03/19
# - package-ecosystem: "pip"
# directory: "/api"
# schedule:
# interval: "daily"
# open-pull-requests-limit: 10
# target-branch: master
# labels:
# - "dependencies"
# - "pip"
# - "component/api"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
open-pull-requests-limit: 25
target-branch: master
labels:
- "dependencies"
- "github_actions"
cooldown:
default-days: 7
# Dependabot Updates are temporary disabled - 2025/03/19
# - package-ecosystem: "npm"
# directory: "/ui"
# schedule:
# interval: "daily"
# open-pull-requests-limit: 10
# target-branch: master
# labels:
# - "dependencies"
# - "npm"
# - "component/ui"
- package-ecosystem: "docker"
directory: "/"
schedule:
interval: "monthly"
open-pull-requests-limit: 25
target-branch: master
labels:
- "dependencies"
- "docker"
cooldown:
default-days: 7
# - package-ecosystem: "pre-commit"
# directory: "/"
# schedule:
# interval: "monthly"
# open-pull-requests-limit: 25
# target-branch: master
# labels:
# - "dependencies"
# - "pre-commit"
# cooldown:
# default-days: 7
# Dependabot Updates are temporary disabled - 2025/04/15
# v4.6
# - package-ecosystem: "pip"
# directory: "/"
# schedule:
# interval: "weekly"
# open-pull-requests-limit: 10
# target-branch: v4.6
# labels:
# - "dependencies"
# - "pip"
# - "v4"
# - package-ecosystem: "github-actions"
# directory: "/"
# schedule:
# interval: "weekly"
# open-pull-requests-limit: 10
# target-branch: v4.6
# labels:
# - "dependencies"
# - "github_actions"
# - "v4"
# - package-ecosystem: "docker"
# directory: "/"
# schedule:
# interval: "weekly"
# open-pull-requests-limit: 10
# target-branch: v4.6
# labels:
# - "dependencies"
# - "docker"
# - "v4"
# Dependabot Updates are temporary disabled - 2025/03/19
# v3
# - package-ecosystem: "pip"
# directory: "/"
# schedule:
# interval: "monthly"
# open-pull-requests-limit: 10
# target-branch: v3
# labels:
# - "dependencies"
# - "pip"
# - "v3"
# - package-ecosystem: "github-actions"
# directory: "/"
# schedule:
# interval: "monthly"
# open-pull-requests-limit: 10
# target-branch: v3
# labels:
# - "dependencies"
# - "github_actions"
# - "v3"
+6 -11
View File
@@ -18,9 +18,8 @@ Please add a detailed description of how to review this PR.
<summary><b>Community Checklist</b></summary>
- [ ] This feature/issue is listed in the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
- [ ] Is it assigned to me, if not, request it via the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](https://goto.prowler.com/slack)
- [ ] I have reviewed the [open pull requests](https://github.com/prowler-cloud/prowler/pulls?q=sort%3Aupdated-desc+is%3Apr+is%3Aopen) and confirmed there is no existing PR that implements the same outcome
- [ ] This feature/issue is listed in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
- [ ] Is it assigned to me, if not, request it via the issue/feature in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](goto.prowler.com/slack)
</details>
@@ -28,8 +27,8 @@ Please add a detailed description of how to review this PR.
- [ ] Review if the code is being covered by tests.
- [ ] Review if code is being documented following this specification https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings
- [ ] Review if backport is needed.
- [ ] Review if is needed to change the [README.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
- [ ] Ensure a changelog fragment is added under [prowler/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/prowler/changelog.d), if applicable.
- [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable.
#### SDK/CLI
- Are there new checks included in this PR? Yes / No
@@ -41,7 +40,7 @@ Please add a detailed description of how to review this PR.
- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px)
- [ ] Screenshots/Video of the functionality flow (if applicable) - Table (640px > X < 1024px)
- [ ] Screenshots/Video of the functionality flow (if applicable) - Desktop (X > 1024px)
- [ ] Ensure a changelog fragment is added under [ui/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/ui/changelog.d), if applicable.
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable.
#### API
- [ ] All issue/task requirements work as expected on the API
@@ -51,11 +50,7 @@ Please add a detailed description of how to review this PR.
- [ ] Any other relevant evidence of the implementation (if applicable)
- [ ] Verify if API specs need to be regenerated.
- [ ] Check if version updates are required (e.g., specs, uv, etc.).
- [ ] Ensure a changelog fragment is added under [api/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/api/changelog.d), if applicable.
#### MCP Server
- [ ] All issue/task requirements work as expected on the MCP Server
- [ ] Ensure a changelog fragment is added under [mcp_server/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/mcp_server/changelog.d), if applicable.
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable.
### License
+2 -13
View File
@@ -23,10 +23,6 @@
"prConcurrentLimit": 20,
"prHourlyLimit": 10,
"vulnerabilityAlerts": {
"labels": [
"dependencies",
"security"
],
"prHourlyLimit": 0,
"prConcurrentLimit": 0
},
@@ -42,7 +38,7 @@
"schedule": [
"* 22-23,0-5 1 * *"
],
"enabled": true
"enabled": false
},
{
"description": "Minors: 8th of every 3 months, Madrid overnight window (22:00-06:00)",
@@ -52,7 +48,7 @@
"schedule": [
"* 22-23,0-5 8 */3 *"
],
"enabled": true
"enabled": false
},
{
"description": "Majors: 15th of every 3 months, Madrid overnight window",
@@ -62,13 +58,6 @@
"schedule": [
"* 22-23,0-5 15 */3 *"
],
"enabled": true
},
{
"description": "gh-aw compiled lock files - generated by 'gh aw compile', action pins must match the compiler version, never bump directly",
"matchFileNames": [
".github/workflows/*.lock.yml"
],
"enabled": false
},
{
-154
View File
@@ -1,154 +0,0 @@
#!/usr/bin/env python3
"""Rename changelog fragments to their PR number before running towncrier.
For every <slug>.<type>.md in <component_dir>/changelog.d/, find the commit that
added it, resolve its PR via the GitHub API (falling back to the squash-commit
subject), and `git mv` it to <PR>.<type>.md so towncrier renders the PR link.
Unresolvable fragments become +<slug>.<type>.md orphans (rendered without link).
"""
import argparse
import json
import os
import re
import subprocess
import sys
import urllib.error
import urllib.request
FRAGMENT_RE = re.compile(
r"^(?P<slug>[A-Za-z0-9][A-Za-z0-9._-]*?)"
r"\.(?P<type>added|changed|deprecated|removed|fixed|security)"
r"(?:\.(?P<counter>[0-9]+))?\.md$"
)
SUBJECT_PR_RE = re.compile(r" \(#([0-9]+)\)$")
IGNORED_FILES = {".gitkeep", "README.md"}
API_TIMEOUT_SECONDS = 10
def git(*args: str) -> str:
result = subprocess.run(["git", *args], check=True, capture_output=True, text=True)
return result.stdout.strip()
def find_adding_commit(path: str) -> str | None:
"""Find the commit that added a file, following renames.
Falls back to a plain (no --follow) lookup: rename detection can lose the
add event for degenerate content (e.g. files identical to many others).
"""
sha = git("log", "--follow", "--diff-filter=A", "--format=%H", "-1", "--", path)
if not sha:
sha = git("log", "--diff-filter=A", "--format=%H", "-1", "--", path)
return sha or None
def pr_from_api(repo: str, sha: str) -> int | None:
"""Resolve the PR associated with a commit via the GitHub API.
Returns None on any network/API failure so the caller can fall back to
parsing the squash-commit subject.
"""
url = f"https://api.github.com/repos/{repo}/commits/{sha}/pulls"
headers = {
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "prowler-changelog-attribution",
}
token = os.environ.get("GITHUB_TOKEN")
if token:
headers["Authorization"] = f"Bearer {token}"
request = urllib.request.Request(url, headers=headers)
try:
with urllib.request.urlopen(request, timeout=API_TIMEOUT_SECONDS) as response:
pulls = json.load(response)
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
return None
if isinstance(pulls, list) and pulls:
return pulls[0].get("number")
return None
def pr_from_subject(sha: str) -> int | None:
subject = git("log", "-1", "--format=%s", sha)
match = SUBJECT_PR_RE.search(subject)
return int(match.group(1)) if match else None
def unique_destination(directory: str, base_name: str, fragment_type: str) -> str:
"""Return a non-colliding fragment path, appending a numeric counter if needed."""
candidate = os.path.join(directory, f"{base_name}.{fragment_type}.md")
counter = 0
while os.path.exists(candidate):
counter += 1
candidate = os.path.join(directory, f"{base_name}.{fragment_type}.{counter}.md")
return candidate
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("component_dir", help="Component directory, e.g. prowler")
parser.add_argument("--repo", default="prowler-cloud/prowler")
parser.add_argument(
"--no-api",
action="store_true",
help="Skip the GitHub API and resolve PRs from commit subjects only",
)
args = parser.parse_args()
fragments_dir = os.path.join(args.component_dir, "changelog.d")
if not os.path.isdir(fragments_dir):
print(f"::error::Fragments directory not found: {fragments_dir}")
return 1
malformed = []
to_process = []
for name in sorted(os.listdir(fragments_dir)):
if name in IGNORED_FILES or name.startswith("+"):
continue
match = FRAGMENT_RE.match(name)
if not match:
malformed.append(name)
continue
if match.group("slug").isdigit():
continue
to_process.append((name, match))
if malformed:
for name in malformed:
print(
f"::error::Malformed fragment filename in {fragments_dir}: {name} "
"(expected <slug>.<type>.md with type one of added|changed|"
"deprecated|removed|fixed|security)"
)
return 1
for name, match in to_process:
slug, fragment_type = match.group("slug"), match.group("type")
path = os.path.join(fragments_dir, name)
sha = find_adding_commit(path)
pr_number = None
if sha:
if not args.no_api:
pr_number = pr_from_api(args.repo, sha)
if pr_number is None:
pr_number = pr_from_subject(sha)
if pr_number is not None:
destination = unique_destination(
fragments_dir, str(pr_number), fragment_type
)
else:
destination = unique_destination(fragments_dir, f"+{slug}", fragment_type)
print(
f"::warning::Could not resolve a PR for {path}; renamed to "
f"{os.path.basename(destination)} (entry will render without a PR link)"
)
git("mv", path, destination)
print(f"{path} -> {destination}")
return 0
if __name__ == "__main__":
sys.exit(main())
+2 -5
View File
@@ -51,8 +51,7 @@ STDERR="$(mktemp)"
trap 'rm -f "${STDERR}"' EXIT
set +e
# ${a[@]+...} guard: an empty array trips `set -u` on bash before 4.4.
OUTPUT="$(osv-scanner scan source ${SCAN_ARGS[@]+"${SCAN_ARGS[@]}"} --format=json "$@" 2>"${STDERR}")"
OUTPUT="$(osv-scanner scan source "${SCAN_ARGS[@]}" --format=json "$@" 2>"${STDERR}")"
RC=$?
set -e
@@ -101,8 +100,6 @@ FINDINGS="$(printf '%s' "${OUTPUT}" | jq --argjson sevs "${SEVERITY_JSON}" '
]
')"
# jq exits 0 with no output on empty stdin, but non-zero on malformed JSON.
# Let the failure abort under set -e rather than reporting zero findings.
COUNT="$(printf '%s' "${FINDINGS}" | jq 'length')"
# Write the findings JSON to OSV_REPORT_FILE so callers (e.g. the composite
@@ -111,7 +108,7 @@ if [ -n "${OSV_REPORT_FILE:-}" ]; then
printf '%s' "${FINDINGS}" > "${OSV_REPORT_FILE}"
fi
if [ "${COUNT:-0}" -gt 0 ]; then
if [ "${COUNT}" -gt 0 ]; then
echo "osv-scanner: ${COUNT} finding(s) at severity ${SEVERITY_LEVELS}"
printf '%s' "${FINDINGS}" | jq -r '
.[] | " [\(.severity)\(if .score then " \(.score)" else "" end)] \(.id) \(.ecosystem)/\(.package)@\(.version) — \(.summary // "(no summary)")"
-11
View File
@@ -249,7 +249,6 @@ modules:
- ui/tests/profile/**
- ui/tests/lighthouse/**
- ui/tests/home/**
- ui/tests/navigation/**
- ui/tests/attack-paths/**
- name: api-serializers
@@ -276,7 +275,6 @@ modules:
- ui/tests/profile/**
- ui/tests/lighthouse/**
- ui/tests/home/**
- ui/tests/navigation/**
- ui/tests/attack-paths/**
- name: api-filters
@@ -434,14 +432,6 @@ modules:
e2e:
- ui/tests/lighthouse/**
- name: ui-navigation
match:
- ui/components/layout/**
- ui/tests/navigation/**
tests: []
e2e:
- ui/tests/navigation/**
- name: ui-overview
match:
- ui/components/overview/**
@@ -474,7 +464,6 @@ modules:
- ui/tests/profile/**
- ui/tests/lighthouse/**
- ui/tests/home/**
- ui/tests/navigation/**
- ui/tests/attack-paths/**
- name: ui-attack-paths
-13
View File
@@ -1,13 +0,0 @@
{% set category_order = definitions.keys() %}
{% for section, _ in sections.items() %}
{% for category in category_order if category in sections[section] %}
### {{ definitions[category]['name'] }}
{% for text, values in sections[section][category].items() -%}
- {{ text }}{% if values %} {{ values|join(', ') }}{% endif %}{{ "\n" }}
{%- endfor %}
{% endfor %}
{% endfor %}
---
{{ "\n" }}
+1 -2
View File
@@ -35,7 +35,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -62,7 +62,6 @@ jobs:
api/docs/**
api/README.md
api/CHANGELOG.md
api/changelog.d/**
api/AGENTS.md
- name: Setup Python with uv
+1 -3
View File
@@ -9,7 +9,6 @@ on:
- 'api/**'
- '.github/workflows/api-codeql.yml'
- '.github/codeql/api-codeql-config.yml'
- '!api/CHANGELOG.md'
pull_request:
branches:
- 'master'
@@ -18,7 +17,6 @@ on:
- 'api/**'
- '.github/workflows/api-codeql.yml'
- '.github/codeql/api-codeql-config.yml'
- '!api/CHANGELOG.md'
schedule:
- cron: '00 12 * * *'
@@ -46,7 +44,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -46,7 +46,7 @@ jobs:
contents: read
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
@@ -65,7 +65,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -108,7 +108,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -175,7 +175,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -215,7 +215,7 @@ jobs:
- name: Install regctl
if: always()
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
- name: Cleanup intermediate architecture tags
if: always()
@@ -236,7 +236,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+3 -17
View File
@@ -33,7 +33,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -69,7 +69,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -92,8 +92,6 @@ jobs:
_http._tcp.deb.debian.org:443
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
get.trivy.dev:443
raw.githubusercontent.com:443
releases.astral.sh:443
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -105,25 +103,13 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
api/**
.github/actions/trivy-scan/**
files: api/**
files_ignore: |
api/docs/**
api/README.md
api/CHANGELOG.md
api/changelog.d/**
api/AGENTS.md
# api-container-build-push.yml resolves the SDK pin to the branch tip
# before building, so match it here and scan what ships. Push only: PRs
# stay deterministic against the committed lock.
- name: Refresh prowler SDK pin to current branch tip
if: steps.check-changes.outputs.any_changed == 'true' && github.event_name == 'push'
run: |
pip install --no-cache-dir "uv==0.11.14"
(cd api && uv lock --upgrade-package prowler)
- name: Set up Docker Buildx
if: steps.check-changes.outputs.any_changed == 'true'
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+1 -2
View File
@@ -43,7 +43,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -77,7 +77,6 @@ jobs:
api/docs/**
api/README.md
api/CHANGELOG.md
api/changelog.d/**
api/AGENTS.md
- name: Setup Python with uv
+3 -5
View File
@@ -48,7 +48,7 @@ jobs:
services:
postgres:
image: postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e
image: postgres:17@sha256:2cd82735a36356842d5eb1ef80db3ae8f1154172f0f653db48fde079b2a0b7f7
env:
POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
@@ -63,7 +63,7 @@ jobs:
--health-timeout 5s
--health-retries 5
valkey:
image: valkey/valkey:7-alpine3.19@sha256:4054fe7fc607b9326ac7c4691ed26e9670d2ff17a9fb28c2577adecf928acbcc
image: valkey/valkey:7-alpine3.19
env:
VALKEY_HOST: ${{ env.VALKEY_HOST }}
VALKEY_PORT: ${{ env.VALKEY_PORT }}
@@ -78,7 +78,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -107,12 +107,10 @@ jobs:
files: |
api/**
.github/workflows/api-tests.yml
codecov.yml
files_ignore: |
api/docs/**
api/README.md
api/CHANGELOG.md
api/changelog.d/**
api/AGENTS.md
- name: Setup Python with uv
+1 -1
View File
@@ -30,7 +30,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
+4 -4
View File
@@ -29,7 +29,7 @@ jobs:
patch_version: ${{ steps.detect.outputs.patch_version }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -75,7 +75,7 @@ jobs:
pull-requests: write
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -202,7 +202,7 @@ jobs:
pull-requests: write
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -307,7 +307,7 @@ jobs:
pull-requests: write
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -1,35 +0,0 @@
name: 'Tools: Check Test Init Files'
on:
pull_request:
branches:
- 'master'
- 'v5.*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions: {}
jobs:
check-test-init-files:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Check for __init__.py files in test directories
run: python3 scripts/check_test_init_files.py .
+1 -1
View File
@@ -36,7 +36,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
+1 -1
View File
@@ -22,7 +22,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
-527
View File
@@ -1,527 +0,0 @@
name: 'Tools: Compile Changelogs'
run-name: 'Compile changelogs for Prowler ${{ inputs.prowler_version }}'
on:
workflow_dispatch:
inputs:
prowler_version:
description: 'Prowler version being released (e.g., 5.31.0)'
required: true
type: string
target_branch:
description: 'Branch to compile on (master for minor releases, v5.X for patches)'
required: true
type: string
sdk_version:
description: 'SDK version override (empty = mirrors prowler_version; "skip" = hold this component back)'
required: false
type: string
api_version:
description: 'API version override (empty = auto-derive 1.<prowler_minor + 1>.<prowler_patch>; "skip" = hold back)'
required: false
type: string
ui_version:
description: 'UI version override (empty = auto-derive 1.<prowler_minor>.<prowler_patch>; "skip" = hold back)'
required: false
type: string
mcp_version:
description: 'MCP Server version override (empty = auto-derive from pending fragment types; "skip" = hold back)'
required: false
type: string
concurrency:
group: ${{ github.workflow }}-${{ inputs.prowler_version }}
cancel-in-progress: false
env:
PROWLER_VERSION: ${{ inputs.prowler_version }}
TARGET_BRANCH: ${{ inputs.target_branch }}
SDK_VERSION: ${{ inputs.sdk_version }}
API_VERSION: ${{ inputs.api_version }}
UI_VERSION: ${{ inputs.ui_version }}
MCP_VERSION: ${{ inputs.mcp_version }}
permissions: {}
jobs:
compile-changelogs:
if: github.event_name == 'workflow_dispatch' && github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: write
pull-requests: write
steps:
- name: Harden the runner (Block outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
with:
egress-policy: block
allowed-endpoints: >
api.github.com:443
github.com:443
objects.githubusercontent.com:443
pypi.org:443
files.pythonhosted.org:443
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ inputs.target_branch }}
fetch-depth: 0 # PR attribution resolves each fragment's adding commit from history
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: '3.12'
- name: Install towncrier
run: pip install --no-cache-dir towncrier==25.8.0
- name: Configure Git
run: |
git config --global user.name 'prowler-bot'
git config --global user.email '179230569+prowler-bot@users.noreply.github.com'
- name: Validate version inputs
run: |
if [[ ! "$PROWLER_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "::error::Invalid prowler_version syntax: '$PROWLER_VERSION' (must be N.N.N)"
exit 1
fi
if [ "$TARGET_BRANCH" != "master" ] && [[ ! "$TARGET_BRANCH" =~ ^v[0-9]+\.[0-9]+$ ]]; then
echo "::error::Invalid target_branch syntax: '$TARGET_BRANCH' (must be 'master' or vN.N, e.g. v5.31)"
exit 1
fi
IFS=. read -r prowler_major prowler_minor prowler_patch <<< "$PROWLER_VERSION"
prowler_major=$((10#$prowler_major))
prowler_minor=$((10#$prowler_minor))
prowler_patch=$((10#$prowler_patch))
if [ "$prowler_patch" -eq 0 ]; then
if [ "$TARGET_BRANCH" != "master" ]; then
echo "::error::target_branch must be 'master' for Prowler ${PROWLER_VERSION}; got '${TARGET_BRANCH}'"
exit 1
fi
else
expected_target_branch="v${prowler_major}.${prowler_minor}"
if [ "$TARGET_BRANCH" != "$expected_target_branch" ]; then
echo "::error::target_branch must be '${expected_target_branch}' for Prowler ${PROWLER_VERSION}; got '${TARGET_BRANCH}'"
exit 1
fi
fi
for pair in "sdk_version:$SDK_VERSION" "api_version:$API_VERSION" "ui_version:$UI_VERSION" "mcp_version:$MCP_VERSION"; do
input_name="${pair%%:*}"
input_value="${pair#*:}"
if [ -n "$input_value" ] && [ "$input_value" != "skip" ] && [[ ! "$input_value" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "::error::Invalid $input_name syntax: '$input_value' (must be N.N.N, empty for auto-derivation, or 'skip')"
exit 1
fi
done
- name: Compile changelogs
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
component_input() {
case "$1" in
prowler) echo "$SDK_VERSION" ;;
api) echo "$API_VERSION" ;;
ui) echo "$UI_VERSION" ;;
mcp_server) echo "$MCP_VERSION" ;;
esac
}
version_key() {
local version="$1"
local major minor patch
IFS=. read -r major minor patch <<< "$version"
printf '%06d.%06d.%06d' "$((10#$major))" "$((10#$minor))" "$((10#$patch))"
}
pending_fragments() {
find "$1/changelog.d" -maxdepth 1 -type f ! -name '.gitkeep' ! -name 'README.md' | sort
}
# The component's last released version is the first stamped heading
# of its CHANGELOG.md, the same source prepare-release.yml greps.
latest_released_version() {
grep -m1 -E '^## \[v?[0-9]+\.[0-9]+\.[0-9]+\]' "$1/CHANGELOG.md" | sed -E 's/^## \[v?([0-9]+\.[0-9]+\.[0-9]+)\].*/\1/'
}
has_removed_fragments() {
echo "$1" | grep -qE '\.removed(\.[0-9]+)?\.md$'
}
# Resolve every component's effective version before compiling
# anything, so a wrong input cannot leave the tree half-compiled.
# Empty input = auto-derive (latest released version + semver bump
# from the pending fragment types). 'skip' = hold the component back.
errors=0
compiling=""
for component in prowler api ui mcp_server; do
input=$(component_input "$component")
fragments=$(pending_fragments "$component")
if [ "$input" = "skip" ]; then
if [ -n "$fragments" ]; then
echo "::warning::${component}: held back by request; these pending fragments stay for a future release:"
echo "$fragments"
fi
continue
fi
if [ -n "$input" ] && [ -z "$fragments" ]; then
echo "::error::${component}: version input '$input' provided but ${component}/changelog.d/ has no pending fragments (wrong input?)"
errors=1
continue
fi
if [ -z "$fragments" ]; then
continue
fi
removed_fragments=false
if has_removed_fragments "$fragments"; then
removed_fragments=true
fi
current=$(latest_released_version "$component")
if [ -z "$current" ]; then
echo "::error::${component}: could not read the latest released version from ${component}/CHANGELOG.md; restore the released heading before compiling"
errors=1
continue
fi
if [ -n "$input" ]; then
effective="$input"
mode="explicit"
current_key=$(version_key "$current")
effective_key=$(version_key "$effective")
if [[ "$effective_key" < "$current_key" || "$effective_key" == "$current_key" ]]; then
echo "::error::${component}: explicit version '${effective}' must be greater than the latest released version (${current})"
errors=1
continue
fi
else
if [ "$removed_fragments" = "true" ]; then
echo "::error::${component}: pending 'removed' fragments imply a major bump (breaking change); provide its version input explicitly"
errors=1
continue
fi
# SDK, UI, and API versions are deterministic mirrors of the
# Prowler version (the scheme bump-version.yml codifies): the SDK
# mirrors it directly, the UI tracks 1.<minor>.<patch>, and the
# API is the independent 1.<minor + 1>.<patch> stream. Only the
# MCP Server has its own cadence, derived from fragment types.
IFS=. read -r _ prowler_minor prowler_patch <<< "$PROWLER_VERSION"
prowler_minor=$((10#$prowler_minor))
prowler_patch=$((10#$prowler_patch))
case "$component" in
prowler) effective="$PROWLER_VERSION" ;;
ui) effective="1.${prowler_minor}.${prowler_patch}" ;;
api) effective="1.$((prowler_minor + 1)).${prowler_patch}" ;;
mcp_server)
IFS=. read -r major minor patch <<< "$current"
major=$((10#$major))
minor=$((10#$minor))
patch=$((10#$patch))
# Prowler patch releases (vN.N target) are maintenance
# releases, so the MCP Server bumps patch regardless of
# fragment types; a deliberate exception needs the explicit
# version input.
if [ "$TARGET_BRANCH" != "master" ]; then
effective="${major}.${minor}.$((patch + 1))"
if echo "$fragments" | grep -qE '\.(added|deprecated)(\.[0-9]+)?\.md$'; then
echo "::warning::${component}: 'added'/'deprecated' fragments are shipping in a Prowler patch; auto-derived a patch bump (${current} -> ${effective}), pass the version input to override"
fi
elif echo "$fragments" | grep -qE '\.(added|changed|deprecated)(\.[0-9]+)?\.md$'; then
effective="${major}.$((minor + 1)).0"
else
effective="${major}.${minor}.$((patch + 1))"
fi
;;
esac
current_key=$(version_key "$current")
effective_key=$(version_key "$effective")
if [[ "$effective_key" < "$current_key" || "$effective_key" == "$current_key" ]]; then
echo "::error::${component}: auto-derived version '${effective}' is not greater than the latest released version (${current}); check prowler_version or pass the version input explicitly"
errors=1
continue
fi
mode="auto"
echo "::notice::${component}: version auto-derived ${current} -> ${effective}"
fi
if [ "$removed_fragments" = "true" ]; then
IFS=. read -r current_major _ <<< "$current"
current_major=$((10#$current_major))
IFS=. read -r effective_major effective_minor effective_patch <<< "$effective"
effective_major=$((10#$effective_major))
effective_minor=$((10#$effective_minor))
effective_patch=$((10#$effective_patch))
if [ "$effective_major" -le "$current_major" ] || [ "$effective_minor" -ne 0 ] || [ "$effective_patch" -ne 0 ]; then
echo "::error::${component}: removed fragments require a major component release (${current} -> X.0.0 with X > ${current_major}); got ${effective}"
errors=1
continue
fi
fi
# Without the marker the build would insert the new block above the
# file header instead of below it.
if ! grep -q '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md"; then
echo "::error::${component}/CHANGELOG.md is missing the '<!-- changelog: release notes start -->' marker; restore it after the intro line before compiling"
errors=1
continue
fi
# A hand-written UNRELEASED block means someone followed the old
# convention; its entries would be left out of the compiled block
# and out of the release notes extraction.
if grep -q '(Prowler UNRELEASED)' "$component/CHANGELOG.md"; then
echo "::error::${component}/CHANGELOG.md contains a hand-written '(Prowler UNRELEASED)' block; convert its entries to fragments in ${component}/changelog.d/ and delete the block before compiling"
errors=1
continue
fi
echo "${effective} ${mode}" > "${RUNNER_TEMP}/version-${component}.txt"
compiling="${compiling}${component} "
done
if [ "$errors" -ne 0 ]; then
exit 1
fi
if [ -z "$compiling" ]; then
echo "::error::Nothing to compile: no component has pending fragments to release"
exit 1
fi
body_file="${RUNNER_TEMP}/compile-changelogs-pr-body.md"
{
echo "### Description"
echo ""
echo "Compiles the pending changelog fragments into the per-component \`CHANGELOG.md\` files for Prowler v${PROWLER_VERSION}, replacing the manual stamping PR."
echo ""
echo "| Component | Version | Fragments consumed |"
echo "|---|---|---|"
} > "$body_file"
compiled_components=""
for component in prowler api ui mcp_server; do
if [ ! -f "${RUNNER_TEMP}/version-${component}.txt" ]; then
echo "Skipping ${component} (no pending fragments or held back)"
echo "| \`${component}\` | - | 0 |" >> "$body_file"
continue
fi
read -r version mode < "${RUNNER_TEMP}/version-${component}.txt"
version_label="$version"
if [ "$mode" = "auto" ]; then
version_label="${version} (auto)"
fi
count=$(pending_fragments "$component" | wc -l | tr -d ' ')
echo "Compiling ${component} ${version} (${count} fragments, ${mode} version)..."
# Captured before attribution renames them: these original paths are
# what the forward-sync deletes on master (backports copy fragments
# verbatim, so filenames match across branches).
pending_fragments "$component" > "${RUNNER_TEMP}/consumed-${component}.txt"
pre_lines=$(wc -l < "$component/CHANGELOG.md")
# Attribution must run before the build: towncrier renders the
# first dotted segment of each filename as the PR number.
python .github/scripts/changelog_attribution.py "$component"
towncrier build --config "$component/towncrier.toml" --version "$version" --name "Prowler v${PROWLER_VERSION}" --yes
# The build only inserts lines right after the marker, so the new
# stamped block is exactly the added lines following it. Captured
# for the forward-sync to master.
post_lines=$(wc -l < "$component/CHANGELOG.md")
delta=$((post_lines - pre_lines))
marker_line=$(grep -n -m1 '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md" | cut -d: -f1)
sed -n "$((marker_line + 1)),$((marker_line + delta))p" "$component/CHANGELOG.md" > "${RUNNER_TEMP}/block-${component}.md"
compiled_components="${compiled_components}${component} "
echo "| \`${component}\` | ${version_label} | ${count} |" >> "$body_file"
done
echo "COMPILED_COMPONENTS=${compiled_components}" >> "$GITHUB_ENV"
{
echo ""
echo "Review that no pending fragment was dropped (the diff must delete every consumed fragment) and that each new version block is correct, then squash-merge."
echo ""
echo "### License"
echo ""
echo "By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license."
} >> "$body_file"
echo "PR_BODY_FILE=${body_file}" >> "$GITHUB_ENV"
- name: Create compile PR
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
commit-message: 'chore(changelog): v${{ env.PROWLER_VERSION }}'
branch: compile-changelogs-${{ env.PROWLER_VERSION }}
base: ${{ env.TARGET_BRANCH }}
title: 'chore(changelog): v${{ env.PROWLER_VERSION }}'
body-path: ${{ env.PR_BODY_FILE }}
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
labels: |
no-changelog
skip-sync
# Patch compiles (target_branch = v5.X) leave master holding the consumed
# fragments and missing the new version block. This applies the equivalent
# change to master: insert the same stamped blocks under the marker and
# delete the consumed fragments, so the next minor compile cannot
# re-release entries that already shipped in the patch.
- name: Apply forward-sync to master
if: env.TARGET_BRANCH != 'master'
run: |
set -euo pipefail
git checkout -B master origin/master
version_key() {
local version="$1"
local major minor patch
IFS=. read -r major minor patch <<< "$version"
printf '%06d.%06d.%06d' "$((10#$major))" "$((10#$minor))" "$((10#$patch))"
}
release_from_heading() {
local heading="$1"
echo "$heading" | sed -E 's/^## \[[^]]+\] \(Prowler v?([0-9]+\.[0-9]+\.[0-9]+)\).*/\1/'
}
insert_changelog_block_ordered() {
local component="$1"
local block_file="$2"
local changelog="${component}/CHANGELOG.md"
local incoming_heading incoming_release incoming_key
local marker_line insertion_line duplicate_line total_lines
local line heading existing_release existing_key
marker_line=$(grep -n -m1 '^<!-- changelog: release notes start -->$' "$changelog" | cut -d: -f1)
incoming_heading=$(grep -m1 -E '^## \[[^]]+\] \(Prowler v?[0-9]+\.[0-9]+\.[0-9]+\)' "$block_file" || true)
if [ -z "$incoming_heading" ]; then
echo "::error::${block_file} does not contain a stamped Prowler release heading"
exit 1
fi
incoming_release=$(release_from_heading "$incoming_heading")
incoming_key=$(version_key "$incoming_release")
insertion_line=""
duplicate_line=""
while IFS=: read -r line heading; do
existing_release=$(release_from_heading "$heading")
existing_key=$(version_key "$existing_release")
if [[ "$incoming_key" == "$existing_key" ]]; then
duplicate_line="$line"
break
fi
if [[ "$incoming_key" > "$existing_key" ]]; then
insertion_line="$line"
break
fi
done < <(grep -n -E '^## \[[^]]+\] \(Prowler v?[0-9]+\.[0-9]+\.[0-9]+\)' "$changelog" || true)
if [ -n "$duplicate_line" ]; then
echo "::error::${changelog} already contains a block for Prowler v${incoming_release} at line ${duplicate_line}; refusing to insert a duplicate"
exit 1
fi
if [ -z "$insertion_line" ]; then
insertion_line=$(($(wc -l < "$changelog") + 1))
fi
if [ "$insertion_line" -le "$marker_line" ]; then
insertion_line=$((marker_line + 1))
fi
# The captured block window can be off by one blank line on either
# end (towncrier re-emits the blank after the marker), so strip the
# outer blank lines and pad exactly one on each side: the block
# must never glue to the marker above or the next heading below.
awk '
/[^[:space:]]/ { for (i = 0; i < pending; i++) print ""; pending = 0; print; started = 1; next }
started { pending++ }
' "$block_file" > "${RUNNER_TEMP}/block-normalized.md"
total_lines=$(wc -l < "$changelog")
{
head -n "$((insertion_line - 1))" "$changelog"
if [ "$insertion_line" -gt 1 ] && [ -n "$(sed -n "$((insertion_line - 1))p" "$changelog")" ]; then
echo ""
fi
cat "${RUNNER_TEMP}/block-normalized.md"
if [ "$insertion_line" -le "$total_lines" ]; then
echo ""
fi
tail -n +"$insertion_line" "$changelog"
} > "${RUNNER_TEMP}/changelog.tmp"
mv "${RUNNER_TEMP}/changelog.tmp" "$changelog"
echo "::notice::Inserted ${component} changelog block for Prowler v${incoming_release} at line ${insertion_line}"
}
sync_body="${RUNNER_TEMP}/forward-sync-pr-body.md"
{
echo "### Description"
echo ""
echo "Forward-syncs the v${PROWLER_VERSION} compiled changelogs from \`${TARGET_BRANCH}\` to \`master\`: inserts the same stamped version blocks under the insertion marker and deletes the consumed fragments, so the next minor compile cannot re-release entries that already shipped in this patch. Opened automatically by the same run that opened the compile PR; review and squash-merge after it."
echo ""
echo "| Component | Fragments deleted on master | Skipped (only on ${TARGET_BRANCH}) |"
echo "|---|---|---|"
} > "$sync_body"
for component in $COMPILED_COMPONENTS; do
block_file="${RUNNER_TEMP}/block-${component}.md"
consumed_file="${RUNNER_TEMP}/consumed-${component}.txt"
if ! grep -qm1 '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md"; then
echo "::error::${component}/CHANGELOG.md on master is missing the insertion marker; cannot forward-sync"
exit 1
fi
deleted=0
skipped=0
while IFS= read -r fragment; do
if [ -z "$fragment" ]; then
continue
fi
if [ -f "$fragment" ]; then
git rm -q "$fragment"
deleted=$((deleted + 1))
else
echo "::notice::${fragment} does not exist on master (change landed only on ${TARGET_BRANCH}); skipping its deletion"
skipped=$((skipped + 1))
fi
done < "$consumed_file"
insert_changelog_block_ordered "$component" "$block_file"
echo "| \`${component}\` | ${deleted} | ${skipped} |" >> "$sync_body"
done
{
echo ""
echo "### License"
echo ""
echo "By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license."
} >> "$sync_body"
echo "SYNC_BODY_FILE=${sync_body}" >> "$GITHUB_ENV"
- name: Create forward-sync PR
if: env.TARGET_BRANCH != 'master'
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
with:
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
commit-message: 'chore(changelog): v${{ env.PROWLER_VERSION }} forward-sync to master'
branch: forward-sync-changelogs-${{ env.PROWLER_VERSION }}
base: master
title: 'chore(changelog): v${{ env.PROWLER_VERSION }} forward-sync to master'
body-path: ${{ env.SYNC_BODY_FILE }}
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
labels: |
no-changelog
skip-sync
+1 -1
View File
@@ -26,7 +26,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+1 -1
View File
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -1,50 +0,0 @@
name: 'Docs: Check Provider Cards Snippet'
on:
pull_request:
branches:
- 'master'
- 'v5.*'
paths:
- 'docs/user-guide/providers/**/getting-started-*.mdx'
- 'docs/scripts/generate_provider_cards.py'
- 'docs/snippets/provider-cards.mdx'
- 'api/src/backend/api/models.py'
- '.github/workflows/docs-check-provider-cards.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions: {}
jobs:
check-provider-cards:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- name: Harden Runner
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
github.com:443
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Verify provider cards snippet is up to date
run: |
if ! python3 docs/scripts/generate_provider_cards.py; then
echo "::error::docs/snippets/provider-cards.mdx is out of sync with the provider getting-started pages or the API ProviderChoices enum."
echo "Run 'python3 docs/scripts/generate_provider_cards.py' locally and commit the regenerated snippet."
echo "--- diff ---"
git diff docs/snippets/provider-cards.mdx
exit 1
fi
+1 -1
View File
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
# We can't block as Trufflehog needs to verify secrets against vendors
egress-policy: audit
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+5 -8
View File
@@ -26,7 +26,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -38,14 +38,11 @@ jobs:
- name: Set up Helm
uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
- name: Set chart version and appVersion from release tag
- name: Set appVersion from release tag
run: |
# Strip any leading "v" so the chart version is valid SemVer 2.
RELEASE_TAG="${GITHUB_EVENT_RELEASE_TAG_NAME#v}"
echo "Setting chart version and appVersion to ${RELEASE_TAG}"
# Publish an immutable chart version per release instead of the static
# 0.0.1 in source, so every release is a distinct, addressable artifact.
yq -i ".version = \"${RELEASE_TAG}\" | .appVersion = \"${RELEASE_TAG}\"" ${{ env.CHART_PATH }}/Chart.yaml
RELEASE_TAG="${GITHUB_EVENT_RELEASE_TAG_NAME}"
echo "Setting appVersion to ${RELEASE_TAG}"
sed -i "s/^appVersion:.*/appVersion: \"${RELEASE_TAG}\"/" ${{ env.CHART_PATH }}/Chart.yaml
env:
GITHUB_EVENT_RELEASE_TAG_NAME: ${{ github.event.release.tag_name }}
+1 -1
View File
@@ -22,7 +22,7 @@ jobs:
issues: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
+700 -1298
View File
File diff suppressed because one or more lines are too long
+3 -9
View File
@@ -12,8 +12,8 @@ if: contains(toJson(github.event.issue.labels), 'status/needs-triage')
timeout-minutes: 12
user-rate-limit:
max-runs-per-window: 5
rate-limit:
max: 5
window: 60
concurrency:
@@ -30,12 +30,6 @@ permissions:
engine: copilot
strict: false
pre-steps:
- name: Harden the runner
uses: step-security/harden-runner@v2.20.0
with:
egress-policy: audit
imports:
- ../agents/issue-triage.md
@@ -114,7 +108,7 @@ Triage the following GitHub issue using the Prowler Issue Triage Agent persona.
## Sanitized Issue Content
${{ steps.sanitized.outputs.text }}
${{ needs.activation.outputs.text }}
## Instructions
+3 -3
View File
@@ -27,7 +27,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -46,7 +46,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -93,7 +93,7 @@ jobs:
fi
- name: Add community label
if: steps.check_membership.outputs.is_member == 'false' && github.event.pull_request.user.type != 'Bot'
if: steps.check_membership.outputs.is_member == 'false'
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
GH_TOKEN: ${{ github.token }}
+1 -1
View File
@@ -26,7 +26,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
with:
egress-policy: block
allowed-endpoints: >
@@ -45,7 +45,7 @@ jobs:
contents: read
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
@@ -64,7 +64,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -106,7 +106,7 @@ jobs:
packages: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -165,7 +165,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -206,7 +206,7 @@ jobs:
- name: Install regctl
if: always()
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
- name: Cleanup intermediate architecture tags
if: always()
@@ -227,7 +227,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+3 -6
View File
@@ -33,7 +33,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -68,7 +68,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -98,13 +98,10 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
mcp_server/**
.github/actions/trivy-scan/**
files: mcp_server/**
files_ignore: |
mcp_server/README.md
mcp_server/CHANGELOG.md
mcp_server/changelog.d/**
- name: Set up Docker Buildx
if: steps.check-changes.outputs.any_changed == 'true'
+2 -2
View File
@@ -29,7 +29,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -67,7 +67,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+1 -1
View File
@@ -32,7 +32,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
with:
egress-policy: block
allowed-endpoints: >
@@ -48,7 +48,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -83,7 +83,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+29 -186
View File
@@ -19,60 +19,6 @@ concurrency:
permissions: {}
jobs:
test-changelog-attribution:
if: github.repository == 'prowler-cloud/prowler'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
with:
egress-policy: block
allowed-endpoints: >
api.github.com:443
github.com:443
objects.githubusercontent.com:443
pypi.org:443
files.pythonhosted.org:443
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 1
persist-credentials: false
- name: Fetch PR base ref for tj-actions/changed-files
env:
BASE_REF: ${{ github.event.pull_request.base.ref }}
run: git fetch --depth=1 origin "${BASE_REF}"
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
.github/scripts/changelog_attribution.py
.github/workflows/pr-check-changelog.yml
.github/workflows/compile-changelogs.yml
.github/towncrier/template.md.jinja
*/towncrier.toml
tests/github/**
- name: Set up Python
if: steps.changed-files.outputs.any_changed == 'true'
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: '3.12'
- name: Test changelog attribution
if: steps.changed-files.outputs.any_changed == 'true'
run: |
python3 -m pip install --user --disable-pip-version-check pytest==9.0.3 towncrier==25.8.0
python3 -m pytest tests/github
check-changelog:
if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false
runs-on: ubuntu-latest
@@ -85,7 +31,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -116,160 +62,53 @@ jobs:
uv.lock
pyproject.toml
- name: Check for folder changes and changelog fragment presence
- name: Check for folder changes and changelog presence
id: check-folders
run: |
fragment_name_re='^[A-Za-z0-9][A-Za-z0-9._-]*\.(added|changed|deprecated|removed|fixed|security)(\.[0-9]+)?\.md$'
manual_pr_link_re='(\[\(#[0-9]+\)\]|\[#[0-9]+\]\(|\(#[0-9]+\)|github\.com/[^[:space:]/]+/[^[:space:]/]+/(pull|issues)/[0-9]+)'
folder_alt=$(echo "$MONITORED_FOLDERS" | tr ' ' '|')
missing_fragments=""
invalid_fragments=""
linked_fragments=""
handwritten_changelogs=""
all_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n')
added=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" | tr ' ' '\n')
added_or_renamed=$(printf '%s\n%s' "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES}" | tr ' ' '\n')
added_modified_or_renamed=$(printf '%s\n%s\n%s' "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_MODIFIED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES}" | tr ' ' '\n')
# Returns success if the folder has a valid fragment added, modified, or renamed.
has_changelog_update() {
local folder="$1"
if echo "$added_modified_or_renamed" | grep "^${folder}/changelog.d/" | sed "s|^${folder}/changelog.d/||" | grep -qE "$fragment_name_re"; then
return 0
fi
return 1
}
missing_changelogs=""
if [[ "${STEPS_CHANGED_FILES_OUTPUTS_ANY_CHANGED}" == "true" ]]; then
# Check monitored folders
for folder in $MONITORED_FOLDERS; do
if echo "$all_changed" | grep -q "^${folder}/CHANGELOG.md$"; then
echo "Direct CHANGELOG.md edits are not allowed for ${folder}/"
handwritten_changelogs="${handwritten_changelogs}- \`${folder}/CHANGELOG.md\`"$'\n'
fi
changed_in_folder=$(echo "$all_changed" | grep "^${folder}/" | grep -v "^${folder}/CHANGELOG.md$" || true)
# Get files changed in this folder
changed_in_folder=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep "^${folder}/" || true)
if [ -n "$changed_in_folder" ]; then
echo "Detected changes in ${folder}/"
if ! has_changelog_update "$folder"; then
echo "No changelog fragment found for ${folder}/"
missing_fragments="${missing_fragments}- \`${folder}\`"$'\n'
# Check if CHANGELOG.md was updated
if ! echo "$changed_in_folder" | grep -q "^${folder}/CHANGELOG.md$"; then
echo "No changelog update found for ${folder}/"
missing_changelogs="${missing_changelogs}- \`${folder}\`"$'\n'
fi
fi
done
# Check root-level dependency files (uv.lock, pyproject.toml)
# These are associated with the prowler folder changelog
root_deps_changed=$(echo "$all_changed" | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
root_deps_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
if [ -n "$root_deps_changed" ]; then
echo "Detected changes in root dependency files: $root_deps_changed"
if ! has_changelog_update "prowler"; then
# Check if prowler/CHANGELOG.md was already updated (might have been caught above)
prowler_changelog_updated=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep "^prowler/CHANGELOG.md$" || true)
if [ -z "$prowler_changelog_updated" ]; then
# Only add if prowler wasn't already flagged
if ! echo "$missing_fragments" | grep -q "prowler"; then
echo "No changelog fragment found for root dependency changes"
missing_fragments="${missing_fragments}- \`prowler\` (root dependency files changed)"$'\n'
if ! echo "$missing_changelogs" | grep -q "prowler"; then
echo "No changelog update found for root dependency changes"
missing_changelogs="${missing_changelogs}- \`prowler\` (root dependency files changed)"$'\n'
fi
fi
fi
# Validate the filename of every fragment added by this PR
added_fragments=$(echo "$added_or_renamed" | grep -E "^(${folder_alt})/changelog\.d/" || true)
for fragment in $added_fragments; do
name=$(basename "$fragment")
if [ "$name" = ".gitkeep" ] || [ "$name" = "README.md" ]; then
continue
fi
if ! echo "$name" | grep -qE "$fragment_name_re"; then
echo "Invalid fragment filename: $fragment"
invalid_fragments="${invalid_fragments}- \`${fragment}\`"$'\n'
fi
done
# Lint fragment content: the PR link is attached automatically at
# compile time, so a hand-written PR or issue link would be wrong
touched_fragments=$(echo "$added_modified_or_renamed" | grep -E "^(${folder_alt})/changelog\.d/" || true)
for fragment in $touched_fragments; do
name=$(basename "$fragment")
if [ "$name" = ".gitkeep" ] || [ "$name" = "README.md" ] || [ ! -f "$fragment" ]; then
continue
fi
if grep -qE "$manual_pr_link_re" "$fragment"; then
echo "Fragment contains a hand-written PR or issue link: $fragment"
linked_fragments="${linked_fragments}- \`${fragment}\`"$'\n'
fi
done
fi
# Suggest a slug derived from the branch name for the bot comment
suggested_slug=$(echo "$HEAD_REF" | tr '[:upper:]' '[:lower:]' | sed 's|.*/||; s/[^a-z0-9._-]/-/g; s/^[^a-z0-9]*//')
if [ -z "$suggested_slug" ]; then
suggested_slug="my-change"
fi
fragment_help="A changelog fragment is a small Markdown file named \`<slug>.<type>.md\` under \`<component>/changelog.d/\`, where \`<type>\` is one of \`added\`, \`changed\`, \`deprecated\`, \`removed\`, \`fixed\` or \`security\`. Its content is the changelog entry text, without the PR link (added automatically at release time) and without a trailing period. For example:
\`\`\`
echo 'Entry text describing the change' > <component>/changelog.d/${suggested_slug}.fixed.md
\`\`\`
If this PR does not need a changelog entry, add the \`no-changelog\` label instead."
if [ -n "$missing_fragments" ] || [ -n "$invalid_fragments" ] || [ -n "$linked_fragments" ] || [ -n "$handwritten_changelogs" ]; then
comment_body=""
if [ -n "$missing_fragments" ]; then
comment_body="⚠️ **Changes detected in the following folders without a changelog fragment:**"$'\n\n'"${missing_fragments}"$'\n'
fi
if [ -n "$invalid_fragments" ]; then
comment_body="${comment_body}⚠️ **Changelog fragment filenames that do not follow the naming convention:**"$'\n\n'"${invalid_fragments}"$'\n'
fi
if [ -n "$linked_fragments" ]; then
comment_body="${comment_body}⚠️ **Changelog fragments containing a hand-written PR or issue link (remove it; the PR link is attached automatically at release time):**"$'\n\n'"${linked_fragments}"$'\n'
fi
if [ -n "$handwritten_changelogs" ]; then
comment_body="${comment_body}⚠️ **Direct \`CHANGELOG.md\` edits are not allowed in regular PRs:**"$'\n\n'"${handwritten_changelogs}"$'\n'
fi
comment_body="${comment_body}${fragment_help}"
else
comment_body="✅ All required changelog fragments are present."
fi
write_multiline_output() {
local name="$1"
local value="$2"
local delimiter
while true; do
delimiter="EOF_$(openssl rand -hex 16)"
if ! grep -qxF "$delimiter" <<< "$value"; then
break
fi
done
{
echo "${name}<<${delimiter}"
if [ -n "$value" ]; then
printf '%s\n' "$value"
fi
echo "${delimiter}"
} >> "$GITHUB_OUTPUT"
}
write_multiline_output "missing_fragments" "$missing_fragments"
write_multiline_output "invalid_fragments" "$invalid_fragments"
write_multiline_output "linked_fragments" "$linked_fragments"
write_multiline_output "handwritten_changelogs" "$handwritten_changelogs"
write_multiline_output "comment_body" "$comment_body"
{
echo "missing_changelogs<<EOF"
echo -e "${missing_changelogs}"
echo "EOF"
} >> $GITHUB_OUTPUT
env:
STEPS_CHANGED_FILES_OUTPUTS_ANY_CHANGED: ${{ steps.changed-files.outputs.any_changed }}
STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
STEPS_CHANGED_FILES_OUTPUTS_MODIFIED_FILES: ${{ steps.changed-files.outputs.modified_files }}
STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES: ${{ steps.changed-files.outputs.renamed_files }}
HEAD_REF: ${{ github.event.pull_request.head.ref }}
- name: Find existing changelog comment
if: github.event.pull_request.head.repo.full_name == github.repository
@@ -289,10 +128,14 @@ jobs:
edit-mode: replace
body: |
<!-- changelog-check -->
${{ steps.check-folders.outputs.comment_body }}
${{ steps.check-folders.outputs.missing_changelogs != '' && format('⚠️ **Changes detected in the following folders without a corresponding update to the `CHANGELOG.md`:**
- name: Fail if changelog fragment is missing or invalid
if: steps.check-folders.outputs.missing_fragments != '' || steps.check-folders.outputs.invalid_fragments != '' || steps.check-folders.outputs.linked_fragments != '' || steps.check-folders.outputs.handwritten_changelogs != ''
{0}
Please add an entry to the corresponding `CHANGELOG.md` file to maintain a clear history of changes.', steps.check-folders.outputs.missing_changelogs) || '✅ All necessary `CHANGELOG.md` files have been updated.' }}
- name: Fail if changelog is missing
if: steps.check-folders.outputs.missing_changelogs != ''
run: |
echo "::error::Missing, invalid, or disallowed changelog updates"
echo "::error::Missing changelog updates in some folders"
exit 1
@@ -35,7 +35,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
+27 -63
View File
@@ -28,7 +28,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -37,7 +37,8 @@ jobs:
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 1
persist-credentials: false # No write token in the untrusted PR-head tree; public repo so base fetch/changed-files work unauthenticated
# zizmor: ignore[artipacked]
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
- name: Fetch PR base ref for tj-actions/changed-files
env:
@@ -49,8 +50,6 @@ jobs:
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: '**'
safe_output: false # Raw paths (list read via env var, injection-safe); default escaping backslash-quotes chars like () and breaks the -f test
separator: "\n" # Newline-delimited so the reader tolerates spaces and glob chars in paths
- name: Check for conflict markers
id: conflict-check
@@ -60,18 +59,19 @@ jobs:
CONFLICT_FILES=""
HAS_CONFLICTS=false
# Read newline-delimited paths so spaces/globs neither word-split nor glob-expand
while IFS= read -r file; do
[ -n "$file" ] || continue
[ -f "$file" ] || continue
echo "Checking file: $file"
# Check each changed file for conflict markers
for file in ${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}; do
if [ -f "$file" ]; then
echo "Checking file: $file"
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
echo "Conflict markers found in: $file"
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
HAS_CONFLICTS=true
# Look for conflict markers (more precise regex)
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
echo "Conflict markers found in: $file"
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
HAS_CONFLICTS=true
fi
fi
done <<< "$STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES"
done
if [ "$HAS_CONFLICTS" = true ]; then
echo "has_conflicts=true" >> $GITHUB_OUTPUT
@@ -88,49 +88,18 @@ jobs:
env:
STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
- name: Check base-branch mergeability
id: merge-check
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
REPO: ${{ github.repository }}
run: |
MERGEABLE=null
# GitHub computes mergeability async, so .mergeable is null until ready; poll until resolved
for attempt in 1 2 3 4 5; do
MERGEABLE=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.mergeable')
if [ "$MERGEABLE" != "null" ]; then
break
fi
echo "Attempt ${attempt}: mergeability not computed yet, retrying..."
sleep 3
done
# Keep 'unknown' distinct from 'clean' so we never assert a clean merge we could not confirm
case "$MERGEABLE" in
false) STATUS=conflict; echo "PR branch cannot be merged cleanly into its base branch" ;;
true) STATUS=clean; echo "PR branch merges cleanly into its base branch" ;;
*) STATUS=unknown; echo "::warning::Mergeability did not resolve after retries; leaving it undetermined" ;;
esac
echo "merge_status=${STATUS}" >> "$GITHUB_OUTPUT"
- name: Manage conflict label
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
run: |
LABEL_NAME="has-conflicts"
if [ "$HAS_CONFLICTS" = "true" ] || [ "$MERGE_STATUS" = "conflict" ]; then
# Add or remove label based on conflict status
if [ "$HAS_CONFLICTS" = "true" ]; then
echo "Adding conflict label to PR #${PR_NUMBER}..."
gh pr edit "$PR_NUMBER" --add-label "$LABEL_NAME" --repo ${{ github.repository }} || true
elif [ "$MERGE_STATUS" = "unknown" ]; then
# Don't drop the label on an undetermined merge state; a later run will settle it
echo "Mergeability undetermined; leaving label unchanged"
else
echo "Removing conflict label from PR #${PR_NUMBER}..."
gh pr edit "$PR_NUMBER" --remove-label "$LABEL_NAME" --repo ${{ github.repository }} || true
@@ -152,25 +121,20 @@ jobs:
edit-mode: replace
body: |
<!-- conflict-checker-comment -->
${{ (steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict') && '⚠️ **Conflicts Detected**' || (steps.merge-check.outputs.merge_status == 'unknown' && '️ **Conflict Check Incomplete**' || '✅ **No Conflicts**') }}
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('
**Conflict markers** are present in the following files:
${{ steps.conflict-check.outputs.has_conflicts == 'true' && '⚠️ **Conflict Markers Detected**' || '✅ **Conflict Markers Resolved**' }}
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('This pull request contains unresolved conflict markers in the following files:
{0}
Resolve them by removing every `<<<<<<<`, `=======`, and `>>>>>>>` marker, then commit and push.', steps.conflict-check.outputs.conflict_files) || '' }}
${{ steps.merge-check.outputs.merge_status == 'conflict' && '
**Merge conflict with the base branch.** This PR cannot be merged cleanly. Update your branch with the latest base (rebase or merge) and resolve the conflicts.' || '' }}
${{ steps.merge-check.outputs.merge_status == 'unknown' && '
GitHub had not finished computing mergeability, so base-branch conflict status could not be verified on this run.' || '' }}
${{ (steps.conflict-check.outputs.has_conflicts != 'true' && steps.merge-check.outputs.merge_status == 'clean') && '
No conflict markers, and the branch merges cleanly into its base.' || '' }}
Please resolve these conflicts by:
1. Locating the conflict markers: `<<<<<<<`, `=======`, and `>>>>>>>`
2. Manually editing the files to resolve the conflicts
3. Removing all conflict markers
4. Committing and pushing the changes', steps.conflict-check.outputs.conflict_files) || 'All conflict markers have been successfully resolved in this pull request.' }}
- name: Fail workflow if conflicts detected
if: steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict'
env:
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
if: steps.conflict-check.outputs.has_conflicts == 'true'
run: |
[ "$HAS_CONFLICTS" = "true" ] && echo "::error::Conflict markers detected in changed files"
[ "$MERGE_STATUS" = "conflict" ] && echo "::error::PR branch has merge conflicts with the base branch"
echo "::error::Workflow failed due to conflict markers detected in the PR"
exit 1
+3 -3
View File
@@ -26,7 +26,7 @@ jobs:
contents: read
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -56,6 +56,6 @@ jobs:
"PROWLER_PR_BODY": ${{ toJson(github.event.pull_request.body) }},
"PROWLER_PR_URL": ${{ toJson(github.event.pull_request.html_url) }},
"PROWLER_PR_MERGED_BY": "${{ github.event.pull_request.merged_by.login }}",
"PROWLER_PR_BASE_BRANCH": ${{ toJson(github.event.pull_request.base.ref) }},
"PROWLER_PR_HEAD_BRANCH": ${{ toJson(github.event.pull_request.head.ref) }}
"PROWLER_PR_BASE_BRANCH": "${{ github.event.pull_request.base.ref }}",
"PROWLER_PR_HEAD_BRANCH": "${{ github.event.pull_request.head.ref }}"
}
+1 -8
View File
@@ -25,21 +25,14 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
actions: write
contents: write
pull-requests: write
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
- name: Enable release freeze
env:
GH_TOKEN: ${{ github.token }}
run: |
gh variable set RELEASE_FREEZE --body true --repo "${GITHUB_REPOSITORY}"
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
-45
View File
@@ -1,45 +0,0 @@
name: 'Tools: Release Freeze Gate'
on:
pull_request:
branches:
- 'master'
types:
- opened
- synchronize
- reopened
- ready_for_review
merge_group:
branches:
- 'master'
types:
- checks_requested
workflow_dispatch:
permissions: {}
jobs:
release-freeze-gate:
name: release-freeze-gate
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
with:
egress-policy: audit
- name: Check release freeze status
env:
RELEASE_FREEZE: ${{ vars.RELEASE_FREEZE }}
run: |
case "${RELEASE_FREEZE}" in
true|TRUE|True)
echo "::error::Release freeze is active. Merges to master are temporarily blocked."
echo "Set the RELEASE_FREEZE repository variable to false when the release is complete."
exit 1
;;
*)
echo "Release freeze is not active."
;;
esac
@@ -28,13 +28,12 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
api.github.com:443
github.com:443
raw.githubusercontent.com:443
objects.githubusercontent.com:443
codeload.github.com:443
release-assets.githubusercontent.com:443
@@ -42,7 +41,6 @@ jobs:
files.pythonhosted.org:443
registry.npmjs.org:443
nodejs.org:443
releases.astral.sh:443
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
+1 -2
View File
@@ -33,7 +33,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -55,7 +55,6 @@ jobs:
files_ignore: |
.github/**
prowler/CHANGELOG.md
prowler/changelog.d/**
docs/**
permissions/**
api/**
+1 -3
View File
@@ -12,7 +12,6 @@ on:
- '.github/workflows/sdk-codeql.yml'
- '.github/codeql/sdk-codeql-config.yml'
- '!prowler/CHANGELOG.md'
- '!prowler/changelog.d/**'
pull_request:
branches:
- 'master'
@@ -24,7 +23,6 @@ on:
- '.github/workflows/sdk-codeql.yml'
- '.github/codeql/sdk-codeql-config.yml'
- '!prowler/CHANGELOG.md'
- '!prowler/changelog.d/**'
schedule:
- cron: '00 12 * * *'
@@ -53,7 +51,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
+18 -28
View File
@@ -60,7 +60,7 @@ jobs:
contents: read
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -98,7 +98,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -138,18 +138,15 @@ jobs:
permissions:
contents: read
packages: write
id-token: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
api.ecr-public.us-east-1.amazonaws.com:443
public.ecr.aws:443
sts.amazonaws.com:443
sts.us-east-1.amazonaws.com:443
registry-1.docker.io:443
production.cloudflare.docker.com:443
production.cloudfront.docker.com:443
@@ -176,16 +173,14 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Configure AWS credentials (OIDC)
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
with:
aws-region: us-east-1
role-to-assume: ${{ secrets.PUBLIC_ECR_PUSH_ROLE_ARN }}
- name: Login to Public ECR
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry-type: public
registry: public.ecr.aws
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
env:
AWS_REGION: ${{ env.AWS_REGION }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
@@ -211,11 +206,10 @@ jobs:
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -227,8 +221,6 @@ jobs:
github.com:443
release-assets.githubusercontent.com:443
api.ecr-public.us-east-1.amazonaws.com:443
sts.amazonaws.com:443
sts.us-east-1.amazonaws.com:443
- name: Login to DockerHub
@@ -237,16 +229,14 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Configure AWS credentials (OIDC)
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
with:
aws-region: us-east-1
role-to-assume: ${{ secrets.PUBLIC_ECR_PUSH_ROLE_ARN }}
- name: Login to Public ECR
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry-type: public
registry: public.ecr.aws
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
env:
AWS_REGION: ${{ env.AWS_REGION }}
- name: Create and push manifests for push event
if: github.event_name == 'push'
@@ -309,7 +299,7 @@ jobs:
- name: Install regctl
if: always()
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
- name: Cleanup intermediate architecture tags
if: always()
@@ -330,7 +320,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+2 -6
View File
@@ -35,7 +35,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -71,7 +71,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -94,8 +94,6 @@ jobs:
_http._tcp.deb.debian.org:443
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
get.trivy.dev:443
raw.githubusercontent.com:443
releases.astral.sh:443
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -113,10 +111,8 @@ jobs:
pyproject.toml
uv.lock
.github/workflows/sdk-container-checks.yml
.github/actions/trivy-scan/**
files_ignore: |
prowler/CHANGELOG.md
prowler/changelog.d/**
**/AGENTS.md
- name: Set up Docker Buildx
+3 -3
View File
@@ -28,7 +28,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -66,7 +66,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -102,7 +102,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -27,7 +27,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+1 -2
View File
@@ -37,7 +37,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -73,7 +73,6 @@ jobs:
.github/scripts/osv-scan.sh
files_ignore: |
prowler/CHANGELOG.md
prowler/changelog.d/**
**/AGENTS.md
- name: Setup Python with uv
+1 -26
View File
@@ -33,7 +33,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -77,7 +77,6 @@ jobs:
files_ignore: |
.github/**
prowler/CHANGELOG.md
prowler/changelog.d/**
docs/**
permissions/**
api/**
@@ -615,30 +614,6 @@ jobs:
flags: prowler-py${{ matrix.python-version }}-linode
files: ./linode_coverage.xml
# E2E Networks Provider
- name: Check if E2E Networks files changed
if: steps.check-changes.outputs.any_changed == 'true'
id: changed-e2enetworks
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
./prowler/**/e2enetworks/**
./tests/**/e2enetworks/**
./uv.lock
- name: Run E2E Networks tests
if: steps.changed-e2enetworks.outputs.any_changed == 'true'
run: uv run pytest -n auto --cov=./prowler/providers/e2enetworks --cov-report=xml:e2enetworks_coverage.xml tests/providers/e2enetworks
- name: Upload E2E Networks coverage to Codecov
if: steps.changed-e2enetworks.outputs.any_changed == 'true'
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: prowler-py${{ matrix.python-version }}-e2enetworks
files: ./e2enetworks_coverage.xml
# External Provider (dynamic loading)
- name: Check if External Provider files changed
if: steps.check-changes.outputs.any_changed == 'true'
+2 -2
View File
@@ -52,7 +52,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -73,7 +73,7 @@ jobs:
- name: Setup Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: '3.12.13'
python-version: '3.12'
- name: Install PyYAML
run: pip install pyyaml
+1 -3
View File
@@ -10,7 +10,6 @@ on:
- '.github/workflows/ui-codeql.yml'
- '.github/codeql/ui-codeql-config.yml'
- '!ui/CHANGELOG.md'
- '!ui/changelog.d/**'
pull_request:
branches:
- 'master'
@@ -20,7 +19,6 @@ on:
- '.github/workflows/ui-codeql.yml'
- '.github/codeql/ui-codeql-config.yml'
- '!ui/CHANGELOG.md'
- '!ui/changelog.d/**'
schedule:
- cron: '00 12 * * *'
@@ -49,7 +47,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -45,7 +45,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -64,7 +64,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -107,7 +107,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -160,7 +160,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -201,7 +201,7 @@ jobs:
- name: Install regctl
if: always()
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
- name: Cleanup intermediate architecture tags
if: always()
@@ -222,7 +222,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+3 -6
View File
@@ -33,7 +33,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -69,7 +69,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
@@ -99,12 +99,9 @@ jobs:
id: check-changes
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
ui/**
.github/actions/trivy-scan/**
files: ui/**
files_ignore: |
ui/CHANGELOG.md
ui/changelog.d/**
ui/README.md
ui/AGENTS.md
+73 -123
View File
@@ -5,30 +5,71 @@ name: UI - E2E Tests (Optimized)
# critical paths are changed or if impact analysis fails.
on:
pull_request:
push:
branches:
- master
- "v5.*"
paths:
- '.github/workflows/ui-e2e-tests-v2.yml'
- '.github/workflows/test-impact-analysis.yml'
- '.github/test-impact.yml'
- 'ui/**'
pull_request:
types:
- opened
- synchronize
- reopened
- labeled
- unlabeled
branches:
- master
- "v5.*"
paths:
- '.github/workflows/ui-e2e-tests-v2.yml'
- '.github/workflows/test-impact-analysis.yml'
- '.github/test-impact.yml'
- 'ui/**'
- 'api/**' # API changes can affect UI E2E
- '!ui/CHANGELOG.md'
- '!api/CHANGELOG.md'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions: {}
permissions:
contents: read
jobs:
# First, analyze which tests need to run
impact-analysis:
if: github.repository == 'prowler-cloud/prowler'
# Trusted PR authors get the opt-in label automatically. This job does not
# check out or execute PR code; it only calls the GitHub API for trusted users.
auto-label-trusted-pr:
if: |
github.event_name == 'pull_request' &&
(github.event.action == 'opened' || github.event.action == 'reopened') &&
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) &&
!contains(github.event.pull_request.labels.*.name, 'run-ui-e2e')
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
steps:
- name: Add UI E2E opt-in label
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
script: |
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.pull_request.number,
labels: ['run-ui-e2e'],
});
# UI E2E consumes cloud credentials, so PR runs require explicit maintainer opt-in.
# On protected branch pushes, run independently of PR labels.
impact-analysis:
if: |
github.repository == 'prowler-cloud/prowler' &&
(github.event_name == 'push' ||
contains(github.event.pull_request.labels.*.name, 'run-ui-e2e') ||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))
uses: ./.github/workflows/test-impact-analysis.yml
# Run E2E tests based on impact analysis
@@ -36,9 +77,12 @@ jobs:
needs: impact-analysis
if: |
github.repository == 'prowler-cloud/prowler' &&
(github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) &&
(github.event_name == 'push' ||
contains(github.event.pull_request.labels.*.name, 'run-ui-e2e') ||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)) &&
(needs.impact-analysis.outputs.has-ui-e2e == 'true' || needs.impact-analysis.outputs.run-all == 'true')
runs-on: ubuntu-latest
environment: ui-e2e-cloud
env:
AUTH_SECRET: 'fallback-ci-secret-for-testing'
AUTH_TRUST_HOST: true
@@ -97,7 +141,7 @@ jobs:
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
@@ -119,104 +163,6 @@ jobs:
env:
NEEDS_IMPACT_ANALYSIS_OUTPUTS_MODULES: ${{ needs.impact-analysis.outputs.modules }}
- name: Validate E2E prerequisites
shell: bash
run: |
declare -A required=()
suite_selected() {
[[ "${RUN_ALL_TESTS}" == "true" ]] ||
[[ " ${E2E_TEST_PATHS} " == *"ui/tests/$1/"* ]]
}
require_vars() {
local variable
for variable in "$@"; do
required["${variable}"]=1
done
}
if suite_selected auth || suite_selected providers ||
suite_selected invitations || suite_selected scans ||
suite_selected navigation; then
require_vars E2E_ADMIN_USER E2E_ADMIN_PASSWORD
fi
if suite_selected sign-up; then
require_vars E2E_NEW_USER_PASSWORD
fi
if suite_selected invitations; then
require_vars E2E_NEW_USER_PASSWORD E2E_ORGANIZATION_ID
fi
if suite_selected scans; then
require_vars \
E2E_AWS_PROVIDER_ACCOUNT_ID \
E2E_AWS_PROVIDER_ACCESS_KEY \
E2E_AWS_PROVIDER_SECRET_KEY
fi
if suite_selected providers; then
require_vars \
E2E_AWS_PROVIDER_ACCOUNT_ID \
E2E_AWS_PROVIDER_ACCESS_KEY \
E2E_AWS_PROVIDER_SECRET_KEY \
E2E_AWS_PROVIDER_ROLE_ARN \
E2E_AZURE_SUBSCRIPTION_ID \
E2E_AZURE_CLIENT_ID \
E2E_AZURE_SECRET_ID \
E2E_AZURE_TENANT_ID \
E2E_M365_DOMAIN_ID \
E2E_M365_CLIENT_ID \
E2E_M365_SECRET_ID \
E2E_M365_TENANT_ID \
E2E_M365_CERTIFICATE_CONTENT \
E2E_GCP_BASE64_SERVICE_ACCOUNT_KEY \
E2E_GCP_PROJECT_ID \
E2E_GITHUB_APP_ID \
E2E_GITHUB_BASE64_APP_PRIVATE_KEY \
E2E_GITHUB_USERNAME \
E2E_GITHUB_PERSONAL_ACCESS_TOKEN \
E2E_GITHUB_ORGANIZATION \
E2E_GITHUB_ORGANIZATION_ACCESS_TOKEN \
E2E_OCI_TENANCY_ID \
E2E_OCI_USER_ID \
E2E_OCI_FINGERPRINT \
E2E_OCI_KEY_CONTENT \
E2E_ALIBABACLOUD_ACCOUNT_ID \
E2E_ALIBABACLOUD_ACCESS_KEY_ID \
E2E_ALIBABACLOUD_ACCESS_KEY_SECRET \
E2E_ALIBABACLOUD_ROLE_ARN \
E2E_OKTA_DOMAIN \
E2E_OKTA_CLIENT_ID \
E2E_OKTA_BASE64_PRIVATE_KEY \
E2E_GOOGLEWORKSPACE_CUSTOMER_ID \
E2E_GOOGLEWORKSPACE_SERVICE_ACCOUNT_JSON \
E2E_GOOGLEWORKSPACE_DELEGATED_USER \
E2E_VERCEL_TEAM_ID \
E2E_VERCEL_API_TOKEN
fi
missing=()
if (( ${#required[@]} > 0 )); then
while IFS= read -r variable; do
[[ -z "${!variable:-}" ]] && missing+=("${variable}")
done < <(printf '%s\n' "${!required[@]}" | sort)
fi
if (( ${#missing[@]} > 0 )); then
echo "Missing required E2E variables:"
printf ' - %s\n' "${missing[@]}"
{
echo "## Missing E2E prerequisites"
printf -- "- \`%s\`\n" "${missing[@]}"
} >> "${GITHUB_STEP_SUMMARY}"
exit 1
fi
echo "E2E prerequisite preflight passed."
- name: Create k8s Kind Cluster
uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1
with:
@@ -237,8 +183,8 @@ jobs:
- name: Add AWS credentials for testing
run: |
echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env
echo "AWS_ACCESS_KEY_ID=${E2E_AWS_PROVIDER_ACCESS_KEY}" >> .env
echo "AWS_SECRET_ACCESS_KEY=${E2E_AWS_PROVIDER_SECRET_KEY}" >> .env
- name: Build API image from current code
# docker-compose.yml references prowlercloud/prowler-api:latest from the registry,
@@ -403,34 +349,38 @@ jobs:
run: |
docker compose down -v || true
# Fork pull requests cannot access the secrets required by the E2E suites.
fork-e2e-unavailable:
needs: impact-analysis
# Skip job - provides clear feedback when UI E2E is not explicitly authorized.
skip-e2e-no-label:
if: |
github.repository == 'prowler-cloud/prowler' &&
github.event_name == 'pull_request' &&
github.event.pull_request.head.repo.fork == true &&
(needs.impact-analysis.outputs.has-ui-e2e == 'true' || needs.impact-analysis.outputs.run-all == 'true')
!contains(github.event.pull_request.labels.*.name, 'run-ui-e2e') &&
!contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
- name: Report unavailable E2E tests
- name: UI E2E skipped - opt-in label missing
run: |
echo "## E2E Tests Skipped" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "UI E2E tests require repository secrets and cannot run for fork pull requests." >> "$GITHUB_STEP_SUMMARY"
echo "## UI E2E Tests Skipped" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "UI E2E tests consume cloud credentials and are skipped unless a maintainer adds the run-ui-e2e label." >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Add the label to opt in; remove it to stop secret-consuming UI E2E jobs." >> $GITHUB_STEP_SUMMARY
# Skip job - provides clear feedback when no E2E tests needed
skip-e2e:
# Skip job - provides clear feedback when no E2E tests needed after opt-in.
skip-e2e-no-tests:
needs: impact-analysis
if: |
github.repository == 'prowler-cloud/prowler' &&
(github.event_name == 'push' ||
contains(github.event.pull_request.labels.*.name, 'run-ui-e2e') ||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)) &&
needs.impact-analysis.outputs.has-ui-e2e != 'true' &&
needs.impact-analysis.outputs.run-all != 'true'
runs-on: ubuntu-latest
@@ -438,7 +388,7 @@ jobs:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
+1 -1
View File
@@ -32,7 +32,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
+3 -8
View File
@@ -31,18 +31,14 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: block
allowed-endpoints: >
github.com:443
registry.npmjs.org:443
nodejs.org:443
fonts.googleapis.com:443
fonts.gstatic.com:443
api.iconify.design:443
api.simplesvg.com:443
api.unisvg.com:443
api.github.com:443
release-assets.githubusercontent.com:443
cdn.playwright.dev:443
@@ -64,7 +60,6 @@ jobs:
.github/workflows/ui-tests.yml
files_ignore: |
ui/CHANGELOG.md
ui/changelog.d/**
ui/README.md
ui/AGENTS.md
@@ -181,9 +176,9 @@ jobs:
if: steps.check-changes.outputs.any_changed == 'true' && steps.playwright-cache.outputs.cache-hit != 'true'
run: pnpm exec playwright install chromium
- name: Run integration tests
- name: Run browser tests
if: steps.check-changes.outputs.any_changed == 'true'
run: pnpm run test:integration
run: pnpm run test:browser
- name: Build application
if: steps.check-changes.outputs.any_changed == 'true'
-2
View File
@@ -173,5 +173,3 @@ GEMINI.md
# Docker
docker-compose.override.yml
docker-compose-dev.override.yml
# Local Pi runtime state
.atl/
+6 -14
View File
@@ -72,13 +72,13 @@ repos:
exclude: contrib
priority: 30
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/, docs/scripts/)
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/)
- repo: https://github.com/myint/autoflake
rev: v2.3.3
hooks:
- id: autoflake
name: "SDK - autoflake"
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variable"]
priority: 20
@@ -87,7 +87,7 @@ repos:
hooks:
- id: isort
name: "SDK - isort"
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
args: ["--profile", "black"]
stages: ["pre-commit"]
priority: 20
@@ -97,7 +97,7 @@ repos:
hooks:
- id: black
name: "SDK - black"
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
priority: 20
- repo: https://github.com/pycqa/flake8
@@ -105,7 +105,7 @@ repos:
hooks:
- id: flake8
name: "SDK - flake8"
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
args: ["--ignore=E266,W503,E203,E501,W605"]
priority: 30
@@ -142,14 +142,6 @@ repos:
files: { glob: ["mcp_server/**/*.py"] }
priority: 20
- id: generate-provider-cards
name: "Docs - regenerate provider cards snippet"
entry: python3 docs/scripts/generate_provider_cards.py
language: system
files: { glob: ["docs/user-guide/providers/**/getting-started-*.mdx", "docs/scripts/generate_provider_cards.py", "docs/snippets/provider-cards.mdx", "api/src/backend/api/models.py"] }
pass_filenames: false
priority: 20
## PYTHON — uv (API + SDK)
- repo: https://github.com/astral-sh/uv-pre-commit
rev: 0.11.14
@@ -191,7 +183,7 @@ repos:
entry: pylint --disable=W,C,R,E -j 0 -rn -sn
language: system
types: [python]
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
priority: 30
- id: trufflehog
+22 -47
View File
@@ -15,46 +15,21 @@
# neither vulnerable code path (Archive::Tar parsing or regex compilation of
# attacker-controlled input) is reachable from Prowler. No Debian bookworm fix
# is available yet.
CVE-2026-42496 pkg:perl exp:2026-08-15
CVE-2026-42496 pkg:perl-base exp:2026-08-15
CVE-2026-42496 pkg:perl-modules-5.36 exp:2026-08-15
CVE-2026-42496 pkg:libperl5.36 exp:2026-08-15
CVE-2026-8376 pkg:perl exp:2026-08-15
CVE-2026-8376 pkg:perl-base exp:2026-08-15
CVE-2026-8376 pkg:perl-modules-5.36 exp:2026-08-15
CVE-2026-8376 pkg:libperl5.36 exp:2026-08-15
# CVE-2026-13221 - Perl regex trie overflow.
# Packages: perl, perl-base, perl-modules-5.36, libperl5.36.
# Why ignored: upstream confirms Perl 5.36.0 is not affected; the regression
# was introduced after this version. Debian currently marks bookworm as
# vulnerable, which causes Trivy to report a false positive.
# Ref: https://github.com/Perl/perl5/issues/23388
CVE-2026-13221 pkg:perl exp:2026-08-15
CVE-2026-13221 pkg:perl-base exp:2026-08-15
CVE-2026-13221 pkg:perl-modules-5.36 exp:2026-08-15
CVE-2026-13221 pkg:libperl5.36 exp:2026-08-15
# CVE-2026-57433 — Perl Storable signed integer overflow when deserializing a
# crafted SX_HOOK record (retrieve_hook_common passes a wrapped negative count
# to av_extend).
# Packages: perl, perl-base, perl-modules-5.36, libperl5.36.
# Why ignored: perl-base is part of Debian's "Essential: yes" set; it cannot be
# removed without breaking dpkg. Prowler does not invoke perl at runtime and
# never calls Storable's thaw/retrieve on attacker-controlled blobs, so the
# vulnerable deserialization path is unreachable. Fixed upstream in
# Storable 3.41; no Debian bookworm fix is available yet.
CVE-2026-57433 pkg:perl exp:2026-08-15
CVE-2026-57433 pkg:perl-base exp:2026-08-15
CVE-2026-57433 pkg:perl-modules-5.36 exp:2026-08-15
CVE-2026-57433 pkg:libperl5.36 exp:2026-08-15
CVE-2026-42496 pkg:perl exp:2026-07-15
CVE-2026-42496 pkg:perl-base exp:2026-07-15
CVE-2026-42496 pkg:perl-modules-5.36 exp:2026-07-15
CVE-2026-42496 pkg:libperl5.36 exp:2026-07-15
CVE-2026-8376 pkg:perl exp:2026-07-15
CVE-2026-8376 pkg:perl-base exp:2026-07-15
CVE-2026-8376 pkg:perl-modules-5.36 exp:2026-07-15
CVE-2026-8376 pkg:libperl5.36 exp:2026-07-15
# CVE-2025-7458 — SQLite integer overflow.
# Package: libsqlite3-0.
# Why ignored: transitive dependency of CPython's stdlib sqlite3 module. The
# Prowler SDK does not open user-supplied SQLite databases; SQLite usage is
# internal and bounded. No Debian bookworm fix is available.
CVE-2025-7458 pkg:libsqlite3-0 exp:2026-08-15
CVE-2025-7458 pkg:libsqlite3-0 exp:2026-07-15
# CVE-2026-43185 — Linux kernel ksmbd signedness bug.
# Package: linux-libc-dev.
@@ -62,7 +37,7 @@ CVE-2025-7458 pkg:libsqlite3-0 exp:2026-08-15
# not a running kernel. Containers execute against the host kernel, so these
# headers are inert at runtime. The upstream fix landed in kernel 7.0-rc2 and
# has not been backported to Debian's 6.1 LTS line.
CVE-2026-43185 pkg:linux-libc-dev exp:2026-08-15
CVE-2026-43185 pkg:linux-libc-dev exp:2026-07-15
# CVE-2023-45853 — zlib MiniZip integer overflow / heap overflow in
# zipOpenNewFileInZip4_64.
@@ -74,8 +49,8 @@ CVE-2026-43185 pkg:linux-libc-dev exp:2026-08-15
# zlib 1.3.1, available in Debian trixie (13); migrating the base image would
# clear it fully.
# Ref: https://security-tracker.debian.org/tracker/CVE-2023-45853
CVE-2023-45853 pkg:zlib1g exp:2026-08-15
CVE-2023-45853 pkg:zlib1g-dev exp:2026-08-15
CVE-2023-45853 pkg:zlib1g exp:2026-07-15
CVE-2023-45853 pkg:zlib1g-dev exp:2026-07-15
# CVE-2026-55200 — libssh2 out-of-bounds write in ssh2_transport_read() due to
# an unchecked packet_length field in transport.c (heap corruption, possible RCE).
@@ -88,7 +63,7 @@ CVE-2023-45853 pkg:zlib1g-dev exp:2026-08-15
# affected code is unreachable at runtime. Fixed upstream in libssh2 commit
# 97acf3df (PR #2052); no Debian bookworm fix is available yet.
# Ref: https://security-tracker.debian.org/tracker/CVE-2026-55200
CVE-2026-55200 pkg:libssh2-1 exp:2026-08-15
CVE-2026-55200 pkg:libssh2-1 exp:2026-07-15
# --- API container image (api/Dockerfile) ---
# The entries below are specific to the Prowler API image, which ships
@@ -103,13 +78,13 @@ CVE-2026-55200 pkg:libssh2-1 exp:2026-08-15
# at runtime. The vulnerable path requires parsing attacker-controlled XML with
# the affected interpreter, which Prowler does not do with the system Python.
# Full mitigation also needs libexpat >= 2.8.0; no Debian bookworm fix yet.
CVE-2026-7210 pkg:python3.11 exp:2026-08-15
CVE-2026-7210 pkg:python3.11-dev exp:2026-08-15
CVE-2026-7210 pkg:python3.11-minimal exp:2026-08-15
CVE-2026-7210 pkg:libpython3.11 exp:2026-08-15
CVE-2026-7210 pkg:libpython3.11-dev exp:2026-08-15
CVE-2026-7210 pkg:libpython3.11-minimal exp:2026-08-15
CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-08-15
CVE-2026-7210 pkg:python3.11 exp:2026-07-15
CVE-2026-7210 pkg:python3.11-dev exp:2026-07-15
CVE-2026-7210 pkg:python3.11-minimal exp:2026-07-15
CVE-2026-7210 pkg:libpython3.11 exp:2026-07-15
CVE-2026-7210 pkg:libpython3.11-dev exp:2026-07-15
CVE-2026-7210 pkg:libpython3.11-minimal exp:2026-07-15
CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-07-15
# CVE-2026-33278 — Unbound DNSSEC validator use-after-free (DoS, possible RCE).
# CVE-2026-42960 — Unbound DNS cache poisoning via promiscuous additional records.
@@ -119,5 +94,5 @@ CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-08-15
# vulnerabilities require operating a live Unbound recursive DNSSEC validator
# that processes attacker-influenced DNS responses. Prowler never starts an
# Unbound resolver, so neither code path is reachable. No Debian bookworm fix yet.
CVE-2026-33278 pkg:libunbound8 exp:2026-08-15
CVE-2026-42960 pkg:libunbound8 exp:2026-08-15
CVE-2026-33278 pkg:libunbound8 exp:2026-07-15
CVE-2026-42960 pkg:libunbound8 exp:2026-07-15
-3
View File
@@ -62,7 +62,6 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
| Action | Skill |
|--------|-------|
| Add changelog entry for a PR or feature | `prowler-changelog` |
| Adding ConfigRequirements guardrails to compliance requirements | `prowler-compliance` |
| Adding DRF pagination or permissions | `django-drf` |
| Adding a compliance output formatter (per-provider class + table dispatcher) | `prowler-compliance` |
| Adding indexes or constraints to database tables | `django-migration-psql` |
@@ -85,7 +84,6 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
| Creating ViewSets, serializers, or filters in api/ | `django-drf` |
| Creating Zod schemas | `zod-4` |
| Creating a git commit | `prowler-commit` |
| Creating a universal (multi-provider) compliance framework | `prowler-compliance` |
| Creating new checks | `prowler-sdk-check` |
| Creating new skills | `skill-creator` |
| Creating or reviewing Django migrations | `django-migration-psql` |
@@ -116,7 +114,6 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
| Review changelog format and conventions | `prowler-changelog` |
| Reviewing JSON:API compliance | `jsonapi` |
| Reviewing Prowler UI components | `prowler-ui` |
| Reviewing compliance framework PRs | `prowler-compliance-review` |
| Running makemigrations or pgmakemigrations | `django-migration-psql` |
| Syncing compliance framework with upstream catalog | `prowler-compliance` |
+1 -3
View File
@@ -1,12 +1,10 @@
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
LABEL maintainer="https://github.com/prowler-cloud/prowler"
LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
ARG POWERSHELL_VERSION=7.5.0
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
# Opt out of PowerShell telemetry (Application Insights -> dc.services.visualstudio.com)
ENV POWERSHELL_TELEMETRY_OPTOUT=1
ARG TRIVY_VERSION=0.71.2
ENV TRIVY_VERSION=${TRIVY_VERSION}
+27 -70
View File
@@ -3,13 +3,10 @@
<img align="center" alt="Prowler logo" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-white.png#gh-dark-mode-only" width="50%" height="50%">
</p>
<p align="center">
<b><i>Prowler</b> is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With thousands of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
<b><i>Prowler</b> is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
</p>
<p align="center">
<b>The Agentic Cloud Defender</i></b>
</p>
<p align="center">
<a href="https://cloud.prowler.com/sign-up">Try Prowler Cloud</a>
<b>Secure ANY cloud at AI Speed at <a href="https://prowler.com">prowler.com</i></b>
</p>
<p align="center">
@@ -24,7 +21,7 @@
<a href="https://pypi.python.org/pypi/prowler/"><img alt="Python Version" src="https://img.shields.io/pypi/pyversions/prowler.svg"></a>
<a href="https://pypistats.org/packages/prowler"><img alt="PyPI Downloads" src="https://img.shields.io/pypi/dw/prowler.svg?label=downloads"></a>
<a href="https://hub.docker.com/r/toniblyx/prowler"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/toniblyx/prowler"></a>
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height="19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height=19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img alt="Codecov coverage" src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img alt="Linux Foundation insights health score" src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
</p>
@@ -44,7 +41,7 @@
# Description
**Prowler** is the worlds most widely used _Open-Source Cloud Security Platform_ that automates security and compliance across **any cloud environment**. With thousands of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY Cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size.
**Prowler** is the worlds most widely used _Open-Source Cloud Security Platform_ that automates security and compliance across **any cloud environment**. With hundreds of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY Cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size.
Prowler includes hundreds of built-in controls to ensure compliance with standards and frameworks, including:
@@ -57,16 +54,16 @@ Prowler includes hundreds of built-in controls to ensure compliance with standar
- **National Security Standards:** ENS (Spanish National Security Scheme) and KISA ISMS-P (Korean)
- **Custom Security Frameworks:** Tailored to your needs
## Prowler Cloud & Prowler Local Server
## Prowler App / Prowler Cloud
[Prowler Cloud](https://cloud.prowler.com/sign-up) and Prowler Local Server, its self-hosted open-source version, are web applications that simplify running Prowler across your cloud provider accounts. They provide a user-friendly interface to visualize the results and streamline your security assessments.
Prowler App / [Prowler Cloud](https://cloud.prowler.com/) is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments.
![Prowler Cloud](docs/images/products/overview.png)
![Prowler App](docs/images/products/overview.png)
![Risk Pipeline](docs/images/products/risk-pipeline.png)
![Threat Map](docs/images/products/threat-map.png)
>For more details, refer to the [Prowler Local Server documentation](https://docs.prowler.com/getting-started/installation/prowler-app)
>For more details, refer to the [Prowler App Documentation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-app-installation)
## Prowler CLI
@@ -76,12 +73,12 @@ prowler <provider>
![Prowler CLI Execution](docs/img/short-display.png)
## Prowler Local Dashboard
## Prowler Dashboard
```console
prowler dashboard
```
![Prowler Local Dashboard](docs/images/products/dashboard.png)
![Prowler Dashboard](docs/images/products/dashboard.png)
## Attack Paths
@@ -124,28 +121,26 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
> For the most accurate and up-to-date information about checks, services, frameworks, and categories, visit [**Prowler Hub**](https://hub.prowler.com).
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/user-guide/compliance/tutorials/compliance) | [Categories](https://docs.prowler.com/user-guide/cli/tutorials/misc#categories) | Support | Interface |
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|---|---|---|---|---|---|---|
| AWS | 621 | 86 | 47 | 19 | Official | UI, API, CLI |
| Azure | 191 | 22 | 21 | 16 | Official | UI, API, CLI |
| AWS | 615 | 86 | 47 | 19 | Official | UI, API, CLI |
| Azure | 190 | 22 | 21 | 16 | Official | UI, API, CLI |
| GCP | 109 | 20 | 19 | 12 | Official | UI, API, CLI |
| Kubernetes | 92 | 7 | 8 | 11 | Official | UI, API, CLI |
| Kubernetes | 90 | 7 | 8 | 11 | Official | UI, API, CLI |
| GitHub | 24 | 3 | 2 | 5 | Official | UI, API, CLI |
| M365 | 111 | 10 | 6 | 10 | Official | UI, API, CLI |
| M365 | 109 | 10 | 6 | 10 | Official | UI, API, CLI |
| OCI | 52 | 14 | 5 | 10 | Official | UI, API, CLI |
| Alibaba Cloud | 63 | 9 | 6 | 9 | Official | UI, API, CLI |
| Cloudflare | 29 | 3 | 2 | 5 | Official | UI, API, CLI |
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI |
| MongoDB Atlas | 10 | 3 | 1 | 8 | Official | UI, API, CLI |
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI |
| Image | N/A | N/A | N/A | N/A | Official | UI, API, CLI |
| Image | N/A | N/A | N/A | N/A | Official | CLI, API |
| Google Workspace | 65 | 11 | 3 | 6 | Official | UI, API, CLI |
| OpenStack | 34 | 5 | 1 | 9 | Official | UI, API, CLI |
| Vercel | 26 | 6 | 1 | 8 | Official | UI, API, CLI |
| Okta | 29 | 8 | 2 | 2 | Official | UI, API, CLI |
| Linode [Contact us](https://prowler.com/contact) | 10 | 3 | 1 | 4 | Unofficial | CLI |
| Huawei Cloud [Contact us](https://prowler.com/contact) | 25 | 10 | 1 | 6 | Unofficial | CLI |
| E2E Networks [Contact us](https://prowler.com/contact) | 27 | 6 | 0 | 2 | Unofficial | CLI |
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 1 | 1 | Unofficial | CLI |
| StackIT [Contact us](https://prowler.com/contact) | 7 | 2 | 1 | 3 | Unofficial | CLI |
| NHN | 6 | 2 | 2 | 0 | Unofficial | CLI |
@@ -164,11 +159,11 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
# 💻 Installation
## Prowler Local Server
## Prowler App
Prowler Local Server offers flexible installation methods tailored to various environments:
Prowler App offers flexible installation methods tailored to various environments:
> For detailed instructions on using Prowler Local Server, refer to the [usage guide](https://docs.prowler.com/user-guide/tutorials/prowler-app).
> For detailed instructions on using Prowler App, refer to the [Prowler App Usage Guide](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/prowler-app/).
### Docker Compose
@@ -201,7 +196,7 @@ docker compose up -d
> [!WARNING]
> 🔒 For a secure setup, the API auto-generates a unique key pair, `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY`, and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate keys, delete the stored key files and restart the API.
Once configured, access Prowler Local Server at http://localhost:3000. Sign up using your email and password to get started.
Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
### Common Issues with Docker Pull Installation
@@ -273,7 +268,7 @@ pnpm run build
pnpm start
```
> Once configured, access Prowler Local Server at http://localhost:3000. Sign up using your email and password to get started.
> Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
#### Pre-commit Hooks Setup
@@ -291,7 +286,7 @@ Prowler CLI is available as a project in [PyPI](https://pypi.org/project/prowler
pip install prowler
prowler -v
```
>For further guidance, refer to [https://docs.prowler.com](https://docs.prowler.com/getting-started/installation/prowler-cli)
>For further guidance, refer to [https://docs.prowler.com](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-cli-installation)
### Containers
@@ -311,7 +306,7 @@ The container images are available here:
- Prowler CLI:
- [DockerHub](https://hub.docker.com/r/prowlercloud/prowler/tags)
- [AWS Public ECR](https://gallery.ecr.aws/prowler-cloud/prowler)
- Prowler Local Server:
- Prowler App:
- [DockerHub - Prowler UI](https://hub.docker.com/r/prowlercloud/prowler-ui/tags)
- [DockerHub - Prowler API](https://hub.docker.com/r/prowlercloud/prowler-api/tags)
@@ -361,55 +356,17 @@ Full configuration, per-provider authentication, and SARIF examples: [Prowler Gi
# ✏️ High level architecture
## Prowler Local Server
**Prowler Local Server** is composed of four key components:
## Prowler App
**Prowler App** is composed of four key components:
- **Prowler UI**: A web-based interface, built with Next.js, providing a user-friendly experience for executing Prowler scans and visualizing results.
- **Prowler API**: A backend service, developed with Django REST Framework, responsible for running Prowler scans and storing the generated results.
- **Prowler SDK**: A Python SDK designed to extend the functionality of the Prowler CLI for advanced capabilities.
- **Prowler MCP Server**: A Model Context Protocol server that provides AI tools for Lighthouse, the AI-powered security assistant. This is a critical dependency for Lighthouse functionality.
```mermaid
flowchart TB
user([User / Security Team])
cli([Prowler CLI])
![Prowler App Architecture](docs/images/products/prowler-app-architecture.png)
subgraph APP["Prowler Local Server"]
ui["Prowler UI<br/>(Next.js)"]
api["Prowler API<br/>(Django REST Framework)"]
worker["API Worker<br/>(Celery)"]
beat["API Scheduler<br/>(Celery Beat)"]
mcp["Prowler MCP Server<br/>(Lighthouse AI tools)"]
end
sdk["Prowler SDK<br/>(Python)"]
subgraph DATA["Data Layer"]
pg[("PostgreSQL")]
valkey[("Valkey / Redis")]
neo4j[("Neo4j")]
end
providers["Providers"]
user --> ui
user --> cli
ui -->|REST| api
ui -->|MCP HTTP| mcp
mcp -->|REST| api
api --> pg
api --> valkey
beat -->|enqueue jobs| valkey
valkey -->|dispatch| worker
worker --> pg
worker -->|Attack Paths| neo4j
worker -->|invokes| sdk
cli --> sdk
sdk --> providers
```
<!-- Diagram source: docs/images/products/prowler-app-architecture.mmd — keep this inline block, the docs page getting-started/products/prowler-app.mdx, and the .mmd file in sync. -->
<!-- Diagram source: docs/images/products/prowler-app-architecture.mmd — edit there, re-render at https://mermaid.live, and replace the PNG. -->
## Prowler CLI
+5 -111
View File
@@ -2,125 +2,19 @@
All notable changes to the **Prowler API** are documented in this file.
<!-- changelog: release notes start -->
## [1.37.0] (Prowler v5.36.0)
### 🔄 Changed
- OCI provider secrets no longer require `region`; legacy `region` input is accepted for backwards compatibility but ignored before storing or scanning [(#11741)](https://github.com/prowler-cloud/prowler/pull/11741)
- Compliance overview ingest now runs in a single transaction per scan with a configurable `COPY` batch size (`DJANGO_COMPLIANCE_COPY_BATCH_SIZE`, default 2000), reducing write pressure on the database [(#11875)](https://github.com/prowler-cloud/prowler/pull/11875)
### 🐞 Fixed
- Scan findings now recover resources missing from the in-memory cache after resource pre-resolution, preventing valid findings from being skipped [(#12002)](https://github.com/prowler-cloud/prowler/pull/12002)
- Tenant-wide integrations that are not attached to any provider, such as Jira, are now visible and manageable by roles with `manage_integrations` and without unlimited visibility [(#12060)](https://github.com/prowler-cloud/prowler/pull/12060)
- Output generation now removes the scan's temporary output directory before writing, so a re-run of the task for the same scan (e.g. broker redelivery after a worker is killed mid-run) no longer appends to the previous run's files and duplicates finding rows in the exported CSV and other outputs [(#12097)](https://github.com/prowler-cloud/prowler/pull/12097)
### 🔐 Security
- Integration responses no longer disclose providers outside the visibility of the role, including the resources sideloaded through `?include=providers` [(#12060)](https://github.com/prowler-cloud/prowler/pull/12060)
- Integration connection checks, Jira issue type lookups and Jira dispatches now resolve the integration through the provider visibility of the role instead of the whole tenant [(#12060)](https://github.com/prowler-cloud/prowler/pull/12060)
- Roles without unlimited visibility can no longer attach an integration to providers they cannot see, nor edit or delete an integration bound to them [(#12060)](https://github.com/prowler-cloud/prowler/pull/12060)
- Kubernetes kubeconfig validation now rejects legacy `auth-provider.config.cmd-path` command authentication in Prowler Cloud/API [(#12091)](https://github.com/prowler-cloud/prowler/pull/12091)
---
## [1.36.0] (Prowler v5.35.0)
### 🐞 Fixed
- `attack-paths-scan-perform` Celery tasks now use the configurable long-task time limits instead of the six-hour defaults [(#12009)](https://github.com/prowler-cloud/prowler/pull/12009)
- Attack Paths scans handle provider deletion races cleanly, detect stale tasks after 16 hours, use backend-specific graph synchronization batches, and report exhausted Neptune write retries with the original database error [(#12019)](https://github.com/prowler-cloud/prowler/pull/12019)
### 🔐 Security
- Jira integration credentials only accept bare Atlassian site names containing letters, numbers, and hyphens [(#12012)](https://github.com/prowler-cloud/prowler/pull/12012)
- Social account linking requires a verified matching email from both the identity provider and the existing user account without sending account connection notifications [(#12013)](https://github.com/prowler-cloud/prowler/pull/12013)
---
## [1.35.0] (Prowler v5.34.0)
### 🐞 Fixed
- `rls_transaction` now falls back directly to the primary DB for connection-level mid-query read replica failures via `execute_wrapper`, reducing non-streaming read crashes during replica recovery [(#10379)](https://github.com/prowler-cloud/prowler/pull/10379)
- RBAC permission gates now combine permissions from every role assigned to a user in the active tenant [(#11979)](https://github.com/prowler-cloud/prowler/pull/11979)
- `attack-paths-cleanup-stale-scans` now retries worker pings and checks recent scan activity before failing scans and removing temporary databases [(#11986)](https://github.com/prowler-cloud/prowler/pull/11986)
### 🔐 Security
- User role relationship updates are limited to the active tenant to preserve role assignments in other tenants [(#11903)](https://github.com/prowler-cloud/prowler/pull/11903)
- `api` container image removes the unused Debian `libxml2` runtime package and scopes the `CVE-2026-13221` Trivy exception to unaffected Perl 5.36 packages [(#11991)](https://github.com/prowler-cloud/prowler/pull/11991)
---
## [1.34.2] (Prowler v5.33.2)
### 🐞 Fixed
- Attack Paths graph mutations now retry transient Neptune concurrency and deadline failures, while Neo4j mutations use managed transaction retries [(#11968)](https://github.com/prowler-cloud/prowler/pull/11968)
- Attack Paths scans now use bounded child node identifiers for normalized list values in Neo4j and Neptune, preventing Neo4j RANGE index key size failures [(#11969)](https://github.com/prowler-cloud/prowler/pull/11969)
- `scan-summary` aggregation now upserts summaries in deterministic conflict-key order, preventing PostgreSQL deadlocks during concurrent reaggregation [(#11971)](https://github.com/prowler-cloud/prowler/pull/11971)
---
## [1.34.1] (Prowler v5.33.1)
### 🐞 Fixed
- Session tokens are rejected after account password updates [(#11914)](https://github.com/prowler-cloud/prowler/pull/11914)
- Jira dispatch task results now surface user-facing Jira failure messages [(#11925)](https://github.com/prowler-cloud/prowler/pull/11925)
- AWS Attack Paths privilege escalation queries no longer fail on Neo4j with `Aggregation column contains implicit grouping expressions` [(#11939)](https://github.com/prowler-cloud/prowler/pull/11939)
### 🔐 Security
- OpenAI-compatible Lighthouse provider base URLs are restricted before connection checks [(#11940)](https://github.com/prowler-cloud/prowler/pull/11940)
- `LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS` environment variable to allow internal hosts as OpenAI-compatible Lighthouse AI base URLs [(#11942)](https://github.com/prowler-cloud/prowler/pull/11942)
---
## [1.34.0] (Prowler v5.33.0)
### 🚀 Added
- Compliance PDF reports no longer require provider credentials: findings are enriched from the provider metadata stored in the database, so reports generate even after the provider secret is deleted or its credentials become invalid [(#11845)](https://github.com/prowler-cloud/prowler/pull/11845)
### 🐞 Fixed
- Provider scans now queue behind active provider scans instead of dispatching concurrently, and resource failed-finding counters retry database conflicts with stable row locking [(#11848)](https://github.com/prowler-cloud/prowler/pull/11848)
---
## [1.33.1] (Prowler v5.32.1)
### 🐞 Fixed
- Attack Paths: Scan rows now have database defaults for `is_migrated` and `sink_backend` so `scan-perform-scheduled` inserts survive deploy skew [(#11826)](https://github.com/prowler-cloud/prowler/pull/11826)
- Invited users now keep their invitation context when completing authentication with Google, GitHub, or SAML, so the invitation is accepted during login [(#11752)](https://github.com/prowler-cloud/prowler/pull/11752)
### 🔐 Security
- User profile updates now allow users to update their own account while requiring user-management permissions to update other users in the same tenant [(#11792)](https://github.com/prowler-cloud/prowler/pull/11792)
- Kubernetes provider credentials now reject kubeconfigs using `exec` authentication in Prowler Cloud, preventing user-supplied commands from running on Cloud workers [(#11753)](https://github.com/prowler-cloud/prowler/pull/11753)
---
## [1.33.0] (Prowler v5.32.0)
### 🚀 Added
- Timestamp precision support for `/api/v1/findings` `inserted_at` and `updated_at` filters [(#11754)](https://github.com/prowler-cloud/prowler/pull/11754)
## [1.33.0] (Prowler UNRELEASED)
### 🔄 Changed
- Attack Paths: AWS Neptune is now supported as a persistent sink database, selectable via `ATTACK_PATHS_SINK_DATABASE=neptune` (default `neo4j`), Cartography's (bumped to 0.138.1) per-scan ingest database stays on Neo4j [(#11524)](https://github.com/prowler-cloud/prowler/pull/11524)
- Attack Paths: Scan task now checks the ingest Neo4j database and configured graph sink before starting graph ingestion [(#11743)](https://github.com/prowler-cloud/prowler/pull/11743)
- Disable PowerShell telemetry in the API container image [(#11746)](https://github.com/prowler-cloud/prowler/pull/11746)
---
## [1.32.2] (Prowler UNRELEASED)
### 🐞 Fixed
- Attack Paths: Provider graph cleanup now deletes Neo4j and Neptune relationships in directed batches before deleting nodes [(#11755)](https://github.com/prowler-cloud/prowler/pull/11755)
- `scan-perform` no longer reports an error when a provider is deleted during a running scan [(#11696)](https://github.com/prowler-cloud/prowler/pull/11696)
---
+2 -7
View File
@@ -1,11 +1,9 @@
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
LABEL maintainer="https://github.com/prowler-cloud/api"
ARG POWERSHELL_VERSION=7.5.0
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
# Opt out of PowerShell telemetry (Application Insights -> dc.services.visualstudio.com)
ENV POWERSHELL_TELEMETRY_OPTOUT=1
ARG TRIVY_VERSION=0.71.2
ENV TRIVY_VERSION=${TRIVY_VERSION}
@@ -102,9 +100,7 @@ ENV PATH="/home/prowler/.local/bin:$PATH"
RUN uv sync --locked --no-install-project && \
rm -rf ~/.cache/uv
# Invoked as a module so the base image's Python minor version is not baked
# into a site-packages path.
RUN .venv/bin/python -m prowler.providers.m365.lib.powershell.m365_powershell
RUN .venv/bin/python .venv/lib/python3.12/site-packages/prowler/providers/m365/lib/powershell/m365_powershell.py
USER root
@@ -115,7 +111,6 @@ RUN apt-get purge -y --auto-remove \
make \
libxml2-dev \
libxmlsec1-dev \
libxmlsec1-openssl \
pkg-config \
libtool \
libxslt1-dev \
-1
View File
@@ -1 +0,0 @@
Authentication with an API key whose owning user was deleted now returns `401` instead of an unhandled `AttributeError`, and user deletion now revokes the user's API keys across all their tenants
-10
View File
@@ -1,10 +0,0 @@
# Changelog fragments
Each PR adds one small file here instead of editing `CHANGELOG.md` directly, so concurrent PRs never conflict.
- Filename: `<slug>.<type>.md`, e.g. `my-new-check.added.md` (slug is free-form: letters, digits, `.`, `_`, `-`)
- `<type>` is one of: `added`, `changed`, `deprecated`, `removed`, `fixed`, `security`
- Content: one line with the changelog entry text, without the PR link and without a trailing period (the PR link is attached automatically at release time)
- A PR adds as many fragment files as entries it needs, freely mixing types (one file per entry); same-type entries just use different slugs
Fragments are compiled into `CHANGELOG.md` when a release is prepared. Full conventions: `skills/prowler-changelog/SKILL.md`.
@@ -1 +0,0 @@
Attack Paths IAM privilege-escalation queries no longer build an all-nodes × all-resource-items cartesian product, fixing runtime errors and timeouts on accounts with many IAM roles, users, or groups
@@ -1 +0,0 @@
Attack Paths adds four AWS privilege-escalation detection queries from pathfinding.cloud: cross-account role trust (STS-002), wildcard role trust (STS-003), user permissions-boundary removal (IAM-022), and IAM Identity Center permission-set escalation (SSO-001)
@@ -1 +0,0 @@
Attack Paths predefined queries on migrated graphs are now scoped with the provider label, letting the graph database seed from its label index instead of a global label scan and preventing query timeouts on Neptune
@@ -1 +0,0 @@
`task_args` serialization no longer returns HTTP 500 errors when Celery truncates stored task keyword arguments
@@ -1 +0,0 @@
Provider deletion and connection checks, scan creation, provider secrets, provider groups, and daily schedules now respect role provider-group visibility
@@ -1 +0,0 @@
SAML users without a `userType` attribute and without an existing role in the SAML tenant now receive a least-privilege `read_only` fallback role; a numeric suffix is used when that name belongs to a role with different permissions
@@ -1 +0,0 @@
AWS Security Hub integrations now persist successful connection checks during finding delivery so their connection status and last checked timestamp stay current
@@ -1 +0,0 @@
Social signups create users and authentication records in one database transaction, preventing incomplete accounts when provisioning fails
+1 -1
View File
@@ -71,7 +71,7 @@ name = "prowler-api"
package-mode = false
# Needed for the SDK compatibility
requires-python = ">=3.11,<3.13"
version = "1.38.0"
version = "1.33.0"
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
# target-version tracks this project's lowest supported Python.
+25 -73
View File
@@ -1,7 +1,5 @@
from allauth.account.models import EmailAddress
from allauth.core.exceptions import ImmediateHttpResponse
from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
from api.db_router import MainRouter, write_db_alias
from api.db_router import MainRouter
from api.db_utils import rls_transaction
from api.models import (
Membership,
@@ -11,9 +9,7 @@ from api.models import (
User,
UserRoleRelationship,
)
from api.utils import accept_invitation_for_user
from django.db import transaction
from django.http import HttpResponseForbidden
class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
@@ -24,30 +20,9 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
except User.DoesNotExist:
return None
@staticmethod
def _get_invitation_token(request):
for source_name in ("data", "POST"):
data = getattr(request, source_name, None) or {}
if not hasattr(data, "get"):
continue
invitation_token = data.get("invitation_token")
if invitation_token:
return invitation_token
wrapped_request = getattr(request, "_request", None)
if wrapped_request and wrapped_request is not request:
return ProwlerSocialAccountAdapter._get_invitation_token(wrapped_request)
return None
def pre_social_login(self, request, sociallogin):
# The provider account is already bound, so no email-based linking is needed.
if sociallogin.account.pk:
return
# Prefer the normalized email populated by allauth. GitHub can return the
# primary email separately from the profile stored in extra_data.
email = sociallogin.user.email or sociallogin.account.extra_data.get("email")
# Link existing accounts with the same email address
email = sociallogin.account.extra_data.get("email")
if sociallogin.provider.id == "saml":
# For SAML, the asserted NameID email cannot be trusted on its own:
# any tenant can claim any email domain in its SAML configuration. To
@@ -88,17 +63,6 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
if email:
existing_user = self.get_user_by_email(email)
if existing_user:
email_is_verified = EmailAddress.objects.filter(
user=existing_user,
email__iexact=email,
verified=True,
).exists()
provider_verified_email = any(
address.verified and address.email.casefold() == email.casefold()
for address in sociallogin.email_addresses
)
if not email_is_verified or not provider_verified_email:
raise ImmediateHttpResponse(HttpResponseForbidden())
sociallogin.connect(request, existing_user)
def save_user(self, request, sociallogin, form=None):
@@ -107,10 +71,7 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
and is about to be saved to the DB for the first time.
"""
with transaction.atomic(using=MainRouter.admin_db):
# Allauth saves the user without an explicit alias. Route that save
# through admin so every signup record shares this transaction.
with write_db_alias(MainRouter.admin_db):
user = super().save_user(request, sociallogin, form)
user = super().save_user(request, sociallogin, form)
provider = sociallogin.provider.id
extra = sociallogin.account.extra_data
@@ -122,38 +83,29 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
user.name = social_account_name
user.save(using=MainRouter.admin_db)
invitation_token = self._get_invitation_token(request)
if invitation_token:
invitation, _ = accept_invitation_for_user(
tenant = Tenant.objects.using(MainRouter.admin_db).create(
name=f"{user.email.split('@')[0]} default tenant"
)
with rls_transaction(str(tenant.id)):
Membership.objects.using(MainRouter.admin_db).create(
user=user, tenant=tenant, role=Membership.RoleChoices.OWNER
)
role = Role.objects.using(MainRouter.admin_db).create(
name="admin",
tenant_id=tenant.id,
manage_users=True,
manage_account=True,
manage_billing=True,
manage_providers=True,
manage_integrations=True,
manage_scans=True,
unlimited_visibility=True,
)
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
user=user,
invitation_token=invitation_token,
role=role,
tenant_id=tenant.id,
)
request.prowler_invitation_token = invitation_token
request.prowler_invitation_tenant_id = str(invitation.tenant_id)
else:
tenant = Tenant.objects.using(MainRouter.admin_db).create(
name=f"{user.email.split('@')[0]} default tenant"
)
with rls_transaction(str(tenant.id)):
Membership.objects.using(MainRouter.admin_db).create(
user=user, tenant=tenant, role=Membership.RoleChoices.OWNER
)
role = Role.objects.using(MainRouter.admin_db).create(
name="admin",
tenant_id=tenant.id,
manage_users=True,
manage_account=True,
manage_billing=True,
manage_providers=True,
manage_integrations=True,
manage_scans=True,
unlimited_visibility=True,
)
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
user=user,
role=role,
tenant_id=tenant.id,
)
else:
request.session["saml_user_created"] = str(user.id)
@@ -27,7 +27,6 @@ from django.conf import (
MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250)
TEMP_DB_PREFIX = "db-tmp-scan-"
DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
# Exceptions
@@ -45,10 +44,6 @@ class GraphDatabaseQueryException(Exception):
return self.message
class NeptuneWriteRetryExhaustedException(GraphDatabaseQueryException):
pass
class WriteQueryNotAllowedException(GraphDatabaseQueryException):
pass
+92 -358
View File
@@ -418,8 +418,7 @@ AWS_APPRUNNER_PRIVESC_UPDATE_SERVICE = AttackPathsQueryDefinition(
// Find existing App Runner services with roles attached (potential targets)
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'tasks.apprunner.amazonaws.com'}})
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -524,8 +523,7 @@ AWS_BEDROCK_PRIVESC_INVOKE_CODE_INTERPRETER = AttackPathsQueryDefinition(
// Find roles that trust the Bedrock AgentCore service (already attached to existing code interpreters)
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock-agentcore.amazonaws.com'}})
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -609,8 +607,7 @@ AWS_CLOUDFORMATION_PRIVESC_UPDATE_STACK = AttackPathsQueryDefinition(
// Find roles that trust CloudFormation service (already attached to existing stacks)
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}})
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -756,8 +753,7 @@ AWS_CLOUDFORMATION_PRIVESC_CHANGESET = AttackPathsQueryDefinition(
// Find roles that trust CloudFormation service (already attached to existing stacks)
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}})
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -848,8 +844,7 @@ AWS_CODEBUILD_PRIVESC_START_BUILD = AttackPathsQueryDefinition(
// Find roles that trust CodeBuild service (already attached to existing projects)
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}})
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -885,8 +880,7 @@ AWS_CODEBUILD_PRIVESC_START_BUILD_BATCH = AttackPathsQueryDefinition(
// Find roles that trust CodeBuild service (already attached to existing projects)
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}})
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -1102,8 +1096,7 @@ AWS_EC2_PRIVESC_MODIFY_INSTANCE_ATTRIBUTE = AttackPathsQueryDefinition(
// Find EC2 instances with instance profiles (potential targets)
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -1194,8 +1187,7 @@ AWS_EC2_PRIVESC_LAUNCH_TEMPLATE = AttackPathsQueryDefinition(
// Find launch templates in the account (potential targets)
MATCH path_target = (aws)--(template:LaunchTemplate)
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -1231,8 +1223,7 @@ AWS_EC2INSTANCECONNECT_PRIVESC_SEND_SSH_PUBLIC_KEY = AttackPathsQueryDefinition(
// Find EC2 instances with attached roles (targets for credential theft via IMDS)
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -1548,8 +1539,7 @@ AWS_ECS_PRIVESC_EXECUTE_COMMAND = AttackPathsQueryDefinition(
// Target: roles already attached to running tasks (trust ECS tasks service)
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ecs-tasks.amazonaws.com'}})
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -1632,8 +1622,7 @@ AWS_GLUE_PRIVESC_UPDATE_DEV_ENDPOINT = AttackPathsQueryDefinition(
// Find roles that trust Glue service (already attached to existing dev endpoints)
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'glue.amazonaws.com'}})
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -1927,20 +1916,12 @@ AWS_IAM_PRIVESC_CREATE_ACCESS_KEY = AttackPathsQueryDefinition(
OR act.value = '*'
WITH DISTINCT aws, principal, stmt, path_principal
// Pre-aggregate this statement's resource values into a list so the user
// match below is evaluated once per user (in-memory `any`) instead of
// building an (all-users x all-resource-items) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, path_principal, collect(DISTINCT res.value) AS res_values
WITH aws, path_principal, res_values, ('*' IN res_values) AS res_wildcard
// Find target users that the principal can create access keys for.
// Bind name/arn once so the `any` predicate reads locals.
// Find target users that the principal can create access keys for
MATCH path_target = (aws)--(target_user:AWSUser)
WITH path_principal, path_target, res_values, res_wildcard,
target_user.name AS uname, target_user.arn AS uarn
WHERE res_wildcard
OR size([rv IN res_values WHERE rv CONTAINS uname OR uarn CONTAINS rv]) > 0
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_user.name
OR target_user.arn CONTAINS res.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -1983,24 +1964,16 @@ AWS_IAM_PRIVESC_DELETE_CREATE_ACCESS_KEY = AttackPathsQueryDefinition(
OR act2.value = '*'
WITH DISTINCT aws, principal, stmt, stmt2, path_principal
// Pre-aggregate both statements' resource values into lists so the user
// match below is evaluated once per user (in-memory `any`) instead of
// building an (all-users x res x res2) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, stmt2, path_principal, collect(DISTINCT res.value) AS res_values
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WITH aws, path_principal, res_values, collect(DISTINCT res2.value) AS res2_values
WITH aws, path_principal, res_values, res2_values,
('*' IN res_values) AS res_wildcard,
('*' IN res2_values) AS res2_wildcard
// Find target users that the principal can rotate access keys for.
// Bind name/arn once so the `any` predicates read locals.
// Find target users that the principal can rotate access keys for
MATCH path_target = (aws)--(target_user:AWSUser)
WITH path_principal, path_target, res_values, res_wildcard, res2_values, res2_wildcard,
target_user.name AS uname, target_user.arn AS uarn
WHERE (res_wildcard OR size([rv IN res_values WHERE rv CONTAINS uname OR uarn CONTAINS rv]) > 0)
AND (res2_wildcard OR size([rv IN res2_values WHERE rv CONTAINS uname OR uarn CONTAINS rv]) > 0)
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_user.name
OR target_user.arn CONTAINS res.value
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WHERE res2.value = '*'
OR res2.value CONTAINS target_user.name
OR target_user.arn CONTAINS res2.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -2035,20 +2008,12 @@ AWS_IAM_PRIVESC_CREATE_LOGIN_PROFILE = AttackPathsQueryDefinition(
OR act.value = '*'
WITH DISTINCT aws, principal, stmt, path_principal
// Pre-aggregate this statement's resource values into a list so the user
// match below is evaluated once per user (in-memory `any`) instead of
// building an (all-users x all-resource-items) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, path_principal, collect(DISTINCT res.value) AS res_values
WITH aws, path_principal, res_values, ('*' IN res_values) AS res_wildcard
// Find target users that the principal can create login profiles for.
// Bind name/arn once so the `any` predicate reads locals.
// Find target users that the principal can create login profiles for
MATCH path_target = (aws)--(target_user:AWSUser)
WITH path_principal, path_target, res_values, res_wildcard,
target_user.name AS uname, target_user.arn AS uarn
WHERE res_wildcard
OR size([rv IN res_values WHERE rv CONTAINS uname OR uarn CONTAINS rv]) > 0
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_user.name
OR target_user.arn CONTAINS res.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -2121,20 +2086,12 @@ AWS_IAM_PRIVESC_UPDATE_LOGIN_PROFILE = AttackPathsQueryDefinition(
OR act.value = '*'
WITH DISTINCT aws, principal, stmt, path_principal
// Pre-aggregate this statement's resource values into a list so the user
// match below is evaluated once per user (in-memory `any`) instead of
// building an (all-users x all-resource-items) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, path_principal, collect(DISTINCT res.value) AS res_values
WITH aws, path_principal, res_values, ('*' IN res_values) AS res_wildcard
// Find target users that the principal can update login profiles for.
// Bind name/arn once so the `any` predicate reads locals.
// Find target users that the principal can update login profiles for
MATCH path_target = (aws)--(target_user:AWSUser)
WITH path_principal, path_target, res_values, res_wildcard,
target_user.name AS uname, target_user.arn AS uarn
WHERE res_wildcard
OR size([rv IN res_values WHERE rv CONTAINS uname OR uarn CONTAINS rv]) > 0
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_user.name
OR target_user.arn CONTAINS res.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -2365,21 +2322,12 @@ AWS_IAM_PRIVESC_UPDATE_ASSUME_ROLE_POLICY = AttackPathsQueryDefinition(
// Collapse the action-item fan-out: one row per (statement chain), not per matching action
WITH DISTINCT aws, stmt, path_principal
// Pre-aggregate this statement's resource values into a list so the role
// match below is evaluated once per role (in-memory `any`) instead of
// building an (all-roles x all-resource-items) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, path_principal, collect(DISTINCT res.value) AS res_values
WITH aws, path_principal, res_values, ('*' IN res_values) AS res_wildcard
// Find target roles whose trust policy this statement's resource can target.
// Bind the role's name/arn once so the `any` predicate reads them from a
// local variable instead of re-reading the property store per resource.
// Find target roles whose trust policy this statement's resource can target
MATCH path_target = (aws)--(target_role:AWSRole)
WITH path_principal, path_target, res_values, res_wildcard,
target_role.name AS rname, target_role.arn AS rarn
WHERE res_wildcard
OR size([rv IN res_values WHERE rv CONTAINS rname OR rarn CONTAINS rv]) > 0
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_role.name
OR target_role.arn CONTAINS res.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -2414,20 +2362,12 @@ AWS_IAM_PRIVESC_ADD_USER_TO_GROUP = AttackPathsQueryDefinition(
OR act.value = '*'
WITH DISTINCT aws, principal, stmt, path_principal
// Pre-aggregate this statement's resource values into a list so the group
// match below is evaluated once per group (in-memory `any`) instead of
// building an (all-groups x all-resource-items) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, path_principal, collect(DISTINCT res.value) AS res_values
WITH aws, path_principal, res_values, ('*' IN res_values) AS res_wildcard
// Find target groups the principal can add users to.
// Bind name/arn once so the `any` predicate reads locals.
// Find target groups the principal can add users to
MATCH path_target = (aws)--(target_group:AWSGroup)
WITH path_principal, path_target, res_values, res_wildcard,
target_group.name AS gname, target_group.arn AS garn
WHERE res_wildcard
OR size([rv IN res_values WHERE rv CONTAINS gname OR garn CONTAINS rv]) > 0
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_group.name
OR target_group.arn CONTAINS res.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -2511,24 +2451,16 @@ AWS_IAM_PRIVESC_ATTACH_USER_POLICY_CREATE_ACCESS_KEY = AttackPathsQueryDefinitio
OR act2.value = '*'
WITH DISTINCT aws, principal, stmt, stmt2, path_principal
// Pre-aggregate both statements' resource values into lists so the user
// match below is evaluated once per user (in-memory `any`) instead of
// building an (all-users x res x res2) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, stmt2, path_principal, collect(DISTINCT res.value) AS res_values
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WITH aws, path_principal, res_values, collect(DISTINCT res2.value) AS res2_values
WITH aws, path_principal, res_values, res2_values,
('*' IN res_values) AS res_wildcard,
('*' IN res2_values) AS res2_wildcard
// Find target users the principal can attach policies to and create keys for.
// Bind name/arn once so the `any` predicates read locals.
// Find target users the principal can attach policies to and create keys for
MATCH path_target = (aws)--(target_user:AWSUser)
WITH path_principal, path_target, res_values, res_wildcard, res2_values, res2_wildcard,
target_user.name AS uname, target_user.arn AS uarn
WHERE (res_wildcard OR size([rv IN res_values WHERE rv CONTAINS uname OR uarn CONTAINS rv]) > 0)
AND (res2_wildcard OR size([rv IN res2_values WHERE rv CONTAINS uname OR uarn CONTAINS rv]) > 0)
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_user.name
OR target_user.arn CONTAINS res.value
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WHERE res2.value = '*'
OR res2.value CONTAINS target_user.name
OR target_user.arn CONTAINS res2.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -2653,24 +2585,16 @@ AWS_IAM_PRIVESC_PUT_USER_POLICY_CREATE_ACCESS_KEY = AttackPathsQueryDefinition(
OR act2.value = '*'
WITH DISTINCT aws, principal, stmt, stmt2, path_principal
// Pre-aggregate both statements' resource values into lists so the user
// match below is evaluated once per user (in-memory `any`) instead of
// building an (all-users x res x res2) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, stmt2, path_principal, collect(DISTINCT res.value) AS res_values
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WITH aws, path_principal, res_values, collect(DISTINCT res2.value) AS res2_values
WITH aws, path_principal, res_values, res2_values,
('*' IN res_values) AS res_wildcard,
('*' IN res2_values) AS res2_wildcard
// Find target users the principal can put policies on and create keys for.
// Bind name/arn once so the `any` predicates read locals.
// Find target users the principal can put policies on and create keys for
MATCH path_target = (aws)--(target_user:AWSUser)
WITH path_principal, path_target, res_values, res_wildcard, res2_values, res2_wildcard,
target_user.name AS uname, target_user.arn AS uarn
WHERE (res_wildcard OR size([rv IN res_values WHERE rv CONTAINS uname OR uarn CONTAINS rv]) > 0)
AND (res2_wildcard OR size([rv IN res2_values WHERE rv CONTAINS uname OR uarn CONTAINS rv]) > 0)
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_user.name
OR target_user.arn CONTAINS res.value
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WHERE res2.value = '*'
OR res2.value CONTAINS target_user.name
OR target_user.arn CONTAINS res2.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -2712,24 +2636,16 @@ AWS_IAM_PRIVESC_ATTACH_ROLE_POLICY_UPDATE_ASSUME_ROLE = AttackPathsQueryDefiniti
OR act2.value = '*'
WITH DISTINCT aws, principal, stmt, stmt2, path_principal
// Pre-aggregate both statements' resource values into lists so the role
// match below is evaluated once per role (in-memory `any`) instead of
// building an (all-roles x res x res2) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, stmt2, path_principal, collect(DISTINCT res.value) AS res_values
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WITH aws, path_principal, res_values, collect(DISTINCT res2.value) AS res2_values
WITH aws, path_principal, res_values, res2_values,
('*' IN res_values) AS res_wildcard,
('*' IN res2_values) AS res2_wildcard
// Find target roles the principal can attach policies to and update trust
// policy for. Bind name/arn once so the `any` predicates read locals.
// Find target roles the principal can attach policies to and update trust policy for
MATCH path_target = (aws)--(target_role:AWSRole)
WITH path_principal, path_target, res_values, res_wildcard, res2_values, res2_wildcard,
target_role.name AS rname, target_role.arn AS rarn
WHERE (res_wildcard OR size([rv IN res_values WHERE rv CONTAINS rname OR rarn CONTAINS rv]) > 0)
AND (res2_wildcard OR size([rv IN res2_values WHERE rv CONTAINS rname OR rarn CONTAINS rv]) > 0)
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_role.name
OR target_role.arn CONTAINS res.value
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WHERE res2.value = '*'
OR res2.value CONTAINS target_role.name
OR target_role.arn CONTAINS res2.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -2771,31 +2687,17 @@ AWS_IAM_PRIVESC_CREATE_POLICY_VERSION_UPDATE_ASSUME_ROLE = AttackPathsQueryDefin
OR act2.value = '*'
WITH DISTINCT aws, principal, stmt, stmt2, path_principal
// Pre-aggregate both statements' resource values into lists so the role
// and policy matches below are evaluated with an in-memory `any` instead
// of building an (all-roles x res2) x (policies x res) cartesian product.
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WITH aws, stmt, path_principal, collect(DISTINCT res2.value) AS res2_values
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, path_principal, res2_values, collect(DISTINCT res.value) AS res_values
WITH aws, path_principal, res2_values, res_values,
('*' IN res2_values) AS res2_wildcard,
('*' IN res_values) AS res_wildcard
// Find target roles with customer-managed policies the principal can
// modify and update trust policy for. Bind name/arn once so the `any`
// predicates read locals instead of re-reading the property store.
// Find target roles with customer-managed policies the principal can modify and update trust policy for
MATCH path_target = (aws)--(target_role:AWSRole)
WITH path_principal, path_target, target_role, res_values, res_wildcard,
res2_values, res2_wildcard,
target_role.name AS rname, target_role.arn AS rarn
WHERE res2_wildcard
OR size([rv IN res2_values WHERE rv CONTAINS rname OR rarn CONTAINS rv]) > 0
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WHERE res2.value = '*'
OR res2.value CONTAINS target_role.name
OR target_role.arn CONTAINS res2.value
MATCH (target_role)-[:POLICY]->(target_policy:AWSPolicy)
WITH path_principal, path_target, res_values, res_wildcard,
target_policy.arn AS parn
WHERE parn CONTAINS $provider_uid
AND (res_wildcard OR size([rv IN res_values WHERE parn CONTAINS rv]) > 0)
WHERE target_policy.arn CONTAINS $provider_uid
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR target_policy.arn CONTAINS res.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -2837,24 +2739,16 @@ AWS_IAM_PRIVESC_PUT_ROLE_POLICY_UPDATE_ASSUME_ROLE = AttackPathsQueryDefinition(
OR act2.value = '*'
WITH DISTINCT aws, principal, stmt, stmt2, path_principal
// Pre-aggregate both statements' resource values into lists so the role
// match below is evaluated once per role (in-memory `any`) instead of
// building an (all-roles x res x res2) cartesian product.
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WITH aws, stmt2, path_principal, collect(DISTINCT res.value) AS res_values
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WITH aws, path_principal, res_values, collect(DISTINCT res2.value) AS res2_values
WITH aws, path_principal, res_values, res2_values,
('*' IN res_values) AS res_wildcard,
('*' IN res2_values) AS res2_wildcard
// Find target roles the principal can put inline policies on and update
// trust policy for. Bind name/arn once so the `any` predicates read locals.
// Find target roles the principal can put inline policies on and update trust policy for
MATCH path_target = (aws)--(target_role:AWSRole)
WITH path_principal, path_target, res_values, res_wildcard, res2_values, res2_wildcard,
target_role.name AS rname, target_role.arn AS rarn
WHERE (res_wildcard OR size([rv IN res_values WHERE rv CONTAINS rname OR rarn CONTAINS rv]) > 0)
AND (res2_wildcard OR size([rv IN res2_values WHERE rv CONTAINS rname OR rarn CONTAINS rv]) > 0)
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value CONTAINS target_role.name
OR target_role.arn CONTAINS res.value
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WHERE res2.value = '*'
OR res2.value CONTAINS target_role.name
OR target_role.arn CONTAINS res2.value
WITH DISTINCT path_principal, path_target
WITH collect(path_principal) + collect(path_target) AS paths
@@ -3443,8 +3337,7 @@ AWS_SSM_PRIVESC_START_SESSION = AttackPathsQueryDefinition(
// Find EC2 instances with attached roles (targets for credential theft via IMDS)
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -3480,8 +3373,7 @@ AWS_SSM_PRIVESC_SEND_COMMAND = AttackPathsQueryDefinition(
// Find EC2 instances with attached roles (targets for credential theft via IMDS)
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
WITH principal_paths + target_paths AS paths
WITH principal_paths + collect(DISTINCT path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
@@ -3536,160 +3428,6 @@ AWS_STS_PRIVESC_ASSUME_ROLE = AttackPathsQueryDefinition(
parameters=[],
)
# STS-002
AWS_STS_PRIVESC_CROSS_ACCOUNT_TRUST = AttackPathsQueryDefinition(
id="aws-sts-privesc-cross-account-trust",
name="Cross-Account Role Trust for Privilege Escalation (STS-002)",
short_description="Roles that trust an external account's root principal can be assumed by any principal in that account, enabling confused-deputy escalation.",
description="Detect IAM roles whose trust policy allows an external AWS account root principal (arn:aws:iam::<account-id>:root) to assume them. Any principal in the trusted external account that holds sts:AssumeRole can assume the role and gain its permissions, which is the confused-deputy escalation surface. The ingested graph does not record trust-policy conditions, so roles protected by an sts:ExternalId condition cannot be filtered out automatically and are surfaced here for manual review.",
attribution=AttackPathsQueryAttribution(
text="pathfinding.cloud - STS-002 - sts:AssumeRole",
link="https://pathfinding.cloud/paths/sts-002",
),
provider="aws",
cypher=f"""
// Find roles that trust an external account's root principal (cross-account trust)
MATCH path_target = (aws:AWSAccount {{id: $provider_uid}})--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(trusted:AWSRootPrincipal)
WHERE trusted.arn CONTAINS ':root'
AND NOT trusted.arn CONTAINS aws.id
WITH DISTINCT path_target
WITH collect(path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
WITH paths, collect(DISTINCT n) AS unique_nodes
UNWIND unique_nodes AS n
OPTIONAL MATCH (n)-[pfr:HAS_FINDING]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}})
RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
)
# STS-003
AWS_STS_PRIVESC_WILDCARD_TRUST = AttackPathsQueryDefinition(
id="aws-sts-privesc-wildcard-trust",
name="Potential Wildcard Role Trust (STS-003)",
short_description="Potential wildcard role trusts that need manual review before they are treated as assumable.",
description='Find IAM roles linked to a wildcard principal ("AWS": "*"). The ingested graph does not preserve trust-policy Effect or Condition fields, so a match can come from a Deny statement or a restricted Allow statement. Treat each result as a candidate for manual review, not as a confirmed assumable role.',
attribution=AttackPathsQueryAttribution(
text="pathfinding.cloud - STS-003 - sts:AssumeRole",
link="https://pathfinding.cloud/paths/sts-003",
),
provider="aws",
cypher=f"""
// Find roles linked to a wildcard principal for manual review
MATCH path_target = (aws:AWSAccount {{id: $provider_uid}})--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(trusted:AWSPrincipal)
WHERE trusted.arn = '*'
WITH DISTINCT path_target
WITH collect(path_target) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
WITH paths, collect(DISTINCT n) AS unique_nodes
UNWIND unique_nodes AS n
OPTIONAL MATCH (n)-[pfr:HAS_FINDING]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}})
RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
)
# IAM-022
AWS_IAM_PRIVESC_DELETE_USER_PERMISSIONS_BOUNDARY = AttackPathsQueryDefinition(
id="aws-iam-privesc-delete-user-permissions-boundary",
name="Permissions Boundary Removal for Self-Escalation (IAM-022)",
short_description="IAM users that can remove their own permissions boundary, if one is attached.",
description="Find IAM users whose policies allow iam:DeleteUserPermissionsBoundary on their own user ARN. The graph does not record whether a boundary is attached or whether removing it grants more access, so each result needs manual review.",
attribution=AttackPathsQueryAttribution(
text="pathfinding.cloud - IAM-022 - iam:DeleteUserPermissionsBoundary",
link="https://pathfinding.cloud/paths/iam-022",
),
provider="aws",
cypher=f"""
// Find IAM users with iam:DeleteUserPermissionsBoundary permission
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSUser)-[:POLICY]->(policy:AWSPolicy)-[:STATEMENT]->(stmt:AWSPolicyStatement {{effect: 'Allow'}})
MATCH (stmt)-[:HAS_ACTION]->(act:AWSPolicyStatementActionItem)
WHERE toLower(act.value) IN ['iam:*', 'iam:deleteuserpermissionsboundary']
OR act.value = '*'
WITH DISTINCT principal, stmt, path_principal
// Keep only users that can remove the boundary from their own user ARN
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
OR res.value = principal.arn
OR (res.value ENDS WITH '*' AND principal.arn STARTS WITH replace(res.value, '*', ''))
WITH DISTINCT path_principal
WITH collect(path_principal) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
WITH paths, collect(DISTINCT n) AS unique_nodes
UNWIND unique_nodes AS n
OPTIONAL MATCH (n)-[pfr:HAS_FINDING]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}})
RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
)
# SSO-001
AWS_SSO_PRIVESC_PERMISSION_SET_ESCALATION = AttackPathsQueryDefinition(
id="aws-sso-privesc-permission-set-escalation",
name="Identity Center Permission Set Escalation (SSO-001)",
short_description="Create an administrative Identity Center permission set and assign it to gain organization-wide admin access.",
description="Detect principals that hold sso:CreatePermissionSet, sso:AttachManagedPolicyToPermissionSet, and sso:CreateAccountAssignment together. With all three, a principal can create a new IAM Identity Center permission set, attach the AdministratorAccess managed policy to it, and assign it to their own user or group for any account in the organization, gaining administrative access across the organization through the Identity Center portal.",
attribution=AttackPathsQueryAttribution(
text="pathfinding.cloud - SSO-001 - sso:CreatePermissionSet + sso:AttachManagedPolicyToPermissionSet + sso:CreateAccountAssignment",
link="https://pathfinding.cloud/paths/sso-001",
),
provider="aws",
cypher=f"""
// Find principals with sso:CreatePermissionSet permission
MATCH path_principal = (aws:AWSAccount {{id: $provider_uid}})--(principal:AWSPrincipal)-[:POLICY]->(policy:AWSPolicy)-[:STATEMENT]->(stmt:AWSPolicyStatement {{effect: 'Allow'}})
MATCH (stmt)-[:HAS_ACTION]->(act:AWSPolicyStatementActionItem)
WHERE toLower(act.value) IN ['sso:*', 'sso:createpermissionset']
OR act.value = '*'
MATCH (stmt)-[:HAS_RESOURCE]->(res:AWSPolicyStatementResourceItem)
WHERE res.value = '*'
WITH DISTINCT aws, principal, path_principal
// Find sso:AttachManagedPolicyToPermissionSet permission on the same principal
MATCH (principal)-[:POLICY]->(:AWSPolicy)-[:STATEMENT]->(stmt2:AWSPolicyStatement {{effect: 'Allow'}})-[:HAS_ACTION]->(act2:AWSPolicyStatementActionItem)
WHERE toLower(act2.value) IN ['sso:*', 'sso:attachmanagedpolicytopermissionset']
OR act2.value = '*'
MATCH (stmt2)-[:HAS_RESOURCE]->(res2:AWSPolicyStatementResourceItem)
WHERE res2.value = '*'
WITH DISTINCT principal, path_principal
// Find sso:CreateAccountAssignment permission on the same principal
MATCH (principal)-[:POLICY]->(:AWSPolicy)-[:STATEMENT]->(stmt3:AWSPolicyStatement {{effect: 'Allow'}})-[:HAS_ACTION]->(act3:AWSPolicyStatementActionItem)
WHERE toLower(act3.value) IN ['sso:*', 'sso:createaccountassignment']
OR act3.value = '*'
MATCH (stmt3)-[:HAS_RESOURCE]->(res3:AWSPolicyStatementResourceItem)
WHERE res3.value = '*'
WITH DISTINCT path_principal
WITH collect(path_principal) AS paths
UNWIND paths AS p
UNWIND nodes(p) AS n
WITH paths, collect(DISTINCT n) AS unique_nodes
UNWIND unique_nodes AS n
OPTIONAL MATCH (n)-[pfr:HAS_FINDING]-(pf:{PROWLER_FINDING_LABEL} {{status: 'FAIL'}})
RETURN paths, collect(DISTINCT pf) as dpf, collect(DISTINCT pfr) as dpfr
""",
parameters=[],
)
# AWS Queries List
AWS_QUERIES: list[AttackPathsQueryDefinition] = [
@@ -3771,8 +3509,4 @@ AWS_QUERIES: list[AttackPathsQueryDefinition] = [
AWS_SSM_PRIVESC_START_SESSION,
AWS_SSM_PRIVESC_SEND_COMMAND,
AWS_STS_PRIVESC_ASSUME_ROLE,
AWS_STS_PRIVESC_CROSS_ACCOUNT_TRUST,
AWS_STS_PRIVESC_WILDCARD_TRUST,
AWS_IAM_PRIVESC_DELETE_USER_PERMISSIONS_BOUNDARY,
AWS_SSO_PRIVESC_PERMISSION_SET_ESCALATION,
]
@@ -1,6 +1,4 @@
import logging
import random
import time
from collections.abc import Callable
from typing import Any
@@ -10,44 +8,18 @@ import neo4j.exceptions
logger = logging.getLogger(__name__)
class RetryExhaustedError(Exception):
def __init__(
self,
*,
retry_context: str,
method_name: str,
attempts: int,
elapsed_seconds: float,
last_error: Exception,
) -> None:
self.retry_context = retry_context
self.method_name = method_name
self.attempts = attempts
self.elapsed_seconds = elapsed_seconds
self.last_error = last_error
last_message = getattr(last_error, "message", None) or str(last_error)
super().__init__(
f"{retry_context} {method_name} failed after {attempts} attempts over "
f"{elapsed_seconds:.3f}s. Last error: {last_message}"
)
class RetryableSession:
"""Wrapper around ``neo4j.Session`` with a refreshable retry policy."""
"""
Wrapper around `neo4j.Session` that retries `neo4j.exceptions.ServiceUnavailable` errors.
"""
def __init__(
self,
session_factory: Callable[[], neo4j.Session],
max_retries: int,
retry_if: Callable[[Exception], bool] | None = None,
initial_retry_delay_seconds: float = 0,
retry_context: str | None = None,
) -> None:
self._session_factory = session_factory
self._max_retries = max(0, max_retries)
self._retry_if = retry_if
self._initial_retry_delay_seconds = max(0.0, initial_retry_delay_seconds)
self._retry_context = retry_context
self._session = self._session_factory()
def close(self) -> None:
@@ -78,75 +50,30 @@ class RetryableSession:
def _call_with_retry(self, method_name: str, *args: Any, **kwargs: Any) -> Any:
attempt = 0
last_exc: Exception | None = None
started_at = time.monotonic()
while attempt <= self._max_retries:
try:
method = getattr(self._session, method_name)
return method(*args, **kwargs)
except Exception as exc:
if not self._should_retry(exc):
raise
except (
BrokenPipeError,
ConnectionResetError,
neo4j.exceptions.ServiceUnavailable,
) as exc: # pragma: no cover - depends on infra
last_exc = exc
attempt += 1
if attempt > self._max_retries:
if self._retry_context is not None:
raise RetryExhaustedError(
retry_context=self._retry_context,
method_name=method_name,
attempts=attempt,
elapsed_seconds=time.monotonic() - started_at,
last_error=exc,
) from exc
raise
delay = self._retry_delay(attempt)
if self._retry_context is not None:
error_message = getattr(exc, "message", None) or str(exc)
logger.warning(
"%s %s failed with %s: %s; retry %s/%s in %.3fs",
self._retry_context,
method_name,
type(exc).__name__,
error_message,
attempt,
self._max_retries,
delay,
)
else:
logger.warning(
"Graph session %s failed with %s; retry %s/%s in %.3fs",
method_name,
type(exc).__name__,
attempt,
self._max_retries,
delay,
)
logger.warning(
f"Neo4j session {method_name} failed with {type(exc).__name__} ({attempt}/{self._max_retries} attempts). Retrying..."
)
self._refresh_session()
if delay:
time.sleep(delay)
raise last_exc if last_exc else RuntimeError("Unexpected retry loop exit")
def _should_retry(self, exc: Exception) -> bool:
if isinstance(
exc,
(
BrokenPipeError,
ConnectionResetError,
neo4j.exceptions.ServiceUnavailable,
),
):
return True
return self._retry_if(exc) if self._retry_if else False
def _retry_delay(self, attempt: int) -> float:
max_delay = self._initial_retry_delay_seconds * (2**attempt)
return random.uniform(max_delay / 2, max_delay) if max_delay else 0
def _refresh_session(self) -> None:
if self._session is not None:
try:
@@ -15,8 +15,6 @@ class SinkDatabase(Protocol):
has a single graph, and isolation is label-based).
"""
sync_batch_size: int
def init(self) -> None: ...
def close(self) -> None: ...
@@ -1,81 +0,0 @@
"""Shared batched deletion helpers for sink backends."""
import logging
import time
from typing import Any
RELATIONSHIP_DELETE_QUERY_TEMPLATES = {
"outgoing relationship": """
MATCH (n:`{provider_label}`)-[r]->()
WITH r LIMIT $batch_size
DELETE r
RETURN COUNT(r) AS deleted_rels_count
""",
"incoming relationship": """
MATCH (n:`{provider_label}`)<-[r]-()
WITH r LIMIT $batch_size
DELETE r
RETURN COUNT(r) AS deleted_rels_count
""",
}
NODE_DELETE_QUERY_TEMPLATE = """
MATCH (n:{provider_resource_label}:`{provider_label}`)
WITH n LIMIT $batch_size
DELETE n
RETURN COUNT(n) AS deleted_nodes_count
"""
def delete_batches(
*,
session: Any,
logger: logging.Logger,
log_target: str,
provider_id: str,
query: str,
phase: str,
count_key: str,
total_key: str,
deleted_key: str,
initial_total: int,
batch_size: int,
drop_t0: float,
) -> tuple[int, int]:
def delete_batch(tx: Any) -> int:
record = tx.run(query, {"batch_size": batch_size}).single()
return (record[count_key] if record else 0) or 0
deleted_total = initial_total
batches = 0
while True:
logger.info(
"Deleting %s batch from %s "
"(provider=%s, batch=%s, total_%s=%s, elapsed=%.3fs)",
phase,
log_target,
provider_id,
batches + 1,
total_key,
deleted_total,
time.perf_counter() - drop_t0,
)
deleted = session.execute_write(delete_batch)
if deleted == 0:
return deleted_total, batches
batches += 1
deleted_total += deleted
logger.info(
"Deleted %s batch from %s "
"(provider=%s, batch=%s, %s=%s, total_%s=%s, elapsed=%.3fs)",
phase,
log_target,
provider_id,
batches,
deleted_key,
deleted,
total_key,
deleted_total,
time.perf_counter() - drop_t0,
)
+84 -49
View File
@@ -17,11 +17,6 @@ import neo4j
import neo4j.exceptions
from api.attack_paths.retryable_session import RetryableSession
from api.attack_paths.sink.base import SinkDatabase
from api.attack_paths.sink.drop import (
NODE_DELETE_QUERY_TEMPLATE,
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
delete_batches,
)
from config.env import env
from django.conf import settings
@@ -54,8 +49,6 @@ DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
class Neo4jSink(SinkDatabase):
"""Neo4j-backed sink. Multi-database cluster; tenant isolation is physical."""
sync_batch_size = env.int("ATTACK_PATHS_NEO4J_SYNC_BATCH_SIZE", default=1000)
def __init__(self) -> None:
self._driver: neo4j.Driver | None = None
self._lock = threading.Lock()
@@ -205,14 +198,16 @@ class Neo4jSink(SinkDatabase):
"""
from api.attack_paths.database import GraphDatabaseQueryException
from tasks.jobs.attack_paths.config import (
GRAPH_MUTATION_BATCH_SIZE,
BATCH_SIZE,
PROVIDER_RESOURCE_LABEL,
get_provider_label,
)
provider_label = get_provider_label(provider_id)
deleted_nodes = deleted_relationships = 0
relationship_batches = node_batches = 0
deleted_nodes = 0
deleted_relationships = 0
relationship_batches = 0
node_batches = 0
drop_t0 = time.perf_counter()
logger.info(
@@ -237,44 +232,84 @@ class Neo4jSink(SinkDatabase):
database,
provider_id,
)
log_target = f"Neo4j sink database {database}"
for (
phase,
query_template,
) in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
deleted_relationships, phase_batches = delete_batches(
session=session,
logger=logger,
log_target=log_target,
provider_id=provider_id,
query=query_template.format(provider_label=provider_label),
phase=phase,
count_key="deleted_rels_count",
total_key="rels",
deleted_key="deleted_rels",
initial_total=deleted_relationships,
batch_size=GRAPH_MUTATION_BATCH_SIZE,
drop_t0=drop_t0,
# Phase 1: delete relationships incident to provider nodes in
# batches. The undirected pattern matches an edge between two
# provider nodes from both ends, so `DISTINCT r` dedupes it to
# delete a full batch of unique relationships each round.
deleted_count = 1
while deleted_count > 0:
next_batch = relationship_batches + 1
logger.info(
"Deleting relationship batch from Neo4j sink database %s "
"(provider=%s, batch=%s, total_rels=%s, elapsed=%.3fs)",
database,
provider_id,
next_batch,
deleted_relationships,
time.perf_counter() - drop_t0,
)
relationship_batches += phase_batches
result = session.run(
f"""
MATCH (:`{provider_label}`)-[r]-()
WITH DISTINCT r LIMIT $batch_size
DELETE r
RETURN COUNT(r) AS deleted_rels_count
""",
{"batch_size": BATCH_SIZE},
)
deleted_count = result.single().get("deleted_rels_count", 0)
if deleted_count > 0:
relationship_batches += 1
deleted_relationships += deleted_count
logger.info(
"Deleted relationship batch from Neo4j sink database %s "
"(provider=%s, batch=%s, deleted_rels=%s, "
"total_rels=%s, elapsed=%.3fs)",
database,
provider_id,
relationship_batches,
deleted_count,
deleted_relationships,
time.perf_counter() - drop_t0,
)
deleted_nodes, node_batches = delete_batches(
session=session,
logger=logger,
log_target=log_target,
provider_id=provider_id,
query=NODE_DELETE_QUERY_TEMPLATE.format(
provider_label=provider_label,
provider_resource_label=PROVIDER_RESOURCE_LABEL,
),
phase="node",
count_key="deleted_nodes_count",
total_key="nodes",
deleted_key="deleted_nodes",
initial_total=0,
batch_size=GRAPH_MUTATION_BATCH_SIZE,
drop_t0=drop_t0,
)
# Phase 2: delete the now relationship-free nodes in batches.
deleted_count = 1
while deleted_count > 0:
next_batch = node_batches + 1
logger.info(
"Deleting node batch from Neo4j sink database %s "
"(provider=%s, batch=%s, total_nodes=%s, elapsed=%.3fs)",
database,
provider_id,
next_batch,
deleted_nodes,
time.perf_counter() - drop_t0,
)
result = session.run(
f"""
MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`)
WITH n LIMIT $batch_size
DELETE n
RETURN COUNT(n) AS deleted_nodes_count
""",
{"batch_size": BATCH_SIZE},
)
deleted_count = result.single().get("deleted_nodes_count", 0)
if deleted_count > 0:
node_batches += 1
deleted_nodes += deleted_count
logger.info(
"Deleted node batch from Neo4j sink database %s "
"(provider=%s, batch=%s, deleted_nodes=%s, "
"total_nodes=%s, elapsed=%.3fs)",
database,
provider_id,
node_batches,
deleted_count,
deleted_nodes,
time.perf_counter() - drop_t0,
)
except GraphDatabaseQueryException as exc:
if exc.code == DATABASE_NOT_FOUND_CODE:
@@ -357,7 +392,7 @@ class Neo4jSink(SinkDatabase):
f"ON (n.`{PROVIDER_ELEMENT_ID_PROPERTY}`)"
)
with self.get_session(database) as session:
session.execute_write(lambda tx: tx.run(query).consume())
session.run(query).consume()
def write_nodes(
self,
@@ -379,7 +414,7 @@ class Neo4jSink(SinkDatabase):
SET n += row.props
"""
with self.get_session(database) as session:
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
session.run(query, {"rows": rows}).consume()
def write_relationships(
self,
@@ -405,7 +440,7 @@ class Neo4jSink(SinkDatabase):
SET r += row.props
"""
with self.get_session(database) as session:
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
session.run(query, {"rows": rows}).consume()
# For compatibility with test harnesses that patch the concrete driver
def get_driver(self) -> neo4j.Driver:
@@ -25,13 +25,8 @@ from urllib.parse import urlsplit
import neo4j
import neo4j.exceptions
from api.attack_paths.retryable_session import RetryableSession, RetryExhaustedError
from api.attack_paths.retryable_session import RetryableSession
from api.attack_paths.sink.base import SinkDatabase
from api.attack_paths.sink.drop import (
NODE_DELETE_QUERY_TEMPLATE,
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
delete_batches,
)
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.session import Session as BotoSession
@@ -59,34 +54,20 @@ CONNECTION_TIMEOUT = env.int("NEPTUNE_CONNECTION_TIMEOUT", default=10)
# Roll connections hourly so SigV4 rotations and cert refreshes don't strand long-lived pool entries
MAX_CONNECTION_LIFETIME = env.int("NEPTUNE_MAX_CONNECTION_LIFETIME", default=3600)
MAX_CONNECTION_POOL_SIZE = env.int("NEPTUNE_MAX_CONNECTION_POOL_SIZE", default=50)
NEPTUNE_WRITE_RETRY_DELAY_SECONDS = 2
READ_EXCEPTION_CODES = [
"Neo.ClientError.Statement.AccessMode",
"Neo.ClientError.Procedure.ProcedureNotFound",
]
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
RETRYABLE_WRITE_ERROR_FRAGMENTS = (
"Operation failed due to conflicting concurrent operations",
"Operation terminated (deadline exceeded)",
)
# Refresh 60s before the 5-minute SigV4 window closes
SIGV4_TOKEN_LIFETIME_MINUTES = 4
def _is_retryable_write_error(exc: Exception) -> bool:
if not isinstance(exc, neo4j.exceptions.Neo4jError):
return False
message = exc.message or ""
return any(fragment in message for fragment in RETRYABLE_WRITE_ERROR_FRAGMENTS)
class NeptuneSink(SinkDatabase):
"""Neptune-backed sink. Single database; isolation is label-based."""
sync_batch_size = env.int("ATTACK_PATHS_NEPTUNE_SYNC_BATCH_SIZE", default=500)
def __init__(self) -> None:
self._writer: neo4j.Driver | None = None
self._reader: neo4j.Driver | None = None
@@ -208,7 +189,6 @@ class NeptuneSink(SinkDatabase):
from api.attack_paths.database import (
ClientStatementException,
GraphDatabaseQueryException,
NeptuneWriteRetryExhaustedException,
WriteQueryNotAllowedException,
)
@@ -220,27 +200,14 @@ class NeptuneSink(SinkDatabase):
session_wrapper: RetryableSession | None = None
try:
is_write_session = default_access_mode != neo4j.READ_ACCESS
session_wrapper = RetryableSession(
session_factory=lambda: driver.session(
default_access_mode=default_access_mode
),
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
retry_if=_is_retryable_write_error if is_write_session else None,
initial_retry_delay_seconds=(
NEPTUNE_WRITE_RETRY_DELAY_SECONDS if is_write_session else 0
),
retry_context="Neptune write" if is_write_session else None,
)
yield session_wrapper
except RetryExhaustedError as exc:
last_error = exc.last_error
raise NeptuneWriteRetryExhaustedException(
message=str(exc),
code=getattr(last_error, "code", None),
) from last_error
except neo4j.exceptions.Neo4jError as exc:
if (
default_access_mode == neo4j.READ_ACCESS
@@ -302,7 +269,7 @@ class NeptuneSink(SinkDatabase):
graph's branching factor.
"""
from tasks.jobs.attack_paths.config import (
GRAPH_MUTATION_BATCH_SIZE,
BATCH_SIZE,
PROVIDER_RESOURCE_LABEL,
get_provider_label,
)
@@ -329,40 +296,78 @@ class NeptuneSink(SinkDatabase):
"Opened Neptune writer session for provider graph drop (provider=%s)",
provider_id,
)
for phase, query_template in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
deleted_relationships, phase_batches = delete_batches(
session=session,
logger=logger,
log_target="Neptune sink",
provider_id=provider_id,
query=query_template.format(provider_label=provider_label),
phase=phase,
count_key="deleted_rels_count",
total_key="rels",
deleted_key="deleted_rels",
initial_total=deleted_relationships,
batch_size=GRAPH_MUTATION_BATCH_SIZE,
drop_t0=drop_t0,
while True:
next_batch = relationship_batches + 1
logger.info(
"Deleting relationship batch from Neptune sink "
"(provider=%s, batch=%s, total_rels=%s, elapsed=%.3fs)",
provider_id,
next_batch,
deleted_relationships,
time.perf_counter() - drop_t0,
)
result = session.run(
f"""
MATCH (:`{provider_label}`)-[r]-()
WITH DISTINCT r LIMIT $batch_size
DELETE r
RETURN COUNT(r) AS deleted_rels_count
""",
{"batch_size": BATCH_SIZE},
)
record = result.single()
deleted_rels = (record["deleted_rels_count"] if record else 0) or 0
if deleted_rels == 0:
break
relationship_batches += 1
deleted_relationships += deleted_rels
logger.info(
"Deleted relationship batch from Neptune sink "
"(provider=%s, batch=%s, deleted_rels=%s, total_rels=%s, "
"elapsed=%.3fs)",
provider_id,
relationship_batches,
deleted_rels,
deleted_relationships,
time.perf_counter() - drop_t0,
)
relationship_batches += phase_batches
deleted_nodes, node_batches = delete_batches(
session=session,
logger=logger,
log_target="Neptune sink",
provider_id=provider_id,
query=NODE_DELETE_QUERY_TEMPLATE.format(
provider_label=provider_label,
provider_resource_label=PROVIDER_RESOURCE_LABEL,
),
phase="node",
count_key="deleted_nodes_count",
total_key="nodes",
deleted_key="deleted_nodes",
initial_total=0,
batch_size=GRAPH_MUTATION_BATCH_SIZE,
drop_t0=drop_t0,
)
deleted_nodes = 0
while True:
next_batch = node_batches + 1
logger.info(
"Deleting node batch from Neptune sink "
"(provider=%s, batch=%s, total_nodes=%s, elapsed=%.3fs)",
provider_id,
next_batch,
deleted_nodes,
time.perf_counter() - drop_t0,
)
result = session.run(
f"""
MATCH (n:`{PROVIDER_RESOURCE_LABEL}`:`{provider_label}`)
WITH n LIMIT $batch_size
DELETE n
RETURN COUNT(n) AS deleted_nodes_count
""",
{"batch_size": BATCH_SIZE},
)
record = result.single()
deleted = (record["deleted_nodes_count"] if record else 0) or 0
if deleted == 0:
break
node_batches += 1
deleted_nodes += deleted
logger.info(
"Deleted node batch from Neptune sink "
"(provider=%s, batch=%s, deleted_nodes=%s, total_nodes=%s, "
"elapsed=%.3fs)",
provider_id,
node_batches,
deleted,
deleted_nodes,
time.perf_counter() - drop_t0,
)
logger.info(
"Finished dropping provider graph from Neptune sink "
@@ -433,7 +438,7 @@ class NeptuneSink(SinkDatabase):
SET n.`{PROVIDER_ELEMENT_ID_PROPERTY}` = row.provider_element_id
"""
with self.get_session() as session:
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
session.run(query, {"rows": rows}).consume()
def write_relationships(
self,
@@ -457,7 +462,7 @@ class NeptuneSink(SinkDatabase):
SET r += row.props
"""
with self.get_session() as session:
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
session.run(query, {"rows": rows}).consume()
# Test helpers
@@ -115,26 +115,7 @@ def execute_query(
# TODO: drop after Neptune cutover
# Route reads by the scan row's recorded sink, not by current settings.
backend = sink_module.get_backend_for_scan(scan)
cypher = definition.cypher
# Every synced node carries a `_Provider_{uuid}` isolation label (the
# sync labels the whole provider subgraph). Injecting it into the
# predefined query's node patterns gives the planner a selective label
# index to seed from instead of a global label scan (`:AWSRole` across
# every tenant), which on Neptune is the difference between a sub-second
# plan and a query that times out. The custom-query path relies on this
# same injection.
#
# Restrict it to migrated scans: that catalog runs on the Neptune sink
# where the plan blowup happens, while the pre-cutover legacy catalog
# runs on the old sink and is dropped after the cutover, so leave it
# byte-for-byte unchanged. This only affects the query plan, not
# isolation - `_serialize_graph` already label-filters both catalogs.
# TODO: drop the is_migrated guard after Neptune cutover
if scan.is_migrated:
cypher = inject_provider_label(cypher, provider_id)
graph = backend.execute_read_query(database_name, cypher, parameters)
graph = backend.execute_read_query(database_name, definition.cypher, parameters)
return _serialize_graph(graph, provider_id)
except graph_database.WriteQueryNotAllowedException:
@@ -171,10 +152,10 @@ def execute_custom_query(
scan: AttackPathsScan,
) -> dict[str, Any]:
# Defense-in-depth for custom queries:
# 1. `neo4j.READ_ACCESS` - prevents mutations at the driver level
# 2. `inject_provider_label()` - regex-based label injection scopes node patterns
# 3. `_serialize_graph()` - post-query filter drops nodes without the provider label
# 4. `USING QUERY:TIMEOUTMILLISECONDS` on Neptune - server-side runaway cutoff
# 1. `neo4j.READ_ACCESS` prevents mutations at the driver level
# 2. `inject_provider_label()` regex-based label injection scopes node patterns
# 3. `_serialize_graph()` post-query filter drops nodes without the provider label
# 4. `USING QUERY:TIMEOUTMILLISECONDS` on Neptune server-side runaway cutoff
#
# Layer 2 is best-effort (regex can't fully parse Cypher);
# layer 3 is the safety net that guarantees provider isolation.
+30 -63
View File
@@ -1,4 +1,3 @@
import logging
from math import isfinite
from uuid import UUID
@@ -6,7 +5,6 @@ from api.db_router import MainRouter
from api.models import TenantAPIKey, TenantAPIKeyManager
from cryptography.fernet import InvalidToken
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
from django.utils import timezone
from drf_simple_apikey.backends import APIKeyAuthentication as BaseAPIKeyAuth
from drf_simple_apikey.crypto import get_crypto
@@ -16,16 +14,6 @@ from rest_framework.exceptions import AuthenticationFailed
from rest_framework.request import Request
from rest_framework_simplejwt.authentication import JWTAuthentication
logger = logging.getLogger(__name__)
class OrphanedAPIKeyError(Exception):
"""Raised when an API key outlived the user that owns it.
Handled by `authenticate`, which commits the revocation written while detecting it
and then rejects the request with `AuthenticationFailed`.
"""
class TenantAPIKeyAuthentication(BaseAPIKeyAuth):
model = TenantAPIKey
@@ -36,13 +24,10 @@ class TenantAPIKeyAuthentication(BaseAPIKeyAuth):
def _authenticate_credentials(self, request, key):
"""
Override to use admin connection, bypassing RLS during authentication.
Returns the validated API key row, locked with `select_for_update`, so callers
must run inside `transaction.atomic(using=MainRouter.admin_db)`.
"""
try:
payload = self.key_crypto.decrypt(key)
except (ValueError, InvalidToken):
except ValueError:
raise AuthenticationFailed("Invalid API Key.")
if not isinstance(payload, dict):
@@ -67,33 +52,13 @@ class TenantAPIKeyAuthentication(BaseAPIKeyAuth):
raise AuthenticationFailed("API Key has already expired.")
try:
api_key = (
self.model.objects.using(MainRouter.admin_db)
.select_for_update()
.get(id=api_key_pk)
)
api_key = self.model.objects.using(MainRouter.admin_db).get(id=api_key_pk)
except ObjectDoesNotExist:
raise AuthenticationFailed("No entity matching this api key.")
if api_key.revoked:
raise AuthenticationFailed("This API Key has been revoked.")
# `entity` is nullable and `on_delete=SET_NULL` leaves the key behind when its
# owner is deleted, so a key can outlive its user. Reject it here: further down
# the authentication would return `None` as the authenticated user, which blows
# up while building the auth dict and surfaces as a 500 instead of a 401.
# Revoke it as well, so it stops showing up as active and later attempts fail
# the `revoked` check above like any other revoked key.
if api_key.entity_id is None:
api_key.revoked = True
api_key.save(update_fields=["revoked"], using=MainRouter.admin_db)
logger.warning(
"Revoked orphaned API key: prefix=%s tenant=%s",
api_key.prefix,
api_key.tenant_id,
)
raise OrphanedAPIKeyError
client_ip = request.META.get(package_settings.IP_ADDRESS_HEADER)
if api_key.blacklisted_ips and client_ip in api_key.blacklisted_ips:
raise AuthenticationFailed("Access denied from blacklisted IP.")
@@ -101,7 +66,7 @@ class TenantAPIKeyAuthentication(BaseAPIKeyAuth):
if api_key.whitelisted_ips and client_ip not in api_key.whitelisted_ips:
raise AuthenticationFailed("Access restricted to specific IP addresses.")
return api_key
return api_key.entity, key
def authenticate(self, request: Request):
prefixed_key = self.get_key(request)
@@ -112,34 +77,36 @@ class TenantAPIKeyAuthentication(BaseAPIKeyAuth):
except ValueError:
raise AuthenticationFailed("Invalid API Key.")
# Validation, the `last_used_at` update and the auth claims all read the same
# row, locked until the transaction ends. Looking the key up a second time to
# build the claims used to leave a window where a key revoked or orphaned right
# after passing validation still authenticated.
with transaction.atomic(using=MainRouter.admin_db):
try:
api_key = self._authenticate_credentials(request, key)
except OrphanedAPIKeyError:
# Rejected below instead of here: leaving the block normally commits
# the revocation `_authenticate_credentials` wrote, while raising from
# inside would roll it back.
pass
else:
# The prefix used to be checked by the second lookup
if api_key.prefix != prefix:
raise AuthenticationFailed("Invalid API Key.")
try:
entity, _ = self._authenticate_credentials(request, key)
except InvalidToken:
raise AuthenticationFailed("Invalid API Key.")
api_key.last_used_at = timezone.now()
api_key.save(update_fields=["last_used_at"], using=MainRouter.admin_db)
# Get the API key instance to update last_used_at and retrieve tenant info
# We need to decrypt again to get the pk (already validated by _authenticate_credentials)
payload = self.key_crypto.decrypt(key)
api_key_pk = payload["_pk"]
entity = api_key.entity
return entity, {
"tenant_id": str(api_key.tenant_id),
"sub": str(entity.id),
"api_key_prefix": api_key.prefix,
}
# Convert string UUID back to UUID object for lookup
if isinstance(api_key_pk, str):
api_key_pk = UUID(api_key_pk)
raise AuthenticationFailed("No entity matching this api key.")
try:
api_key_instance = TenantAPIKey.objects.using(MainRouter.admin_db).get(
id=api_key_pk, prefix=prefix
)
except TenantAPIKey.DoesNotExist:
raise AuthenticationFailed("Invalid API Key.")
# Update last_used_at
api_key_instance.last_used_at = timezone.now()
api_key_instance.save(update_fields=["last_used_at"], using=MainRouter.admin_db)
return entity, {
"tenant_id": str(api_key_instance.tenant_id),
"sub": str(api_key_instance.entity.id),
"api_key_prefix": prefix,
}
class CombinedJWTOrAPIKeyAuthentication(BaseAuthentication):
+1 -7
View File
@@ -3,10 +3,9 @@ from api.db_router import MainRouter, reset_read_db_alias, set_read_db_alias
from api.db_utils import POSTGRES_USER_VAR, rls_transaction
from api.filters import CustomDjangoFilterBackend
from api.models import Role, UserRoleRelationship
from api.rbac.permissions import HasPermissions, get_role
from api.rbac.permissions import HasPermissions
from django.conf import settings
from django.db import transaction
from django.utils.functional import cached_property
from rest_framework import permissions
from rest_framework.exceptions import NotAuthenticated
from rest_framework.filters import SearchFilter
@@ -101,11 +100,6 @@ class BaseRLSViewSet(BaseViewSet):
context["tenant_id"] = self.request.tenant_id
return context
@cached_property
def user_role(self):
"""Role of the requesting user in the active tenant, resolved once per request."""
return get_role(self.request.user, self.request.tenant_id)
class BaseTenantViewset(BaseViewSet):
def dispatch(self, request, *args, **kwargs):
-40
View File
@@ -1,40 +0,0 @@
import ast
import json
from typing import Any
_UNPARSED = object()
def decode_celery_field(value: Any, default: Any) -> Any:
"""Decode a Celery result field and require JSON-serializable output."""
decoded = value
for _ in range(2):
if not isinstance(decoded, str):
break
text = decoded.strip()
if not text:
decoded = default
break
parsed = _UNPARSED
for parser in (json.loads, ast.literal_eval):
try:
parsed = parser(text)
break
except (TypeError, ValueError, SyntaxError):
continue
if parsed is _UNPARSED:
raise ValueError("Unable to decode Celery result field")
decoded = parsed
decoded = default if decoded is None else decoded
try:
json.dumps(decoded, allow_nan=False)
except (TypeError, ValueError) as error:
raise ValueError(
"Decoded Celery result field is not JSON serializable"
) from error
return decoded
-29
View File
@@ -1,4 +1,3 @@
from contextlib import contextmanager
from contextvars import ContextVar
from django.conf import settings
@@ -6,7 +5,6 @@ from django.conf import settings
ALLOWED_APPS = ("django", "socialaccount", "account", "authtoken", "silk")
_read_db_alias = ContextVar("read_db_alias", default=None)
_write_db_alias = ContextVar("write_db_alias", default=None)
def set_read_db_alias(alias: str | None):
@@ -24,30 +22,6 @@ def reset_read_db_alias(token) -> None:
_read_db_alias.reset(token)
def set_write_db_alias(alias: str | None):
if not alias:
return None
return _write_db_alias.set(alias)
def get_write_db_alias() -> str | None:
return _write_db_alias.get()
def reset_write_db_alias(token) -> None:
if token is not None:
_write_db_alias.reset(token)
@contextmanager
def write_db_alias(alias: str | None):
token = set_write_db_alias(alias)
try:
yield
finally:
reset_write_db_alias(token)
class MainRouter:
default_db = "default"
admin_db = "admin"
@@ -69,9 +43,6 @@ class MainRouter:
model_table_name = model._meta.db_table
if any(model_table_name.startswith(f"{app}_") for app in ALLOWED_APPS):
return self.admin_db
write_alias = get_write_db_alias()
if write_alias:
return write_alias
return None
def allow_migrate(self, db, app_label, model_name=None, **hints): # noqa: F841
+50 -273
View File
@@ -2,7 +2,7 @@ import re
import secrets
import time
import uuid
from contextlib import ExitStack, contextmanager, nullcontext
from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
from api.db_router import (
@@ -48,140 +48,6 @@ REPLICA_MAX_ATTEMPTS = env.int("POSTGRES_REPLICA_MAX_ATTEMPTS", default=3)
REPLICA_RETRY_BASE_DELAY = env.float("POSTGRES_REPLICA_RETRY_BASE_DELAY", default=0.5)
SET_CONFIG_QUERY = "SELECT set_config(%s, %s::text, TRUE);"
SET_TRANSACTION_READ_ONLY_QUERY = "SET TRANSACTION READ ONLY;"
REPLICA_CONNECTION_SQLSTATE_PREFIXES = ("08",)
REPLICA_CONNECTION_SQLSTATES = {"57P01", "57P02", "57P03"}
REPLICA_NON_FAILOVER_SQLSTATES = {"57014", "40001", "40P01"}
REPLICA_CONNECTION_ERROR_MESSAGES = (
"ssl syscall",
"eof detected",
"server closed the connection",
"connection already closed",
"connection not open",
"could not connect to server",
"connection refused",
"connection reset",
"connection timed out",
"lost synchronization",
"terminating connection",
"database system is starting up",
"database system is shutting down",
"database system is in recovery mode",
)
REPLICA_NON_FAILOVER_ERROR_MESSAGES = (
"canceling statement due to user request",
"deadlock detected",
"could not serialize access",
)
def _iter_exception_chain(error: BaseException):
seen = set()
pending = [error]
while pending:
current = pending.pop(0)
if current is None or id(current) in seen:
continue
seen.add(id(current))
yield current
cause = getattr(current, "__cause__", None)
context = getattr(current, "__context__", None)
if cause is not None:
pending.append(cause)
if context is not None:
pending.append(context)
for arg in getattr(current, "args", ()):
if isinstance(arg, BaseException):
pending.append(arg)
def _get_exception_sqlstate(error: BaseException) -> str | None:
for attr in ("pgcode", "sqlstate"):
sqlstate = getattr(error, attr, None)
if sqlstate:
return sqlstate
diag = getattr(error, "diag", None)
if diag is not None:
sqlstate = getattr(diag, "sqlstate", None)
if sqlstate:
return sqlstate
return None
def _is_replica_connection_failure(error: BaseException) -> bool:
"""
Return True only for replica failures where retrying on primary is safe.
Query cancellations, serialization failures, and deadlocks should surface to
callers because replaying them can hide real query or concurrency problems.
"""
messages = []
sqlstates = set()
for chained_error in _iter_exception_chain(error):
sqlstate = _get_exception_sqlstate(chained_error)
if sqlstate:
sqlstates.add(sqlstate)
messages.append(str(chained_error).lower())
if sqlstates & REPLICA_NON_FAILOVER_SQLSTATES:
return False
if any(
sqlstate.startswith(REPLICA_CONNECTION_SQLSTATE_PREFIXES)
or sqlstate in REPLICA_CONNECTION_SQLSTATES
for sqlstate in sqlstates
):
return True
message = " ".join(messages)
if any(marker in message for marker in REPLICA_NON_FAILOVER_ERROR_MESSAGES):
return False
return any(marker in message for marker in REPLICA_CONNECTION_ERROR_MESSAGES)
def _strip_leading_sql_comments(sql: str) -> str:
if not isinstance(sql, str):
return ""
sql_text = sql.lstrip()
while True:
if sql_text.startswith("--"):
newline_index = sql_text.find("\n")
if newline_index == -1:
return ""
sql_text = sql_text[newline_index + 1 :].lstrip()
continue
if sql_text.startswith("/*"):
comment_end_index = sql_text.find("*/", 2)
if comment_end_index == -1:
return ""
sql_text = sql_text[comment_end_index + 2 :].lstrip()
continue
return sql_text
def _is_safe_primary_replay(sql: str, many: bool) -> bool:
if many:
return False
sql_text = _strip_leading_sql_comments(sql)
if not re.match(r"(?is)^SELECT\b", sql_text):
return False
return not any(
re.search(pattern, sql_text, re.IGNORECASE | re.DOTALL)
for pattern in (
r"\bINTO\b",
r"\bFOR\s+(?:NO\s+KEY\s+)?UPDATE\b",
r"\bFOR\s+(?:KEY\s+)?SHARE\b",
)
)
@contextmanager
@@ -211,36 +77,14 @@ def rls_transaction(
retry_on_replica: bool = True,
):
"""
Context manager that opens an RLS-scoped database transaction.
Sets a Postgres configuration variable (``set_config``) so that Row-Level
Security policies can filter by tenant. When *using* points to a read
replica and *retry_on_replica* is True, replica failures are handled in two
places:
1. **Pre-yield** (connection-setup failures): the function retries
up to ``REPLICA_MAX_ATTEMPTS`` times on the replica, then falls
back to the primary DB.
2. **Post-yield** (mid-query failures): an ``execute_wrapper``
intercepts connection-level ``OperationalError`` during
``cursor.execute()`` calls and falls back directly to the primary DB
for single ``SELECT`` statements. The primary fallback transaction is
read-only, and unsafe statements keep raising the original error.
The wrapper swaps the inner cursor so ``fetchall()`` / ``fetchone()``
read from the new connection transparently.
Limitation: server-side cursors (``.iterator()``) fetch rows via
``fetchmany()``, which the wrapper does not intercept. Call sites
that iterate large result sets with ``.iterator()`` on the replica
should add their own retry logic.
Creates a new database transaction setting the given configuration value for Postgres RLS. It validates the
if the value is a valid UUID.
Args:
value: Database configuration parameter value (must be a valid UUID).
parameter: Database configuration parameter name.
using: Optional database alias. Defaults to the active read
alias or Django's default connection.
retry_on_replica: Whether replica setup failures can retry and
connection-level mid-query failures can fall back to primary.
value (str): Database configuration parameter value.
parameter (str): Database configuration parameter name, by default is 'api.tenant_id'.
using (str | None): Optional database alias to run the transaction against. Defaults to the
active read alias (if any) or Django's default connection.
"""
requested_alias = using or get_read_db_alias()
db_alias = requested_alias or DEFAULT_DB_ALIAS
@@ -248,121 +92,54 @@ def rls_transaction(
db_alias = DEFAULT_DB_ALIAS
alias = db_alias
is_replica = bool(READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS)
can_failover = is_replica and retry_on_replica
replica_alias = alias # captured before the loop mutates alias
max_attempts = (REPLICA_MAX_ATTEMPTS + 1) if can_failover else 1
is_replica = READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS
max_attempts = REPLICA_MAX_ATTEMPTS if is_replica and retry_on_replica else 1
# State shared between the generator and the _query_failover closure.
# The fallback transaction.atomic() is registered into fallback_stack
# via enter_context so its __exit__ runs when the outer with-ExitStack
# block exits, with the right exc_info. No manual __enter__/__exit__.
_fallback = {"succeeded": False, "token": None, "caller_exited_cleanly": False}
for attempt in range(1, max_attempts + 1):
router_token = None
yielded_cursor = False
with ExitStack() as fallback_stack:
# On final attempt, fallback to primary
if attempt == max_attempts and is_replica:
logger.warning(
f"RLS transaction failed after {attempt - 1} attempts on replica, "
f"falling back to primary DB"
)
alias = DEFAULT_DB_ALIAS
def _query_failover(execute, sql, params, many, context):
"""execute_wrapper: replay failed replica queries on the primary DB."""
try:
return execute(sql, params, many, context)
except OperationalError as err:
if not _is_replica_connection_failure(err):
raise
if not _is_safe_primary_replay(sql, many):
raise
conn = connections[alias]
try:
if alias != DEFAULT_DB_ALIAS:
router_token = set_read_db_alias(alias)
try:
connections[replica_alias].close()
except Exception:
pass # Best-effort; connection may already be dead
with transaction.atomic(using=alias):
with conn.cursor() as cursor:
try:
# just in case the value is a UUID object
uuid.UUID(str(value))
except ValueError:
raise ValidationError("Must be a valid UUID")
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
yielded_cursor = True
yield cursor
return
except OperationalError as e:
if yielded_cursor:
raise
# If on primary or max attempts reached, raise
if not is_replica or attempt == max_attempts:
raise
logger.warning(
"Mid-query replica connection failure, falling back to primary DB"
)
primary = connections[DEFAULT_DB_ALIAS]
primary.ensure_connection()
fallback_stack.enter_context(transaction.atomic(using=DEFAULT_DB_ALIAS))
fallback_cursor = primary.cursor()
fallback_stack.callback(fallback_cursor.close)
fallback_cursor.execute(SET_TRANSACTION_READ_ONLY_QUERY)
fallback_cursor.execute(SET_CONFIG_QUERY, [parameter, value])
_fallback["token"] = set_read_db_alias(DEFAULT_DB_ALIAS)
fallback_cursor.execute(sql, params)
context["cursor"].db = primary
context["cursor"].cursor = fallback_cursor.cursor
_fallback["succeeded"] = True
return None
for attempt in range(1, max_attempts + 1):
router_token = None
yielded_cursor = False
# On final attempt, fall back to primary
if attempt == max_attempts and can_failover:
if attempt > 1:
logger.warning(
f"RLS transaction failed after {attempt - 1} attempts on replica, "
f"falling back to primary DB"
)
alias = DEFAULT_DB_ALIAS
conn = connections[alias]
try:
if alias != DEFAULT_DB_ALIAS:
router_token = set_read_db_alias(alias)
with transaction.atomic(using=alias):
with conn.cursor() as cursor:
try:
uuid.UUID(str(value))
except ValueError:
raise ValidationError("Must be a valid UUID")
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
wrapper_cm = (
conn.execute_wrapper(_query_failover)
if can_failover and alias == replica_alias
else nullcontext()
)
with wrapper_cm:
yielded_cursor = True
yield cursor
_fallback["caller_exited_cleanly"] = True
return
except OperationalError as e:
if yielded_cursor:
if _fallback["succeeded"] and _fallback["caller_exited_cleanly"]:
# Caller's queries succeeded on primary via failover.
# This error is transaction.atomic() cleanup on the
# dead replica connection, suppress it.
return
raise
if not can_failover or attempt == max_attempts:
raise
try:
connections[alias].close()
except Exception:
pass # Best-effort; connection may already be dead
# Retry with exponential backoff
delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1))
logger.info(
f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), "
f"retrying in {delay}s. Error: {e}"
)
time.sleep(delay)
finally:
if _fallback["token"] is not None:
reset_read_db_alias(_fallback["token"])
_fallback["token"] = None
if router_token is not None:
reset_read_db_alias(router_token)
# Retry with exponential backoff
delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1))
logger.info(
f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), "
f"retrying in {delay}s. Error: {e}"
)
time.sleep(delay)
finally:
if router_token is not None:
reset_read_db_alias(router_token)
class CustomUserManager(BaseUserManager):
+7 -22
View File
@@ -1,13 +1,12 @@
import uuid
from functools import wraps
from api.attack_paths.database import GraphDatabaseQueryException
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY, rls_transaction
from api.exceptions import ProviderDeletedException
from api.models import Membership, Provider, Scan, Tenant
from api.models import Provider, Scan
from django.core.exceptions import ObjectDoesNotExist
from django.db import DEFAULT_DB_ALIAS, DatabaseError, connection, transaction
from django.db import DatabaseError, connection, transaction
from rest_framework_json_api.serializers import ValidationError
@@ -76,11 +75,9 @@ def handle_provider_deletion(func):
"""
Decorator that raises `ProviderDeletedException` if provider was deleted during execution.
Catches `ObjectDoesNotExist`, `DatabaseError` (including `IntegrityError`), and
`GraphDatabaseQueryException`, checks if provider still exists, and raises
`ProviderDeletedException` if not. Graph database errors also check whether the
tenant still exists and has memberships. Otherwise, re-raises the original
exception.
Catches `ObjectDoesNotExist` and `DatabaseError` (including `IntegrityError`), checks if
provider still exists, and raises `ProviderDeletedException` if not. Otherwise,
re-raises original exception.
Requires `tenant_id` and `provider_id` in kwargs.
@@ -95,16 +92,11 @@ def handle_provider_deletion(func):
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except (ObjectDoesNotExist, DatabaseError, GraphDatabaseQueryException) as exc:
except (ObjectDoesNotExist, DatabaseError):
tenant_id = kwargs.get("tenant_id")
provider_id = kwargs.get("provider_id")
database_alias = (
DEFAULT_DB_ALIAS
if isinstance(exc, GraphDatabaseQueryException)
else READ_REPLICA_ALIAS
)
with rls_transaction(tenant_id, using=database_alias):
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
if provider_id is None:
scan_id = kwargs.get("scan_id")
if scan_id is None:
@@ -121,13 +113,6 @@ def handle_provider_deletion(func):
raise ProviderDeletedException(
f"Provider '{provider_id}' was deleted during the scan"
) from None
if isinstance(exc, GraphDatabaseQueryException) and (
not Tenant.objects.filter(pk=tenant_id).exists()
or not Membership.objects.filter(tenant_id=tenant_id).exists()
):
raise ProviderDeletedException(
f"Tenant '{tenant_id}' was deleted during the scan"
) from None
raise
return wrapper
+56 -203
View File
@@ -67,7 +67,6 @@ from django_filters.rest_framework import (
)
from rest_framework_json_api.django_filters.backends import DjangoFilterBackend
from rest_framework_json_api.serializers import ValidationError
from uuid6 import UUID
class CustomDjangoFilterBackend(DjangoFilterBackend):
@@ -673,32 +672,35 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
return queryset.filter(tags__text_search=value)
FINDING_BASE_FILTER_FIELDS = {
"id": ["exact", "in"],
"uid": ["exact", "in"],
"scan": ["exact", "in"],
"delta": ["exact", "in"],
"status": ["exact", "in"],
"severity": ["exact", "in"],
"impact": ["exact", "in"],
"check_id": ["exact", "in", "icontains"],
}
class BaseFindingFilter(CommonFindingFilters):
DATE_FILTER_FIELDS = ()
DATE_FILTER_NAMES = ()
DATE_RANGE_HELP_TEXT = (
f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days."
)
DATE_FILTER_REQUIRED_DETAIL = "At least one date filter is required."
class FindingFilter(CommonFindingFilters):
scan = UUIDFilter(method="filter_scan_id")
scan__in = UUIDInFilter(method="filter_scan_id_in")
inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date")
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
inserted_at__gte = DateFilter(
method="filter_inserted_at_gte",
help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
)
inserted_at__lte = DateFilter(
method="filter_inserted_at_lte",
help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
)
class Meta:
model = Finding
fields = FINDING_BASE_FILTER_FIELDS
fields = {
"id": ["exact", "in"],
"uid": ["exact", "in"],
"scan": ["exact", "in"],
"delta": ["exact", "in"],
"status": ["exact", "in"],
"severity": ["exact", "in"],
"impact": ["exact", "in"],
"check_id": ["exact", "in", "icontains"],
"inserted_at": ["date", "gte", "lte"],
"updated_at": ["gte", "lte"],
}
filter_overrides = {
FindingDeltaEnumField: {
"filter_class": CharFilter,
@@ -721,13 +723,17 @@ class BaseFindingFilter(CommonFindingFilters):
return queryset.filter(resource_services__contains=[value])
def filter_queryset(self, queryset):
if not (self.data.get("scan") or self.data.get("scan__in")) and not any(
self.data.get(filter_name) for filter_name in self.DATE_FILTER_NAMES
if not (self.data.get("scan") or self.data.get("scan__in")) and not (
self.data.get("inserted_at")
or self.data.get("inserted_at__date")
or self.data.get("inserted_at__gte")
or self.data.get("inserted_at__lte")
):
raise ValidationError(
[
{
"detail": self.DATE_FILTER_REQUIRED_DETAIL,
"detail": "At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], "
"or filter[inserted_at.lte].",
"status": 400,
"source": {"pointer": "/data/attributes/inserted_at"},
"code": "required",
@@ -736,41 +742,30 @@ class BaseFindingFilter(CommonFindingFilters):
)
cleaned = self.form.cleaned_data
for field_name in self.DATE_FILTER_FIELDS:
self.validate_datetime_filter_range(cleaned, field_name)
exact_date = cleaned.get("inserted_at") or cleaned.get("inserted_at__date")
gte_date = cleaned.get("inserted_at__gte") or exact_date
lte_date = cleaned.get("inserted_at__lte") or exact_date
return super().filter_queryset(queryset)
if gte_date is None:
gte_date = datetime.now(UTC).date()
if lte_date is None:
lte_date = datetime.now(UTC).date()
def validate_datetime_filter_range(self, cleaned, field_name):
exact_value = cleaned.get(field_name) or cleaned.get(f"{field_name}__date")
gte_value = cleaned.get(f"{field_name}__gte") or exact_value
lte_value = cleaned.get(f"{field_name}__lte") or exact_value
if not (exact_value or gte_value or lte_value):
return
default_value = datetime.now(UTC).date()
gte_value = gte_value or default_value
lte_value = lte_value or default_value
gte_datetime = self.filter_value_to_datetime(gte_value, field_name)
lte_datetime = self.filter_value_to_datetime(lte_value, field_name)
if abs(lte_datetime - gte_datetime) <= timedelta(
if abs(lte_date - gte_date) > timedelta(
days=settings.FINDINGS_MAX_DAYS_IN_RANGE
):
return
raise ValidationError(
[
{
"detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
"status": 400,
"source": {"pointer": "/data/attributes/inserted_at"},
"code": "invalid",
}
]
)
raise ValidationError(
[
{
"detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
"status": 400,
"source": {"pointer": f"/data/attributes/{field_name}"},
"code": "invalid",
}
]
)
return super().filter_queryset(queryset)
# Convert filter values to UUIDv7 values for use with partitioning
def filter_scan_id(self, queryset, name, value):
@@ -829,169 +824,27 @@ class BaseFindingFilter(CommonFindingFilters):
datetime_value = self.maybe_date_to_datetime(value)
start = uuid7_start(datetime_to_uuid7(datetime_value))
end = uuid7_start(datetime_to_uuid7(datetime_value + timedelta(days=1)))
return queryset.filter(id__gte=start, id__lt=end)
def filter_inserted_at_gte(self, queryset, name, value):
datetime_value = self.maybe_date_to_datetime(value)
start = uuid7_start(datetime_to_uuid7(datetime_value))
return queryset.filter(id__gte=start)
def filter_inserted_at_lte(self, queryset, name, value):
datetime_value = self.maybe_date_to_datetime(value)
end = uuid7_start(datetime_to_uuid7(datetime_value + timedelta(days=1)))
return queryset.filter(id__lt=end)
@staticmethod
def maybe_date_to_datetime(value):
if isinstance(value, datetime):
return value
dt = value
if isinstance(value, date):
return datetime.combine(value, datetime.min.time(), tzinfo=UTC)
if isinstance(value, str):
return parse(value)
return value
@classmethod
def filter_value_to_datetime(cls, value, field_name):
try:
datetime_value = cls.maybe_date_to_datetime(value)
except (TypeError, ValueError, OverflowError):
raise ValidationError(
[
{
"detail": "Enter a valid date or datetime.",
"status": 400,
"source": {"pointer": f"/data/attributes/{field_name}"},
"code": "invalid",
}
]
)
if datetime_value.tzinfo is None:
return datetime_value.replace(tzinfo=UTC)
return datetime_value.astimezone(UTC)
class FindingFilter(BaseFindingFilter):
DATE_FILTER_FIELDS = ("inserted_at", "updated_at")
DATE_FILTER_NAMES = (
"inserted_at",
"inserted_at__date",
"inserted_at__gte",
"inserted_at__lte",
"updated_at",
"updated_at__date",
"updated_at__gte",
"updated_at__lte",
)
DATE_FILTER_REQUIRED_DETAIL = (
"At least one date filter is required: filter[inserted_at], filter[updated_at], "
"filter[inserted_at.gte], filter[updated_at.gte], filter[inserted_at.lte], "
"or filter[updated_at.lte]."
)
inserted_at = CharFilter(method="filter_inserted_at")
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
inserted_at__gte = CharFilter(
method="filter_inserted_at",
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
)
inserted_at__lte = CharFilter(
method="filter_inserted_at",
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
)
updated_at = CharFilter(method="filter_updated_at")
updated_at__date = DateFilter(method="filter_updated_at", lookup_expr="date")
updated_at__gte = CharFilter(
method="filter_updated_at",
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
)
updated_at__lte = CharFilter(
method="filter_updated_at",
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
)
class Meta(BaseFindingFilter.Meta):
fields = FINDING_BASE_FILTER_FIELDS | {
"inserted_at": ["date", "gte", "lte"],
"updated_at": ["date", "gte", "lte"],
}
def filter_inserted_at(self, queryset, name, value):
start, end = self.filter_value_to_datetime_bounds(value, "inserted_at")
if name.endswith("__gte"):
return queryset.filter(id__gte=self.datetime_to_uuid7_boundary(start))
if name.endswith("__lte"):
return queryset.filter(id__lt=self.datetime_to_uuid7_boundary(end))
return queryset.filter(
id__gte=self.datetime_to_uuid7_boundary(start),
id__lt=self.datetime_to_uuid7_boundary(end),
)
def filter_updated_at(self, queryset, name, value):
start, end = self.filter_value_to_datetime_bounds(value, "updated_at")
if name.endswith("__gte"):
return queryset.filter(updated_at__gte=start)
if name.endswith("__lte"):
return queryset.filter(updated_at__lt=end)
return queryset.filter(updated_at__gte=start, updated_at__lt=end)
@classmethod
def filter_value_to_datetime_bounds(cls, value, field_name):
start = cls.filter_value_to_datetime(value, field_name)
if cls.is_date_filter_value(value):
return start, start + timedelta(days=1)
return start, start + timedelta(milliseconds=1)
@staticmethod
def datetime_to_uuid7_boundary(datetime_value):
timestamp_ms = int(datetime_value.timestamp() * 1000) & 0xFFFFFFFFFFFF
uuid_int = timestamp_ms << 80
uuid_int |= 0x7 << 76
uuid_int |= 0x2 << 62
return UUID(int=uuid_int)
@staticmethod
def is_date_filter_value(value):
if isinstance(value, datetime):
return False
if isinstance(value, date):
return True
return isinstance(value, str) and len(value.strip()) == 10
class FindingMetadataFilter(BaseFindingFilter):
DATE_FILTER_FIELDS = ("inserted_at",)
DATE_FILTER_NAMES = (
"inserted_at",
"inserted_at__date",
"inserted_at__gte",
"inserted_at__lte",
)
DATE_FILTER_REQUIRED_DETAIL = (
"At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], "
"or filter[inserted_at.lte]."
)
inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date")
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
inserted_at__gte = DateFilter(
method="filter_inserted_at_gte",
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
)
inserted_at__lte = DateFilter(
method="filter_inserted_at_lte",
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
)
class Meta(BaseFindingFilter.Meta):
fields = FINDING_BASE_FILTER_FIELDS | {
"inserted_at": ["date", "gte", "lte"],
}
dt = datetime.combine(value, datetime.min.time(), tzinfo=UTC)
return dt
class LatestFindingFilter(CommonFindingFilters):
@@ -1,25 +0,0 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("api", "0096_attack_paths_scan_is_migrated"),
]
operations = [
migrations.AlterField(
model_name="attackpathsscan",
name="is_migrated",
field=models.BooleanField(db_default=False, default=False),
),
migrations.AlterField(
model_name="attackpathsscan",
name="sink_backend",
field=models.CharField(
choices=[("neo4j", "Neo4j"), ("neptune", "Neptune")],
db_default="neo4j",
default="neo4j",
max_length=16,
),
),
]
+1 -2
View File
@@ -814,10 +814,9 @@ class AttackPathsScan(RowLevelSecurityProtectedModel):
# still using the previous graph shape. Query catalog selection uses this
# flag; physical read routing uses sink_backend below.
# TODO: drop after Neptune cutover
is_migrated = models.BooleanField(default=False, db_default=False)
is_migrated = models.BooleanField(default=False)
sink_backend = models.CharField(
choices=SinkBackendChoices.choices,
db_default=SinkBackendChoices.NEO4J,
default=SinkBackendChoices.NEO4J,
max_length=16,
)
+8 -36
View File
@@ -1,8 +1,8 @@
from enum import Enum
from api.db_router import MainRouter
from api.models import Integration, Provider, Role, User
from django.db.models import Q, QuerySet
from api.models import Provider, Role, User
from django.db.models import QuerySet
from rest_framework.exceptions import PermissionDenied
from rest_framework.permissions import BasePermission
@@ -34,7 +34,7 @@ class HasPermissions(BasePermission):
if not tenant_id:
return False
user_roles = list(
user_roles = (
User.objects.using(MainRouter.admin_db)
.get(id=request.user.id)
.roles.using(MainRouter.admin_db)
@@ -43,10 +43,11 @@ class HasPermissions(BasePermission):
if not user_roles:
return False
return all(
any(getattr(role, permission.value, False) for role in user_roles)
for permission in required_permissions
)
for perm in required_permissions:
if not getattr(user_roles[0], perm.value, False):
return False
return True
def get_role(user: User, tenant_id: str) -> Role:
@@ -83,32 +84,3 @@ def get_providers(role: Role) -> QuerySet[Provider]:
return Provider.objects.filter(
tenant_id=tenant_id, provider_groups__in=provider_groups
).distinct()
def get_integrations(
role: Role, providers: QuerySet[Provider] | None = None
) -> QuerySet[Integration]:
"""
Return a distinct queryset of Integrations visible to the given role.
Integrations with no providers attached are tenant-wide, as is always the case for
Jira, and stay visible regardless of the provider visibility of the role. Integrations
attached to providers are only visible when the role can access at least one of them.
Args:
role: A Role instance.
providers: Optional queryset of the providers accessible by the role, to reuse
an already resolved `get_providers(role)` result within the same request.
Returns:
A QuerySet of Integration objects visible to the role.
"""
queryset = Integration.objects.filter(tenant_id=role.tenant_id)
if role.unlimited_visibility:
return queryset
if providers is None:
providers = get_providers(role)
return queryset.filter(
Q(providers__isnull=True) | Q(providers__in=providers)
).distinct()

Some files were not shown because too many files have changed in this diff Show More