mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-07-24 04:51:51 +00:00
Merge branch 'master' into fix/provider-wizard-modal-loop
This commit is contained in:
@@ -72,8 +72,8 @@ NEO4J_APOC_IMPORT_FILE_ENABLED=false
|
||||
NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG=true
|
||||
NEO4J_APOC_TRIGGER_ENABLED=false
|
||||
NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS=0.0.0.0:7687
|
||||
# Neo4j Prowler settings
|
||||
ATTACK_PATHS_BATCH_SIZE=1000
|
||||
# Attack Paths graph settings
|
||||
ATTACK_PATHS_GRAPH_MUTATION_BATCH_SIZE=1000
|
||||
ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES=3
|
||||
ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS=30
|
||||
ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES=250
|
||||
@@ -145,19 +145,20 @@ DJANGO_BROKER_VISIBILITY_TIMEOUT=86400
|
||||
DJANGO_SENTRY_DSN=
|
||||
DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute
|
||||
|
||||
# Sentry for the web app (server + browser). Empty/unset UI_SENTRY_DSN ⇒
|
||||
# Sentry disabled, zero egress. SENTRY_RELEASE (unprefixed) feeds the web app's
|
||||
# server/edge SDKs.
|
||||
# Sentry for the web app (server + browser). The UI_SENTRY_* values load only
|
||||
# when UI_SENTRY_ENABLED="true"; without it they are ignored (default off, zero
|
||||
# egress). The deprecated NEXT_PUBLIC_SENTRY_DSN still activates Sentry without
|
||||
# the flag. SENTRY_RELEASE (unprefixed) feeds the web app's server/edge SDKs.
|
||||
UI_SENTRY_DSN=
|
||||
UI_SENTRY_ENVIRONMENT=local
|
||||
SENTRY_RELEASE=local
|
||||
# Reserved runtime public config (registered now; no UI consumer yet)
|
||||
# POSTHOG_KEY=
|
||||
# POSTHOG_HOST=
|
||||
# UI_POSTHOG_KEY=
|
||||
# UI_POSTHOG_HOST=
|
||||
# REO_DEV_CLIENT_ID=
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.32.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.36.0
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -46,7 +46,7 @@ runs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -sf \
|
||||
LATEST_COMMIT=$(curl -sf --retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 60 \
|
||||
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/prowler-cloud/prowler/commits/master" \
|
||||
@@ -66,7 +66,7 @@ runs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
LATEST_COMMIT=$(curl -sf \
|
||||
LATEST_COMMIT=$(curl -sf --retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 60 \
|
||||
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/prowler-cloud/prowler/commits/master" \
|
||||
|
||||
@@ -63,7 +63,7 @@ runs:
|
||||
exit-code: '0'
|
||||
scanners: 'vuln'
|
||||
timeout: '5m'
|
||||
version: 'v0.71.0'
|
||||
version: 'v0.71.2'
|
||||
|
||||
- name: Run Trivy vulnerability scan (SARIF)
|
||||
if: inputs.upload-sarif == 'true' && github.event_name == 'push'
|
||||
@@ -76,7 +76,7 @@ runs:
|
||||
exit-code: '0'
|
||||
scanners: 'vuln'
|
||||
timeout: '5m'
|
||||
version: 'v0.71.0'
|
||||
version: 'v0.71.2'
|
||||
|
||||
- name: Upload Trivy results to GitHub Security tab
|
||||
if: inputs.upload-sarif == 'true' && github.event_name == 'push'
|
||||
|
||||
@@ -5,10 +5,20 @@
|
||||
"version": "v8",
|
||||
"sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd"
|
||||
},
|
||||
"github/gh-aw-actions/setup@v0.81.6": {
|
||||
"repo": "github/gh-aw-actions/setup",
|
||||
"version": "v0.81.6",
|
||||
"sha": "ba6380cc6e5be5d21677bebe04d52fb48e3abec7"
|
||||
},
|
||||
"github/gh-aw/actions/setup@v0.43.23": {
|
||||
"repo": "github/gh-aw/actions/setup",
|
||||
"version": "v0.43.23",
|
||||
"sha": "9382be3ca9ac18917e111a99d4e6bbff58d0dccc"
|
||||
},
|
||||
"step-security/harden-runner@v2.20.0": {
|
||||
"repo": "step-security/harden-runner",
|
||||
"version": "v2.20.0",
|
||||
"sha": "bf7454d06d71f1098171f2acdf0cd4708d7b5920"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
# v5
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/api"
|
||||
# schedule:
|
||||
# interval: "daily"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# - "component/api"
|
||||
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "github_actions"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "npm"
|
||||
# directory: "/ui"
|
||||
# schedule:
|
||||
# interval: "daily"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "npm"
|
||||
# - "component/ui"
|
||||
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 25
|
||||
target-branch: master
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "docker"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
# - package-ecosystem: "pre-commit"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pre-commit"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/04/15
|
||||
# v4.6
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v4.6
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# - "v4"
|
||||
|
||||
# - package-ecosystem: "github-actions"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v4.6
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "github_actions"
|
||||
# - "v4"
|
||||
|
||||
# - package-ecosystem: "docker"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v4.6
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "docker"
|
||||
# - "v4"
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# v3
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v3
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# - "v3"
|
||||
|
||||
# - package-ecosystem: "github-actions"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v3
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "github_actions"
|
||||
# - "v3"
|
||||
@@ -18,8 +18,9 @@ Please add a detailed description of how to review this PR.
|
||||
|
||||
<summary><b>Community Checklist</b></summary>
|
||||
|
||||
- [ ] This feature/issue is listed in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
|
||||
- [ ] Is it assigned to me, if not, request it via the issue/feature in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](goto.prowler.com/slack)
|
||||
- [ ] This feature/issue is listed in the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
|
||||
- [ ] Is it assigned to me, if not, request it via the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](https://goto.prowler.com/slack)
|
||||
- [ ] I have reviewed the [open pull requests](https://github.com/prowler-cloud/prowler/pulls?q=sort%3Aupdated-desc+is%3Apr+is%3Aopen) and confirmed there is no existing PR that implements the same outcome
|
||||
|
||||
</details>
|
||||
|
||||
@@ -27,8 +28,8 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Review if the code is being covered by tests.
|
||||
- [ ] Review if code is being documented following this specification https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings
|
||||
- [ ] Review if backport is needed.
|
||||
- [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable.
|
||||
- [ ] Review if is needed to change the [README.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
|
||||
- [ ] Ensure a changelog fragment is added under [prowler/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/prowler/changelog.d), if applicable.
|
||||
|
||||
#### SDK/CLI
|
||||
- Are there new checks included in this PR? Yes / No
|
||||
@@ -40,7 +41,7 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Table (640px > X < 1024px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Desktop (X > 1024px)
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable.
|
||||
- [ ] Ensure a changelog fragment is added under [ui/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/ui/changelog.d), if applicable.
|
||||
|
||||
#### API
|
||||
- [ ] All issue/task requirements work as expected on the API
|
||||
@@ -50,7 +51,11 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Any other relevant evidence of the implementation (if applicable)
|
||||
- [ ] Verify if API specs need to be regenerated.
|
||||
- [ ] Check if version updates are required (e.g., specs, uv, etc.).
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable.
|
||||
- [ ] Ensure a changelog fragment is added under [api/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/api/changelog.d), if applicable.
|
||||
|
||||
#### MCP Server
|
||||
- [ ] All issue/task requirements work as expected on the MCP Server
|
||||
- [ ] Ensure a changelog fragment is added under [mcp_server/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/mcp_server/changelog.d), if applicable.
|
||||
|
||||
### License
|
||||
|
||||
|
||||
+13
-2
@@ -23,6 +23,10 @@
|
||||
"prConcurrentLimit": 20,
|
||||
"prHourlyLimit": 10,
|
||||
"vulnerabilityAlerts": {
|
||||
"labels": [
|
||||
"dependencies",
|
||||
"security"
|
||||
],
|
||||
"prHourlyLimit": 0,
|
||||
"prConcurrentLimit": 0
|
||||
},
|
||||
@@ -38,7 +42,7 @@
|
||||
"schedule": [
|
||||
"* 22-23,0-5 1 * *"
|
||||
],
|
||||
"enabled": false
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "Minors: 8th of every 3 months, Madrid overnight window (22:00-06:00)",
|
||||
@@ -48,7 +52,7 @@
|
||||
"schedule": [
|
||||
"* 22-23,0-5 8 */3 *"
|
||||
],
|
||||
"enabled": false
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "Majors: 15th of every 3 months, Madrid overnight window",
|
||||
@@ -58,6 +62,13 @@
|
||||
"schedule": [
|
||||
"* 22-23,0-5 15 */3 *"
|
||||
],
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "gh-aw compiled lock files - generated by 'gh aw compile', action pins must match the compiler version, never bump directly",
|
||||
"matchFileNames": [
|
||||
".github/workflows/*.lock.yml"
|
||||
],
|
||||
"enabled": false
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rename changelog fragments to their PR number before running towncrier.
|
||||
|
||||
For every <slug>.<type>.md in <component_dir>/changelog.d/, find the commit that
|
||||
added it, resolve its PR via the GitHub API (falling back to the squash-commit
|
||||
subject), and `git mv` it to <PR>.<type>.md so towncrier renders the PR link.
|
||||
Unresolvable fragments become +<slug>.<type>.md orphans (rendered without link).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
FRAGMENT_RE = re.compile(
|
||||
r"^(?P<slug>[A-Za-z0-9][A-Za-z0-9._-]*?)"
|
||||
r"\.(?P<type>added|changed|deprecated|removed|fixed|security)"
|
||||
r"(?:\.(?P<counter>[0-9]+))?\.md$"
|
||||
)
|
||||
SUBJECT_PR_RE = re.compile(r" \(#([0-9]+)\)$")
|
||||
IGNORED_FILES = {".gitkeep", "README.md"}
|
||||
API_TIMEOUT_SECONDS = 10
|
||||
|
||||
|
||||
def git(*args: str) -> str:
|
||||
result = subprocess.run(["git", *args], check=True, capture_output=True, text=True)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def find_adding_commit(path: str) -> str | None:
|
||||
"""Find the commit that added a file, following renames.
|
||||
|
||||
Falls back to a plain (no --follow) lookup: rename detection can lose the
|
||||
add event for degenerate content (e.g. files identical to many others).
|
||||
"""
|
||||
sha = git("log", "--follow", "--diff-filter=A", "--format=%H", "-1", "--", path)
|
||||
if not sha:
|
||||
sha = git("log", "--diff-filter=A", "--format=%H", "-1", "--", path)
|
||||
return sha or None
|
||||
|
||||
|
||||
def pr_from_api(repo: str, sha: str) -> int | None:
|
||||
"""Resolve the PR associated with a commit via the GitHub API.
|
||||
|
||||
Returns None on any network/API failure so the caller can fall back to
|
||||
parsing the squash-commit subject.
|
||||
"""
|
||||
url = f"https://api.github.com/repos/{repo}/commits/{sha}/pulls"
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"User-Agent": "prowler-changelog-attribution",
|
||||
}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
request = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=API_TIMEOUT_SECONDS) as response:
|
||||
pulls = json.load(response)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
return None
|
||||
if isinstance(pulls, list) and pulls:
|
||||
return pulls[0].get("number")
|
||||
return None
|
||||
|
||||
|
||||
def pr_from_subject(sha: str) -> int | None:
|
||||
subject = git("log", "-1", "--format=%s", sha)
|
||||
match = SUBJECT_PR_RE.search(subject)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def unique_destination(directory: str, base_name: str, fragment_type: str) -> str:
|
||||
"""Return a non-colliding fragment path, appending a numeric counter if needed."""
|
||||
candidate = os.path.join(directory, f"{base_name}.{fragment_type}.md")
|
||||
counter = 0
|
||||
while os.path.exists(candidate):
|
||||
counter += 1
|
||||
candidate = os.path.join(directory, f"{base_name}.{fragment_type}.{counter}.md")
|
||||
return candidate
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("component_dir", help="Component directory, e.g. prowler")
|
||||
parser.add_argument("--repo", default="prowler-cloud/prowler")
|
||||
parser.add_argument(
|
||||
"--no-api",
|
||||
action="store_true",
|
||||
help="Skip the GitHub API and resolve PRs from commit subjects only",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
fragments_dir = os.path.join(args.component_dir, "changelog.d")
|
||||
if not os.path.isdir(fragments_dir):
|
||||
print(f"::error::Fragments directory not found: {fragments_dir}")
|
||||
return 1
|
||||
|
||||
malformed = []
|
||||
to_process = []
|
||||
for name in sorted(os.listdir(fragments_dir)):
|
||||
if name in IGNORED_FILES or name.startswith("+"):
|
||||
continue
|
||||
match = FRAGMENT_RE.match(name)
|
||||
if not match:
|
||||
malformed.append(name)
|
||||
continue
|
||||
if match.group("slug").isdigit():
|
||||
continue
|
||||
to_process.append((name, match))
|
||||
|
||||
if malformed:
|
||||
for name in malformed:
|
||||
print(
|
||||
f"::error::Malformed fragment filename in {fragments_dir}: {name} "
|
||||
"(expected <slug>.<type>.md with type one of added|changed|"
|
||||
"deprecated|removed|fixed|security)"
|
||||
)
|
||||
return 1
|
||||
|
||||
for name, match in to_process:
|
||||
slug, fragment_type = match.group("slug"), match.group("type")
|
||||
|
||||
path = os.path.join(fragments_dir, name)
|
||||
sha = find_adding_commit(path)
|
||||
pr_number = None
|
||||
if sha:
|
||||
if not args.no_api:
|
||||
pr_number = pr_from_api(args.repo, sha)
|
||||
if pr_number is None:
|
||||
pr_number = pr_from_subject(sha)
|
||||
|
||||
if pr_number is not None:
|
||||
destination = unique_destination(
|
||||
fragments_dir, str(pr_number), fragment_type
|
||||
)
|
||||
else:
|
||||
destination = unique_destination(fragments_dir, f"+{slug}", fragment_type)
|
||||
print(
|
||||
f"::warning::Could not resolve a PR for {path}; renamed to "
|
||||
f"{os.path.basename(destination)} (entry will render without a PR link)"
|
||||
)
|
||||
git("mv", path, destination)
|
||||
print(f"{path} -> {destination}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,13 @@
|
||||
{% set category_order = definitions.keys() %}
|
||||
{% for section, _ in sections.items() %}
|
||||
{% for category in category_order if category in sections[section] %}
|
||||
### {{ definitions[category]['name'] }}
|
||||
|
||||
{% for text, values in sections[section][category].items() -%}
|
||||
- {{ text }}{% if values %} {{ values|join(', ') }}{% endif %}{{ "\n" }}
|
||||
{%- endfor %}
|
||||
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
---
|
||||
{{ "\n" }}
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -62,6 +62,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -9,6 +9,7 @@ on:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-codeql.yml'
|
||||
- '.github/codeql/api-codeql-config.yml'
|
||||
- '!api/CHANGELOG.md'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
@@ -17,6 +18,7 @@ on:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-codeql.yml'
|
||||
- '.github/codeql/api-codeql-config.yml'
|
||||
- '!api/CHANGELOG.md'
|
||||
schedule:
|
||||
- cron: '00 12 * * *'
|
||||
|
||||
@@ -44,7 +46,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -175,7 +175,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -215,7 +215,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
@@ -236,7 +236,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -272,27 +272,3 @@ jobs:
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
|
||||
- name: Trigger API deployment
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
event-type: api-prowler-deployment
|
||||
client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}'
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -92,6 +92,8 @@ jobs:
|
||||
_http._tcp.deb.debian.org:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
get.trivy.dev:443
|
||||
raw.githubusercontent.com:443
|
||||
releases.astral.sh:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -108,6 +110,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -77,6 +77,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17@sha256:2cd82735a36356842d5eb1ef80db3ae8f1154172f0f653db48fde079b2a0b7f7
|
||||
image: postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e
|
||||
env:
|
||||
POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
|
||||
POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
valkey:
|
||||
image: valkey/valkey:7-alpine3.19
|
||||
image: valkey/valkey:7-alpine3.19@sha256:4054fe7fc607b9326ac7c4691ed26e9670d2ff17a9fb28c2577adecf928acbcc
|
||||
env:
|
||||
VALKEY_HOST: ${{ env.VALKEY_HOST }}
|
||||
VALKEY_PORT: ${{ env.VALKEY_PORT }}
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -111,6 +111,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
patch_version: ${{ steps.detect.outputs.patch_version }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -202,7 +202,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -307,7 +307,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
name: 'Tools: Check Test Init Files'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-test-init-files:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check for __init__.py files in test directories
|
||||
run: python3 scripts/check_test_init_files.py .
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -0,0 +1,527 @@
|
||||
name: 'Tools: Compile Changelogs'
|
||||
|
||||
run-name: 'Compile changelogs for Prowler ${{ inputs.prowler_version }}'
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
prowler_version:
|
||||
description: 'Prowler version being released (e.g., 5.31.0)'
|
||||
required: true
|
||||
type: string
|
||||
target_branch:
|
||||
description: 'Branch to compile on (master for minor releases, v5.X for patches)'
|
||||
required: true
|
||||
type: string
|
||||
sdk_version:
|
||||
description: 'SDK version override (empty = mirrors prowler_version; "skip" = hold this component back)'
|
||||
required: false
|
||||
type: string
|
||||
api_version:
|
||||
description: 'API version override (empty = auto-derive 1.<prowler_minor + 1>.<prowler_patch>; "skip" = hold back)'
|
||||
required: false
|
||||
type: string
|
||||
ui_version:
|
||||
description: 'UI version override (empty = auto-derive 1.<prowler_minor>.<prowler_patch>; "skip" = hold back)'
|
||||
required: false
|
||||
type: string
|
||||
mcp_version:
|
||||
description: 'MCP Server version override (empty = auto-derive from pending fragment types; "skip" = hold back)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ inputs.prowler_version }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ inputs.prowler_version }}
|
||||
TARGET_BRANCH: ${{ inputs.target_branch }}
|
||||
SDK_VERSION: ${{ inputs.sdk_version }}
|
||||
API_VERSION: ${{ inputs.api_version }}
|
||||
UI_VERSION: ${{ inputs.ui_version }}
|
||||
MCP_VERSION: ${{ inputs.mcp_version }}
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
compile-changelogs:
|
||||
if: github.event_name == 'workflow_dispatch' && github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Block outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ inputs.target_branch }}
|
||||
fetch-depth: 0 # PR attribution resolves each fragment's adding commit from history
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install towncrier
|
||||
run: pip install --no-cache-dir towncrier==25.8.0
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.name 'prowler-bot'
|
||||
git config --global user.email '179230569+prowler-bot@users.noreply.github.com'
|
||||
|
||||
- name: Validate version inputs
|
||||
run: |
|
||||
if [[ ! "$PROWLER_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::Invalid prowler_version syntax: '$PROWLER_VERSION' (must be N.N.N)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$TARGET_BRANCH" != "master" ] && [[ ! "$TARGET_BRANCH" =~ ^v[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::Invalid target_branch syntax: '$TARGET_BRANCH' (must be 'master' or vN.N, e.g. v5.31)"
|
||||
exit 1
|
||||
fi
|
||||
IFS=. read -r prowler_major prowler_minor prowler_patch <<< "$PROWLER_VERSION"
|
||||
prowler_major=$((10#$prowler_major))
|
||||
prowler_minor=$((10#$prowler_minor))
|
||||
prowler_patch=$((10#$prowler_patch))
|
||||
if [ "$prowler_patch" -eq 0 ]; then
|
||||
if [ "$TARGET_BRANCH" != "master" ]; then
|
||||
echo "::error::target_branch must be 'master' for Prowler ${PROWLER_VERSION}; got '${TARGET_BRANCH}'"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
expected_target_branch="v${prowler_major}.${prowler_minor}"
|
||||
if [ "$TARGET_BRANCH" != "$expected_target_branch" ]; then
|
||||
echo "::error::target_branch must be '${expected_target_branch}' for Prowler ${PROWLER_VERSION}; got '${TARGET_BRANCH}'"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
for pair in "sdk_version:$SDK_VERSION" "api_version:$API_VERSION" "ui_version:$UI_VERSION" "mcp_version:$MCP_VERSION"; do
|
||||
input_name="${pair%%:*}"
|
||||
input_value="${pair#*:}"
|
||||
if [ -n "$input_value" ] && [ "$input_value" != "skip" ] && [[ ! "$input_value" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::Invalid $input_name syntax: '$input_value' (must be N.N.N, empty for auto-derivation, or 'skip')"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Compile changelogs
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
component_input() {
|
||||
case "$1" in
|
||||
prowler) echo "$SDK_VERSION" ;;
|
||||
api) echo "$API_VERSION" ;;
|
||||
ui) echo "$UI_VERSION" ;;
|
||||
mcp_server) echo "$MCP_VERSION" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
version_key() {
|
||||
local version="$1"
|
||||
local major minor patch
|
||||
IFS=. read -r major minor patch <<< "$version"
|
||||
printf '%06d.%06d.%06d' "$((10#$major))" "$((10#$minor))" "$((10#$patch))"
|
||||
}
|
||||
|
||||
pending_fragments() {
|
||||
find "$1/changelog.d" -maxdepth 1 -type f ! -name '.gitkeep' ! -name 'README.md' | sort
|
||||
}
|
||||
|
||||
# The component's last released version is the first stamped heading
|
||||
# of its CHANGELOG.md, the same source prepare-release.yml greps.
|
||||
latest_released_version() {
|
||||
grep -m1 -E '^## \[v?[0-9]+\.[0-9]+\.[0-9]+\]' "$1/CHANGELOG.md" | sed -E 's/^## \[v?([0-9]+\.[0-9]+\.[0-9]+)\].*/\1/'
|
||||
}
|
||||
|
||||
has_removed_fragments() {
|
||||
echo "$1" | grep -qE '\.removed(\.[0-9]+)?\.md$'
|
||||
}
|
||||
|
||||
# Resolve every component's effective version before compiling
|
||||
# anything, so a wrong input cannot leave the tree half-compiled.
|
||||
# Empty input = auto-derive (latest released version + semver bump
|
||||
# from the pending fragment types). 'skip' = hold the component back.
|
||||
errors=0
|
||||
compiling=""
|
||||
for component in prowler api ui mcp_server; do
|
||||
input=$(component_input "$component")
|
||||
fragments=$(pending_fragments "$component")
|
||||
|
||||
if [ "$input" = "skip" ]; then
|
||||
if [ -n "$fragments" ]; then
|
||||
echo "::warning::${component}: held back by request; these pending fragments stay for a future release:"
|
||||
echo "$fragments"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ -n "$input" ] && [ -z "$fragments" ]; then
|
||||
echo "::error::${component}: version input '$input' provided but ${component}/changelog.d/ has no pending fragments (wrong input?)"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
if [ -z "$fragments" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
removed_fragments=false
|
||||
if has_removed_fragments "$fragments"; then
|
||||
removed_fragments=true
|
||||
fi
|
||||
current=$(latest_released_version "$component")
|
||||
if [ -z "$current" ]; then
|
||||
echo "::error::${component}: could not read the latest released version from ${component}/CHANGELOG.md; restore the released heading before compiling"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ -n "$input" ]; then
|
||||
effective="$input"
|
||||
mode="explicit"
|
||||
current_key=$(version_key "$current")
|
||||
effective_key=$(version_key "$effective")
|
||||
if [[ "$effective_key" < "$current_key" || "$effective_key" == "$current_key" ]]; then
|
||||
echo "::error::${component}: explicit version '${effective}' must be greater than the latest released version (${current})"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
else
|
||||
if [ "$removed_fragments" = "true" ]; then
|
||||
echo "::error::${component}: pending 'removed' fragments imply a major bump (breaking change); provide its version input explicitly"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
# SDK, UI, and API versions are deterministic mirrors of the
|
||||
# Prowler version (the scheme bump-version.yml codifies): the SDK
|
||||
# mirrors it directly, the UI tracks 1.<minor>.<patch>, and the
|
||||
# API is the independent 1.<minor + 1>.<patch> stream. Only the
|
||||
# MCP Server has its own cadence, derived from fragment types.
|
||||
IFS=. read -r _ prowler_minor prowler_patch <<< "$PROWLER_VERSION"
|
||||
prowler_minor=$((10#$prowler_minor))
|
||||
prowler_patch=$((10#$prowler_patch))
|
||||
case "$component" in
|
||||
prowler) effective="$PROWLER_VERSION" ;;
|
||||
ui) effective="1.${prowler_minor}.${prowler_patch}" ;;
|
||||
api) effective="1.$((prowler_minor + 1)).${prowler_patch}" ;;
|
||||
mcp_server)
|
||||
IFS=. read -r major minor patch <<< "$current"
|
||||
major=$((10#$major))
|
||||
minor=$((10#$minor))
|
||||
patch=$((10#$patch))
|
||||
# Prowler patch releases (vN.N target) are maintenance
|
||||
# releases, so the MCP Server bumps patch regardless of
|
||||
# fragment types; a deliberate exception needs the explicit
|
||||
# version input.
|
||||
if [ "$TARGET_BRANCH" != "master" ]; then
|
||||
effective="${major}.${minor}.$((patch + 1))"
|
||||
if echo "$fragments" | grep -qE '\.(added|deprecated)(\.[0-9]+)?\.md$'; then
|
||||
echo "::warning::${component}: 'added'/'deprecated' fragments are shipping in a Prowler patch; auto-derived a patch bump (${current} -> ${effective}), pass the version input to override"
|
||||
fi
|
||||
elif echo "$fragments" | grep -qE '\.(added|changed|deprecated)(\.[0-9]+)?\.md$'; then
|
||||
effective="${major}.$((minor + 1)).0"
|
||||
else
|
||||
effective="${major}.${minor}.$((patch + 1))"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
current_key=$(version_key "$current")
|
||||
effective_key=$(version_key "$effective")
|
||||
if [[ "$effective_key" < "$current_key" || "$effective_key" == "$current_key" ]]; then
|
||||
echo "::error::${component}: auto-derived version '${effective}' is not greater than the latest released version (${current}); check prowler_version or pass the version input explicitly"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
mode="auto"
|
||||
echo "::notice::${component}: version auto-derived ${current} -> ${effective}"
|
||||
fi
|
||||
|
||||
if [ "$removed_fragments" = "true" ]; then
|
||||
IFS=. read -r current_major _ <<< "$current"
|
||||
current_major=$((10#$current_major))
|
||||
IFS=. read -r effective_major effective_minor effective_patch <<< "$effective"
|
||||
effective_major=$((10#$effective_major))
|
||||
effective_minor=$((10#$effective_minor))
|
||||
effective_patch=$((10#$effective_patch))
|
||||
if [ "$effective_major" -le "$current_major" ] || [ "$effective_minor" -ne 0 ] || [ "$effective_patch" -ne 0 ]; then
|
||||
echo "::error::${component}: removed fragments require a major component release (${current} -> X.0.0 with X > ${current_major}); got ${effective}"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Without the marker the build would insert the new block above the
|
||||
# file header instead of below it.
|
||||
if ! grep -q '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md"; then
|
||||
echo "::error::${component}/CHANGELOG.md is missing the '<!-- changelog: release notes start -->' marker; restore it after the intro line before compiling"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
# A hand-written UNRELEASED block means someone followed the old
|
||||
# convention; its entries would be left out of the compiled block
|
||||
# and out of the release notes extraction.
|
||||
if grep -q '(Prowler UNRELEASED)' "$component/CHANGELOG.md"; then
|
||||
echo "::error::${component}/CHANGELOG.md contains a hand-written '(Prowler UNRELEASED)' block; convert its entries to fragments in ${component}/changelog.d/ and delete the block before compiling"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "${effective} ${mode}" > "${RUNNER_TEMP}/version-${component}.txt"
|
||||
compiling="${compiling}${component} "
|
||||
done
|
||||
if [ "$errors" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$compiling" ]; then
|
||||
echo "::error::Nothing to compile: no component has pending fragments to release"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body_file="${RUNNER_TEMP}/compile-changelogs-pr-body.md"
|
||||
{
|
||||
echo "### Description"
|
||||
echo ""
|
||||
echo "Compiles the pending changelog fragments into the per-component \`CHANGELOG.md\` files for Prowler v${PROWLER_VERSION}, replacing the manual stamping PR."
|
||||
echo ""
|
||||
echo "| Component | Version | Fragments consumed |"
|
||||
echo "|---|---|---|"
|
||||
} > "$body_file"
|
||||
|
||||
compiled_components=""
|
||||
for component in prowler api ui mcp_server; do
|
||||
if [ ! -f "${RUNNER_TEMP}/version-${component}.txt" ]; then
|
||||
echo "Skipping ${component} (no pending fragments or held back)"
|
||||
echo "| \`${component}\` | - | 0 |" >> "$body_file"
|
||||
continue
|
||||
fi
|
||||
read -r version mode < "${RUNNER_TEMP}/version-${component}.txt"
|
||||
version_label="$version"
|
||||
if [ "$mode" = "auto" ]; then
|
||||
version_label="${version} (auto)"
|
||||
fi
|
||||
|
||||
count=$(pending_fragments "$component" | wc -l | tr -d ' ')
|
||||
echo "Compiling ${component} ${version} (${count} fragments, ${mode} version)..."
|
||||
|
||||
# Captured before attribution renames them: these original paths are
|
||||
# what the forward-sync deletes on master (backports copy fragments
|
||||
# verbatim, so filenames match across branches).
|
||||
pending_fragments "$component" > "${RUNNER_TEMP}/consumed-${component}.txt"
|
||||
pre_lines=$(wc -l < "$component/CHANGELOG.md")
|
||||
|
||||
# Attribution must run before the build: towncrier renders the
|
||||
# first dotted segment of each filename as the PR number.
|
||||
python .github/scripts/changelog_attribution.py "$component"
|
||||
towncrier build --config "$component/towncrier.toml" --version "$version" --name "Prowler v${PROWLER_VERSION}" --yes
|
||||
|
||||
# The build only inserts lines right after the marker, so the new
|
||||
# stamped block is exactly the added lines following it. Captured
|
||||
# for the forward-sync to master.
|
||||
post_lines=$(wc -l < "$component/CHANGELOG.md")
|
||||
delta=$((post_lines - pre_lines))
|
||||
marker_line=$(grep -n -m1 '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md" | cut -d: -f1)
|
||||
sed -n "$((marker_line + 1)),$((marker_line + delta))p" "$component/CHANGELOG.md" > "${RUNNER_TEMP}/block-${component}.md"
|
||||
|
||||
compiled_components="${compiled_components}${component} "
|
||||
echo "| \`${component}\` | ${version_label} | ${count} |" >> "$body_file"
|
||||
done
|
||||
echo "COMPILED_COMPONENTS=${compiled_components}" >> "$GITHUB_ENV"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "Review that no pending fragment was dropped (the diff must delete every consumed fragment) and that each new version block is correct, then squash-merge."
|
||||
echo ""
|
||||
echo "### License"
|
||||
echo ""
|
||||
echo "By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license."
|
||||
} >> "$body_file"
|
||||
|
||||
echo "PR_BODY_FILE=${body_file}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Create compile PR
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
commit-message: 'chore(changelog): v${{ env.PROWLER_VERSION }}'
|
||||
branch: compile-changelogs-${{ env.PROWLER_VERSION }}
|
||||
base: ${{ env.TARGET_BRANCH }}
|
||||
title: 'chore(changelog): v${{ env.PROWLER_VERSION }}'
|
||||
body-path: ${{ env.PR_BODY_FILE }}
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
labels: |
|
||||
no-changelog
|
||||
skip-sync
|
||||
|
||||
# Patch compiles (target_branch = v5.X) leave master holding the consumed
|
||||
# fragments and missing the new version block. This applies the equivalent
|
||||
# change to master: insert the same stamped blocks under the marker and
|
||||
# delete the consumed fragments, so the next minor compile cannot
|
||||
# re-release entries that already shipped in the patch.
|
||||
- name: Apply forward-sync to master
|
||||
if: env.TARGET_BRANCH != 'master'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
git checkout -B master origin/master
|
||||
|
||||
version_key() {
|
||||
local version="$1"
|
||||
local major minor patch
|
||||
IFS=. read -r major minor patch <<< "$version"
|
||||
printf '%06d.%06d.%06d' "$((10#$major))" "$((10#$minor))" "$((10#$patch))"
|
||||
}
|
||||
|
||||
release_from_heading() {
|
||||
local heading="$1"
|
||||
echo "$heading" | sed -E 's/^## \[[^]]+\] \(Prowler v?([0-9]+\.[0-9]+\.[0-9]+)\).*/\1/'
|
||||
}
|
||||
|
||||
insert_changelog_block_ordered() {
|
||||
local component="$1"
|
||||
local block_file="$2"
|
||||
local changelog="${component}/CHANGELOG.md"
|
||||
local incoming_heading incoming_release incoming_key
|
||||
local marker_line insertion_line duplicate_line total_lines
|
||||
local line heading existing_release existing_key
|
||||
|
||||
marker_line=$(grep -n -m1 '^<!-- changelog: release notes start -->$' "$changelog" | cut -d: -f1)
|
||||
incoming_heading=$(grep -m1 -E '^## \[[^]]+\] \(Prowler v?[0-9]+\.[0-9]+\.[0-9]+\)' "$block_file" || true)
|
||||
if [ -z "$incoming_heading" ]; then
|
||||
echo "::error::${block_file} does not contain a stamped Prowler release heading"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
incoming_release=$(release_from_heading "$incoming_heading")
|
||||
incoming_key=$(version_key "$incoming_release")
|
||||
insertion_line=""
|
||||
duplicate_line=""
|
||||
|
||||
while IFS=: read -r line heading; do
|
||||
existing_release=$(release_from_heading "$heading")
|
||||
existing_key=$(version_key "$existing_release")
|
||||
if [[ "$incoming_key" == "$existing_key" ]]; then
|
||||
duplicate_line="$line"
|
||||
break
|
||||
fi
|
||||
if [[ "$incoming_key" > "$existing_key" ]]; then
|
||||
insertion_line="$line"
|
||||
break
|
||||
fi
|
||||
done < <(grep -n -E '^## \[[^]]+\] \(Prowler v?[0-9]+\.[0-9]+\.[0-9]+\)' "$changelog" || true)
|
||||
|
||||
if [ -n "$duplicate_line" ]; then
|
||||
echo "::error::${changelog} already contains a block for Prowler v${incoming_release} at line ${duplicate_line}; refusing to insert a duplicate"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$insertion_line" ]; then
|
||||
insertion_line=$(($(wc -l < "$changelog") + 1))
|
||||
fi
|
||||
if [ "$insertion_line" -le "$marker_line" ]; then
|
||||
insertion_line=$((marker_line + 1))
|
||||
fi
|
||||
|
||||
# The captured block window can be off by one blank line on either
|
||||
# end (towncrier re-emits the blank after the marker), so strip the
|
||||
# outer blank lines and pad exactly one on each side: the block
|
||||
# must never glue to the marker above or the next heading below.
|
||||
awk '
|
||||
/[^[:space:]]/ { for (i = 0; i < pending; i++) print ""; pending = 0; print; started = 1; next }
|
||||
started { pending++ }
|
||||
' "$block_file" > "${RUNNER_TEMP}/block-normalized.md"
|
||||
|
||||
total_lines=$(wc -l < "$changelog")
|
||||
{
|
||||
head -n "$((insertion_line - 1))" "$changelog"
|
||||
if [ "$insertion_line" -gt 1 ] && [ -n "$(sed -n "$((insertion_line - 1))p" "$changelog")" ]; then
|
||||
echo ""
|
||||
fi
|
||||
cat "${RUNNER_TEMP}/block-normalized.md"
|
||||
if [ "$insertion_line" -le "$total_lines" ]; then
|
||||
echo ""
|
||||
fi
|
||||
tail -n +"$insertion_line" "$changelog"
|
||||
} > "${RUNNER_TEMP}/changelog.tmp"
|
||||
mv "${RUNNER_TEMP}/changelog.tmp" "$changelog"
|
||||
|
||||
echo "::notice::Inserted ${component} changelog block for Prowler v${incoming_release} at line ${insertion_line}"
|
||||
}
|
||||
|
||||
sync_body="${RUNNER_TEMP}/forward-sync-pr-body.md"
|
||||
{
|
||||
echo "### Description"
|
||||
echo ""
|
||||
echo "Forward-syncs the v${PROWLER_VERSION} compiled changelogs from \`${TARGET_BRANCH}\` to \`master\`: inserts the same stamped version blocks under the insertion marker and deletes the consumed fragments, so the next minor compile cannot re-release entries that already shipped in this patch. Opened automatically by the same run that opened the compile PR; review and squash-merge after it."
|
||||
echo ""
|
||||
echo "| Component | Fragments deleted on master | Skipped (only on ${TARGET_BRANCH}) |"
|
||||
echo "|---|---|---|"
|
||||
} > "$sync_body"
|
||||
|
||||
for component in $COMPILED_COMPONENTS; do
|
||||
block_file="${RUNNER_TEMP}/block-${component}.md"
|
||||
consumed_file="${RUNNER_TEMP}/consumed-${component}.txt"
|
||||
|
||||
if ! grep -qm1 '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md"; then
|
||||
echo "::error::${component}/CHANGELOG.md on master is missing the insertion marker; cannot forward-sync"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
deleted=0
|
||||
skipped=0
|
||||
while IFS= read -r fragment; do
|
||||
if [ -z "$fragment" ]; then
|
||||
continue
|
||||
fi
|
||||
if [ -f "$fragment" ]; then
|
||||
git rm -q "$fragment"
|
||||
deleted=$((deleted + 1))
|
||||
else
|
||||
echo "::notice::${fragment} does not exist on master (change landed only on ${TARGET_BRANCH}); skipping its deletion"
|
||||
skipped=$((skipped + 1))
|
||||
fi
|
||||
done < "$consumed_file"
|
||||
|
||||
insert_changelog_block_ordered "$component" "$block_file"
|
||||
|
||||
echo "| \`${component}\` | ${deleted} | ${skipped} |" >> "$sync_body"
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### License"
|
||||
echo ""
|
||||
echo "By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license."
|
||||
} >> "$sync_body"
|
||||
|
||||
echo "SYNC_BODY_FILE=${sync_body}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Create forward-sync PR
|
||||
if: env.TARGET_BRANCH != 'master'
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
commit-message: 'chore(changelog): v${{ env.PROWLER_VERSION }} forward-sync to master'
|
||||
branch: forward-sync-changelogs-${{ env.PROWLER_VERSION }}
|
||||
base: master
|
||||
title: 'chore(changelog): v${{ env.PROWLER_VERSION }} forward-sync to master'
|
||||
body-path: ${{ env.SYNC_BODY_FILE }}
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
labels: |
|
||||
no-changelog
|
||||
skip-sync
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
name: 'Docs: Check Provider Cards Snippet'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'v5.*'
|
||||
paths:
|
||||
- 'docs/user-guide/providers/**/getting-started-*.mdx'
|
||||
- 'docs/scripts/generate_provider_cards.py'
|
||||
- 'docs/snippets/provider-cards.mdx'
|
||||
- 'api/src/backend/api/models.py'
|
||||
- '.github/workflows/docs-check-provider-cards.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-provider-cards:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Verify provider cards snippet is up to date
|
||||
run: |
|
||||
if ! python3 docs/scripts/generate_provider_cards.py; then
|
||||
echo "::error::docs/snippets/provider-cards.mdx is out of sync with the provider getting-started pages or the API ProviderChoices enum."
|
||||
echo "Run 'python3 docs/scripts/generate_provider_cards.py' locally and commit the regenerated snippet."
|
||||
echo "--- diff ---"
|
||||
git diff docs/snippets/provider-cards.mdx
|
||||
exit 1
|
||||
fi
|
||||
@@ -25,14 +25,15 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
# We can't block as Trufflehog needs to verify secrets against vendors
|
||||
egress-policy: audit
|
||||
# allowed-endpoints: >
|
||||
# github.com:443
|
||||
# ghcr.io:443
|
||||
# pkg-containers.githubusercontent.com:443
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
ghcr.io:443
|
||||
pkg-containers.githubusercontent.com:443
|
||||
www.formbucket.com:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
Generated
+1301
-703
File diff suppressed because one or more lines are too long
@@ -12,8 +12,8 @@ if: contains(toJson(github.event.issue.labels), 'status/needs-triage')
|
||||
|
||||
timeout-minutes: 12
|
||||
|
||||
rate-limit:
|
||||
max: 5
|
||||
user-rate-limit:
|
||||
max-runs-per-window: 5
|
||||
window: 60
|
||||
|
||||
concurrency:
|
||||
@@ -30,6 +30,12 @@ permissions:
|
||||
engine: copilot
|
||||
strict: false
|
||||
|
||||
pre-steps:
|
||||
- name: Harden the runner
|
||||
uses: step-security/harden-runner@v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
imports:
|
||||
- ../agents/issue-triage.md
|
||||
|
||||
@@ -108,7 +114,7 @@ Triage the following GitHub issue using the Prowler Issue Triage Agent persona.
|
||||
|
||||
## Sanitized Issue Content
|
||||
|
||||
${{ needs.activation.outputs.text }}
|
||||
${{ steps.sanitized.outputs.text }}
|
||||
|
||||
## Instructions
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Add community label
|
||||
if: steps.check_membership.outputs.is_member == 'false'
|
||||
if: steps.check_membership.outputs.is_member == 'false' && github.event.pull_request.user.type != 'Bot'
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -165,7 +165,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -206,7 +206,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
@@ -227,7 +227,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -263,27 +263,3 @@ jobs:
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
|
||||
- name: Trigger MCP deployment
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
event-type: mcp-prowler-deployment
|
||||
client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}'
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -102,6 +102,7 @@ jobs:
|
||||
files_ignore: |
|
||||
mcp_server/README.md
|
||||
mcp_server/CHANGELOG.md
|
||||
mcp_server/changelog.d/**
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -83,7 +83,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -19,6 +19,60 @@ concurrency:
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test-changelog-attribution:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: git fetch --depth=1 origin "${BASE_REF}"
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
.github/scripts/changelog_attribution.py
|
||||
.github/workflows/pr-check-changelog.yml
|
||||
.github/workflows/compile-changelogs.yml
|
||||
.github/towncrier/template.md.jinja
|
||||
*/towncrier.toml
|
||||
tests/github/**
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Test changelog attribution
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: |
|
||||
python3 -m pip install --user --disable-pip-version-check pytest==9.0.3 towncrier==25.8.0
|
||||
python3 -m pytest tests/github
|
||||
|
||||
check-changelog:
|
||||
if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false
|
||||
runs-on: ubuntu-latest
|
||||
@@ -31,7 +85,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -62,53 +116,160 @@ jobs:
|
||||
uv.lock
|
||||
pyproject.toml
|
||||
|
||||
- name: Check for folder changes and changelog presence
|
||||
- name: Check for folder changes and changelog fragment presence
|
||||
id: check-folders
|
||||
run: |
|
||||
missing_changelogs=""
|
||||
fragment_name_re='^[A-Za-z0-9][A-Za-z0-9._-]*\.(added|changed|deprecated|removed|fixed|security)(\.[0-9]+)?\.md$'
|
||||
manual_pr_link_re='(\[\(#[0-9]+\)\]|\[#[0-9]+\]\(|\(#[0-9]+\)|github\.com/[^[:space:]/]+/[^[:space:]/]+/(pull|issues)/[0-9]+)'
|
||||
folder_alt=$(echo "$MONITORED_FOLDERS" | tr ' ' '|')
|
||||
|
||||
missing_fragments=""
|
||||
invalid_fragments=""
|
||||
linked_fragments=""
|
||||
handwritten_changelogs=""
|
||||
|
||||
all_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n')
|
||||
added=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" | tr ' ' '\n')
|
||||
added_or_renamed=$(printf '%s\n%s' "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES}" | tr ' ' '\n')
|
||||
added_modified_or_renamed=$(printf '%s\n%s\n%s' "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_MODIFIED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES}" | tr ' ' '\n')
|
||||
|
||||
# Returns success if the folder has a valid fragment added, modified, or renamed.
|
||||
has_changelog_update() {
|
||||
local folder="$1"
|
||||
if echo "$added_modified_or_renamed" | grep "^${folder}/changelog.d/" | sed "s|^${folder}/changelog.d/||" | grep -qE "$fragment_name_re"; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ "${STEPS_CHANGED_FILES_OUTPUTS_ANY_CHANGED}" == "true" ]]; then
|
||||
# Check monitored folders
|
||||
for folder in $MONITORED_FOLDERS; do
|
||||
# Get files changed in this folder
|
||||
changed_in_folder=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep "^${folder}/" || true)
|
||||
if echo "$all_changed" | grep -q "^${folder}/CHANGELOG.md$"; then
|
||||
echo "Direct CHANGELOG.md edits are not allowed for ${folder}/"
|
||||
handwritten_changelogs="${handwritten_changelogs}- \`${folder}/CHANGELOG.md\`"$'\n'
|
||||
fi
|
||||
|
||||
changed_in_folder=$(echo "$all_changed" | grep "^${folder}/" | grep -v "^${folder}/CHANGELOG.md$" || true)
|
||||
|
||||
if [ -n "$changed_in_folder" ]; then
|
||||
echo "Detected changes in ${folder}/"
|
||||
|
||||
# Check if CHANGELOG.md was updated
|
||||
if ! echo "$changed_in_folder" | grep -q "^${folder}/CHANGELOG.md$"; then
|
||||
echo "No changelog update found for ${folder}/"
|
||||
missing_changelogs="${missing_changelogs}- \`${folder}\`"$'\n'
|
||||
if ! has_changelog_update "$folder"; then
|
||||
echo "No changelog fragment found for ${folder}/"
|
||||
missing_fragments="${missing_fragments}- \`${folder}\`"$'\n'
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Check root-level dependency files (uv.lock, pyproject.toml)
|
||||
# These are associated with the prowler folder changelog
|
||||
root_deps_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
|
||||
root_deps_changed=$(echo "$all_changed" | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
|
||||
if [ -n "$root_deps_changed" ]; then
|
||||
echo "Detected changes in root dependency files: $root_deps_changed"
|
||||
# Check if prowler/CHANGELOG.md was already updated (might have been caught above)
|
||||
prowler_changelog_updated=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep "^prowler/CHANGELOG.md$" || true)
|
||||
if [ -z "$prowler_changelog_updated" ]; then
|
||||
if ! has_changelog_update "prowler"; then
|
||||
# Only add if prowler wasn't already flagged
|
||||
if ! echo "$missing_changelogs" | grep -q "prowler"; then
|
||||
echo "No changelog update found for root dependency changes"
|
||||
missing_changelogs="${missing_changelogs}- \`prowler\` (root dependency files changed)"$'\n'
|
||||
if ! echo "$missing_fragments" | grep -q "prowler"; then
|
||||
echo "No changelog fragment found for root dependency changes"
|
||||
missing_fragments="${missing_fragments}- \`prowler\` (root dependency files changed)"$'\n'
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Validate the filename of every fragment added by this PR
|
||||
added_fragments=$(echo "$added_or_renamed" | grep -E "^(${folder_alt})/changelog\.d/" || true)
|
||||
for fragment in $added_fragments; do
|
||||
name=$(basename "$fragment")
|
||||
if [ "$name" = ".gitkeep" ] || [ "$name" = "README.md" ]; then
|
||||
continue
|
||||
fi
|
||||
if ! echo "$name" | grep -qE "$fragment_name_re"; then
|
||||
echo "Invalid fragment filename: $fragment"
|
||||
invalid_fragments="${invalid_fragments}- \`${fragment}\`"$'\n'
|
||||
fi
|
||||
done
|
||||
|
||||
# Lint fragment content: the PR link is attached automatically at
|
||||
# compile time, so a hand-written PR or issue link would be wrong
|
||||
touched_fragments=$(echo "$added_modified_or_renamed" | grep -E "^(${folder_alt})/changelog\.d/" || true)
|
||||
for fragment in $touched_fragments; do
|
||||
name=$(basename "$fragment")
|
||||
if [ "$name" = ".gitkeep" ] || [ "$name" = "README.md" ] || [ ! -f "$fragment" ]; then
|
||||
continue
|
||||
fi
|
||||
if grep -qE "$manual_pr_link_re" "$fragment"; then
|
||||
echo "Fragment contains a hand-written PR or issue link: $fragment"
|
||||
linked_fragments="${linked_fragments}- \`${fragment}\`"$'\n'
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
{
|
||||
echo "missing_changelogs<<EOF"
|
||||
echo -e "${missing_changelogs}"
|
||||
echo "EOF"
|
||||
} >> $GITHUB_OUTPUT
|
||||
# Suggest a slug derived from the branch name for the bot comment
|
||||
suggested_slug=$(echo "$HEAD_REF" | tr '[:upper:]' '[:lower:]' | sed 's|.*/||; s/[^a-z0-9._-]/-/g; s/^[^a-z0-9]*//')
|
||||
if [ -z "$suggested_slug" ]; then
|
||||
suggested_slug="my-change"
|
||||
fi
|
||||
|
||||
fragment_help="A changelog fragment is a small Markdown file named \`<slug>.<type>.md\` under \`<component>/changelog.d/\`, where \`<type>\` is one of \`added\`, \`changed\`, \`deprecated\`, \`removed\`, \`fixed\` or \`security\`. Its content is the changelog entry text, without the PR link (added automatically at release time) and without a trailing period. For example:
|
||||
|
||||
\`\`\`
|
||||
echo 'Entry text describing the change' > <component>/changelog.d/${suggested_slug}.fixed.md
|
||||
\`\`\`
|
||||
|
||||
If this PR does not need a changelog entry, add the \`no-changelog\` label instead."
|
||||
|
||||
if [ -n "$missing_fragments" ] || [ -n "$invalid_fragments" ] || [ -n "$linked_fragments" ] || [ -n "$handwritten_changelogs" ]; then
|
||||
comment_body=""
|
||||
if [ -n "$missing_fragments" ]; then
|
||||
comment_body="⚠️ **Changes detected in the following folders without a changelog fragment:**"$'\n\n'"${missing_fragments}"$'\n'
|
||||
fi
|
||||
if [ -n "$invalid_fragments" ]; then
|
||||
comment_body="${comment_body}⚠️ **Changelog fragment filenames that do not follow the naming convention:**"$'\n\n'"${invalid_fragments}"$'\n'
|
||||
fi
|
||||
if [ -n "$linked_fragments" ]; then
|
||||
comment_body="${comment_body}⚠️ **Changelog fragments containing a hand-written PR or issue link (remove it; the PR link is attached automatically at release time):**"$'\n\n'"${linked_fragments}"$'\n'
|
||||
fi
|
||||
if [ -n "$handwritten_changelogs" ]; then
|
||||
comment_body="${comment_body}⚠️ **Direct \`CHANGELOG.md\` edits are not allowed in regular PRs:**"$'\n\n'"${handwritten_changelogs}"$'\n'
|
||||
fi
|
||||
comment_body="${comment_body}${fragment_help}"
|
||||
else
|
||||
comment_body="✅ All required changelog fragments are present."
|
||||
fi
|
||||
|
||||
write_multiline_output() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
local delimiter
|
||||
|
||||
while true; do
|
||||
delimiter="EOF_$(openssl rand -hex 16)"
|
||||
if ! grep -qxF "$delimiter" <<< "$value"; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo "${name}<<${delimiter}"
|
||||
if [ -n "$value" ]; then
|
||||
printf '%s\n' "$value"
|
||||
fi
|
||||
echo "${delimiter}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
}
|
||||
|
||||
write_multiline_output "missing_fragments" "$missing_fragments"
|
||||
write_multiline_output "invalid_fragments" "$invalid_fragments"
|
||||
write_multiline_output "linked_fragments" "$linked_fragments"
|
||||
write_multiline_output "handwritten_changelogs" "$handwritten_changelogs"
|
||||
write_multiline_output "comment_body" "$comment_body"
|
||||
env:
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ANY_CHANGED: ${{ steps.changed-files.outputs.any_changed }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_MODIFIED_FILES: ${{ steps.changed-files.outputs.modified_files }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES: ${{ steps.changed-files.outputs.renamed_files }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
|
||||
- name: Find existing changelog comment
|
||||
if: github.event.pull_request.head.repo.full_name == github.repository
|
||||
@@ -128,14 +289,10 @@ jobs:
|
||||
edit-mode: replace
|
||||
body: |
|
||||
<!-- changelog-check -->
|
||||
${{ steps.check-folders.outputs.missing_changelogs != '' && format('⚠️ **Changes detected in the following folders without a corresponding update to the `CHANGELOG.md`:**
|
||||
${{ steps.check-folders.outputs.comment_body }}
|
||||
|
||||
{0}
|
||||
|
||||
Please add an entry to the corresponding `CHANGELOG.md` file to maintain a clear history of changes.', steps.check-folders.outputs.missing_changelogs) || '✅ All necessary `CHANGELOG.md` files have been updated.' }}
|
||||
|
||||
- name: Fail if changelog is missing
|
||||
if: steps.check-folders.outputs.missing_changelogs != ''
|
||||
- name: Fail if changelog fragment is missing or invalid
|
||||
if: steps.check-folders.outputs.missing_fragments != '' || steps.check-folders.outputs.invalid_fragments != '' || steps.check-folders.outputs.linked_fragments != '' || steps.check-folders.outputs.handwritten_changelogs != ''
|
||||
run: |
|
||||
echo "::error::Missing changelog updates in some folders"
|
||||
echo "::error::Missing, invalid, or disallowed changelog updates"
|
||||
exit 1
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -37,8 +37,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 1
|
||||
# zizmor: ignore[artipacked]
|
||||
persist-credentials: true # Required by tj-actions/changed-files to fetch PR branch
|
||||
persist-credentials: false # No write token in the untrusted PR-head tree; public repo so base fetch/changed-files work unauthenticated
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
@@ -50,6 +49,8 @@ jobs:
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: '**'
|
||||
safe_output: false # Raw paths (list read via env var, injection-safe); default escaping backslash-quotes chars like () and breaks the -f test
|
||||
separator: "\n" # Newline-delimited so the reader tolerates spaces and glob chars in paths
|
||||
|
||||
- name: Check for conflict markers
|
||||
id: conflict-check
|
||||
@@ -59,19 +60,18 @@ jobs:
|
||||
CONFLICT_FILES=""
|
||||
HAS_CONFLICTS=false
|
||||
|
||||
# Check each changed file for conflict markers
|
||||
for file in ${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}; do
|
||||
if [ -f "$file" ]; then
|
||||
echo "Checking file: $file"
|
||||
# Read newline-delimited paths so spaces/globs neither word-split nor glob-expand
|
||||
while IFS= read -r file; do
|
||||
[ -n "$file" ] || continue
|
||||
[ -f "$file" ] || continue
|
||||
echo "Checking file: $file"
|
||||
|
||||
# Look for conflict markers (more precise regex)
|
||||
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
|
||||
echo "Conflict markers found in: $file"
|
||||
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
|
||||
HAS_CONFLICTS=true
|
||||
fi
|
||||
if grep -qE '^(<<<<<<<|=======|>>>>>>>)' "$file" 2>/dev/null; then
|
||||
echo "Conflict markers found in: $file"
|
||||
CONFLICT_FILES="${CONFLICT_FILES}- \`${file}\`"$'\n'
|
||||
HAS_CONFLICTS=true
|
||||
fi
|
||||
done
|
||||
done <<< "$STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES"
|
||||
|
||||
if [ "$HAS_CONFLICTS" = true ]; then
|
||||
echo "has_conflicts=true" >> $GITHUB_OUTPUT
|
||||
@@ -88,18 +88,49 @@ jobs:
|
||||
env:
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
|
||||
|
||||
- name: Check base-branch mergeability
|
||||
id: merge-check
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
MERGEABLE=null
|
||||
|
||||
# GitHub computes mergeability async, so .mergeable is null until ready; poll until resolved
|
||||
for attempt in 1 2 3 4 5; do
|
||||
MERGEABLE=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.mergeable')
|
||||
if [ "$MERGEABLE" != "null" ]; then
|
||||
break
|
||||
fi
|
||||
echo "Attempt ${attempt}: mergeability not computed yet, retrying..."
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# Keep 'unknown' distinct from 'clean' so we never assert a clean merge we could not confirm
|
||||
case "$MERGEABLE" in
|
||||
false) STATUS=conflict; echo "PR branch cannot be merged cleanly into its base branch" ;;
|
||||
true) STATUS=clean; echo "PR branch merges cleanly into its base branch" ;;
|
||||
*) STATUS=unknown; echo "::warning::Mergeability did not resolve after retries; leaving it undetermined" ;;
|
||||
esac
|
||||
|
||||
echo "merge_status=${STATUS}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Manage conflict label
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
|
||||
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
|
||||
run: |
|
||||
LABEL_NAME="has-conflicts"
|
||||
|
||||
# Add or remove label based on conflict status
|
||||
if [ "$HAS_CONFLICTS" = "true" ]; then
|
||||
if [ "$HAS_CONFLICTS" = "true" ] || [ "$MERGE_STATUS" = "conflict" ]; then
|
||||
echo "Adding conflict label to PR #${PR_NUMBER}..."
|
||||
gh pr edit "$PR_NUMBER" --add-label "$LABEL_NAME" --repo ${{ github.repository }} || true
|
||||
elif [ "$MERGE_STATUS" = "unknown" ]; then
|
||||
# Don't drop the label on an undetermined merge state; a later run will settle it
|
||||
echo "Mergeability undetermined; leaving label unchanged"
|
||||
else
|
||||
echo "Removing conflict label from PR #${PR_NUMBER}..."
|
||||
gh pr edit "$PR_NUMBER" --remove-label "$LABEL_NAME" --repo ${{ github.repository }} || true
|
||||
@@ -121,20 +152,25 @@ jobs:
|
||||
edit-mode: replace
|
||||
body: |
|
||||
<!-- conflict-checker-comment -->
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && '⚠️ **Conflict Markers Detected**' || '✅ **Conflict Markers Resolved**' }}
|
||||
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('This pull request contains unresolved conflict markers in the following files:
|
||||
${{ (steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict') && '⚠️ **Conflicts Detected**' || (steps.merge-check.outputs.merge_status == 'unknown' && 'ℹ️ **Conflict Check Incomplete**' || '✅ **No Conflicts**') }}
|
||||
${{ steps.conflict-check.outputs.has_conflicts == 'true' && format('
|
||||
**Conflict markers** are present in the following files:
|
||||
|
||||
{0}
|
||||
|
||||
Please resolve these conflicts by:
|
||||
1. Locating the conflict markers: `<<<<<<<`, `=======`, and `>>>>>>>`
|
||||
2. Manually editing the files to resolve the conflicts
|
||||
3. Removing all conflict markers
|
||||
4. Committing and pushing the changes', steps.conflict-check.outputs.conflict_files) || 'All conflict markers have been successfully resolved in this pull request.' }}
|
||||
Resolve them by removing every `<<<<<<<`, `=======`, and `>>>>>>>` marker, then commit and push.', steps.conflict-check.outputs.conflict_files) || '' }}
|
||||
${{ steps.merge-check.outputs.merge_status == 'conflict' && '
|
||||
**Merge conflict with the base branch.** This PR cannot be merged cleanly. Update your branch with the latest base (rebase or merge) and resolve the conflicts.' || '' }}
|
||||
${{ steps.merge-check.outputs.merge_status == 'unknown' && '
|
||||
GitHub had not finished computing mergeability, so base-branch conflict status could not be verified on this run.' || '' }}
|
||||
${{ (steps.conflict-check.outputs.has_conflicts != 'true' && steps.merge-check.outputs.merge_status == 'clean') && '
|
||||
No conflict markers, and the branch merges cleanly into its base.' || '' }}
|
||||
|
||||
- name: Fail workflow if conflicts detected
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true'
|
||||
if: steps.conflict-check.outputs.has_conflicts == 'true' || steps.merge-check.outputs.merge_status == 'conflict'
|
||||
env:
|
||||
HAS_CONFLICTS: ${{ steps.conflict-check.outputs.has_conflicts }}
|
||||
MERGE_STATUS: ${{ steps.merge-check.outputs.merge_status }}
|
||||
run: |
|
||||
echo "::error::Workflow failed due to conflict markers detected in the PR"
|
||||
[ "$HAS_CONFLICTS" = "true" ] && echo "::error::Conflict markers detected in changed files"
|
||||
[ "$MERGE_STATUS" = "conflict" ] && echo "::error::PR branch has merge conflicts with the base branch"
|
||||
exit 1
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -56,6 +56,6 @@ jobs:
|
||||
"PROWLER_PR_BODY": ${{ toJson(github.event.pull_request.body) }},
|
||||
"PROWLER_PR_URL": ${{ toJson(github.event.pull_request.html_url) }},
|
||||
"PROWLER_PR_MERGED_BY": "${{ github.event.pull_request.merged_by.login }}",
|
||||
"PROWLER_PR_BASE_BRANCH": "${{ github.event.pull_request.base.ref }}",
|
||||
"PROWLER_PR_HEAD_BRANCH": "${{ github.event.pull_request.head.ref }}"
|
||||
"PROWLER_PR_BASE_BRANCH": ${{ toJson(github.event.pull_request.base.ref) }},
|
||||
"PROWLER_PR_HEAD_BRANCH": ${{ toJson(github.event.pull_request.head.ref) }}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -28,12 +28,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
raw.githubusercontent.com:443
|
||||
objects.githubusercontent.com:443
|
||||
codeload.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
@@ -41,6 +42,7 @@ jobs:
|
||||
files.pythonhosted.org:443
|
||||
registry.npmjs.org:443
|
||||
nodejs.org:443
|
||||
releases.astral.sh:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -55,6 +55,7 @@ jobs:
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
|
||||
@@ -12,6 +12,7 @@ on:
|
||||
- '.github/workflows/sdk-codeql.yml'
|
||||
- '.github/codeql/sdk-codeql-config.yml'
|
||||
- '!prowler/CHANGELOG.md'
|
||||
- '!prowler/changelog.d/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
@@ -23,6 +24,7 @@ on:
|
||||
- '.github/workflows/sdk-codeql.yml'
|
||||
- '.github/codeql/sdk-codeql-config.yml'
|
||||
- '!prowler/CHANGELOG.md'
|
||||
- '!prowler/changelog.d/**'
|
||||
schedule:
|
||||
- cron: '00 12 * * *'
|
||||
|
||||
@@ -51,7 +53,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -138,15 +138,18 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
public.ecr.aws:443
|
||||
sts.amazonaws.com:443
|
||||
sts.us-east-1.amazonaws.com:443
|
||||
registry-1.docker.io:443
|
||||
production.cloudflare.docker.com:443
|
||||
production.cloudfront.docker.com:443
|
||||
@@ -173,14 +176,16 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
- name: Configure AWS credentials (OIDC)
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
|
||||
env:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_PUSH_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
with:
|
||||
registry-type: public
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
@@ -206,10 +211,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -221,6 +227,8 @@ jobs:
|
||||
github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
api.ecr-public.us-east-1.amazonaws.com:443
|
||||
sts.amazonaws.com:443
|
||||
sts.us-east-1.amazonaws.com:443
|
||||
|
||||
|
||||
- name: Login to DockerHub
|
||||
@@ -229,14 +237,16 @@ jobs:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
- name: Configure AWS credentials (OIDC)
|
||||
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
username: ${{ secrets.PUBLIC_ECR_AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.PUBLIC_ECR_AWS_SECRET_ACCESS_KEY }}
|
||||
env:
|
||||
AWS_REGION: ${{ env.AWS_REGION }}
|
||||
aws-region: us-east-1
|
||||
role-to-assume: ${{ secrets.PUBLIC_ECR_PUSH_ROLE_ARN }}
|
||||
|
||||
- name: Login to Public ECR
|
||||
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
|
||||
with:
|
||||
registry-type: public
|
||||
|
||||
- name: Create and push manifests for push event
|
||||
if: github.event_name == 'push'
|
||||
@@ -299,7 +309,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
@@ -320,7 +330,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -94,6 +94,8 @@ jobs:
|
||||
_http._tcp.deb.debian.org:443
|
||||
powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net:443
|
||||
get.trivy.dev:443
|
||||
raw.githubusercontent.com:443
|
||||
releases.astral.sh:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -113,6 +115,7 @@ jobs:
|
||||
.github/workflows/sdk-container-checks.yml
|
||||
files_ignore: |
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -73,6 +73,7 @@ jobs:
|
||||
.github/scripts/osv-scan.sh
|
||||
files_ignore: |
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -77,6 +77,7 @@ jobs:
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
@@ -614,6 +615,30 @@ jobs:
|
||||
flags: prowler-py${{ matrix.python-version }}-linode
|
||||
files: ./linode_coverage.xml
|
||||
|
||||
# E2E Networks Provider
|
||||
- name: Check if E2E Networks files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
id: changed-e2enetworks
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
./prowler/**/e2enetworks/**
|
||||
./tests/**/e2enetworks/**
|
||||
./uv.lock
|
||||
|
||||
- name: Run E2E Networks tests
|
||||
if: steps.changed-e2enetworks.outputs.any_changed == 'true'
|
||||
run: uv run pytest -n auto --cov=./prowler/providers/e2enetworks --cov-report=xml:e2enetworks_coverage.xml tests/providers/e2enetworks
|
||||
|
||||
- name: Upload E2E Networks coverage to Codecov
|
||||
if: steps.changed-e2enetworks.outputs.any_changed == 'true'
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
flags: prowler-py${{ matrix.python-version }}-e2enetworks
|
||||
files: ./e2enetworks_coverage.xml
|
||||
|
||||
# External Provider (dynamic loading)
|
||||
- name: Check if External Provider files changed
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
python-version: '3.12.13'
|
||||
|
||||
- name: Install PyYAML
|
||||
run: pip install pyyaml
|
||||
|
||||
@@ -10,6 +10,7 @@ on:
|
||||
- '.github/workflows/ui-codeql.yml'
|
||||
- '.github/codeql/ui-codeql-config.yml'
|
||||
- '!ui/CHANGELOG.md'
|
||||
- '!ui/changelog.d/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
@@ -19,6 +20,7 @@ on:
|
||||
- '.github/workflows/ui-codeql.yml'
|
||||
- '.github/codeql/ui-codeql-config.yml'
|
||||
- '!ui/CHANGELOG.md'
|
||||
- '!ui/changelog.d/**'
|
||||
schedule:
|
||||
- cron: '00 12 * * *'
|
||||
|
||||
@@ -47,7 +49,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -201,7 +201,7 @@ jobs:
|
||||
|
||||
- name: Install regctl
|
||||
if: always()
|
||||
uses: regclient/actions/regctl-installer@da9319db8e44e8b062b3a147e1dfb2f574d41a03 # main
|
||||
uses: regclient/actions/regctl-installer@9a2d4216180dbb3e2dccfa60d2dd4afd98e42ec5 # main
|
||||
|
||||
- name: Cleanup intermediate architecture tags
|
||||
if: always()
|
||||
@@ -222,7 +222,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -258,27 +258,3 @@ jobs:
|
||||
payload-file-path: "./.github/scripts/slack-messages/container-release-completed.json"
|
||||
step-outcome: ${{ steps.outcome.outputs.outcome }}
|
||||
update-ts: ${{ needs.notify-release-started.outputs.message-ts }}
|
||||
|
||||
trigger-deployment:
|
||||
needs: [setup, container-build-push]
|
||||
if: always() && github.event_name == 'push' && needs.setup.result == 'success' && needs.container-build-push.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
|
||||
- name: Trigger UI deployment
|
||||
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
repository: ${{ secrets.CLOUD_DISPATCH }}
|
||||
event-type: ui-prowler-deployment
|
||||
client-payload: '{"sha": "${{ github.sha }}", "short_sha": "${{ needs.setup.outputs.short-sha }}"}'
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
@@ -102,6 +102,7 @@ jobs:
|
||||
files: ui/**
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/changelog.d/**
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ on:
|
||||
- '.github/test-impact.yml'
|
||||
- 'ui/**'
|
||||
- 'api/**' # API changes can affect UI E2E
|
||||
- '!ui/CHANGELOG.md'
|
||||
- '!api/CHANGELOG.md'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
@@ -94,7 +96,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -314,7 +316,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
|
||||
@@ -31,14 +31,18 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
github.com:443
|
||||
registry.npmjs.org:443
|
||||
nodejs.org:443
|
||||
fonts.googleapis.com:443
|
||||
fonts.gstatic.com:443
|
||||
api.iconify.design:443
|
||||
api.simplesvg.com:443
|
||||
api.unisvg.com:443
|
||||
api.github.com:443
|
||||
release-assets.githubusercontent.com:443
|
||||
cdn.playwright.dev:443
|
||||
@@ -60,6 +64,7 @@ jobs:
|
||||
.github/workflows/ui-tests.yml
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/changelog.d/**
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
|
||||
@@ -169,3 +169,7 @@ GEMINI.md
|
||||
|
||||
# Claude Code
|
||||
.claude/*
|
||||
|
||||
# Docker
|
||||
docker-compose.override.yml
|
||||
docker-compose-dev.override.yml
|
||||
|
||||
+14
-6
@@ -72,13 +72,13 @@ repos:
|
||||
exclude: contrib
|
||||
priority: 30
|
||||
|
||||
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/)
|
||||
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/, docs/scripts/)
|
||||
- repo: https://github.com/myint/autoflake
|
||||
rev: v2.3.3
|
||||
hooks:
|
||||
- id: autoflake
|
||||
name: "SDK - autoflake"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variable"]
|
||||
priority: 20
|
||||
|
||||
@@ -87,7 +87,7 @@ repos:
|
||||
hooks:
|
||||
- id: isort
|
||||
name: "SDK - isort"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
args: ["--profile", "black"]
|
||||
stages: ["pre-commit"]
|
||||
priority: 20
|
||||
@@ -97,7 +97,7 @@ repos:
|
||||
hooks:
|
||||
- id: black
|
||||
name: "SDK - black"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
@@ -105,7 +105,7 @@ repos:
|
||||
hooks:
|
||||
- id: flake8
|
||||
name: "SDK - flake8"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
args: ["--ignore=E266,W503,E203,E501,W605"]
|
||||
priority: 30
|
||||
|
||||
@@ -142,6 +142,14 @@ repos:
|
||||
files: { glob: ["mcp_server/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
- id: generate-provider-cards
|
||||
name: "Docs - regenerate provider cards snippet"
|
||||
entry: python3 docs/scripts/generate_provider_cards.py
|
||||
language: system
|
||||
files: { glob: ["docs/user-guide/providers/**/getting-started-*.mdx", "docs/scripts/generate_provider_cards.py", "docs/snippets/provider-cards.mdx", "api/src/backend/api/models.py"] }
|
||||
pass_filenames: false
|
||||
priority: 20
|
||||
|
||||
## PYTHON — uv (API + SDK)
|
||||
- repo: https://github.com/astral-sh/uv-pre-commit
|
||||
rev: 0.11.14
|
||||
@@ -183,7 +191,7 @@ repos:
|
||||
entry: pylint --disable=W,C,R,E -j 0 -rn -sn
|
||||
language: system
|
||||
types: [python]
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
priority: 30
|
||||
|
||||
- id: trufflehog
|
||||
|
||||
+45
-21
@@ -15,21 +15,32 @@
|
||||
# neither vulnerable code path (Archive::Tar parsing or regex compilation of
|
||||
# attacker-controlled input) is reachable from Prowler. No Debian bookworm fix
|
||||
# is available yet.
|
||||
CVE-2026-42496 pkg:perl exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl-base exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl-modules-5.36 exp:2026-07-15
|
||||
CVE-2026-42496 pkg:libperl5.36 exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl-base exp:2026-07-15
|
||||
CVE-2026-8376 pkg:perl-modules-5.36 exp:2026-07-15
|
||||
CVE-2026-8376 pkg:libperl5.36 exp:2026-07-15
|
||||
CVE-2026-42496 pkg:perl exp:2026-08-15
|
||||
CVE-2026-42496 pkg:perl-base exp:2026-08-15
|
||||
CVE-2026-42496 pkg:perl-modules-5.36 exp:2026-08-15
|
||||
CVE-2026-42496 pkg:libperl5.36 exp:2026-08-15
|
||||
CVE-2026-8376 pkg:perl exp:2026-08-15
|
||||
CVE-2026-8376 pkg:perl-base exp:2026-08-15
|
||||
CVE-2026-8376 pkg:perl-modules-5.36 exp:2026-08-15
|
||||
CVE-2026-8376 pkg:libperl5.36 exp:2026-08-15
|
||||
|
||||
# CVE-2026-13221 - Perl regex trie overflow.
|
||||
# Packages: perl, perl-base, perl-modules-5.36, libperl5.36.
|
||||
# Why ignored: upstream confirms Perl 5.36.0 is not affected; the regression
|
||||
# was introduced after this version. Debian currently marks bookworm as
|
||||
# vulnerable, which causes Trivy to report a false positive.
|
||||
# Ref: https://github.com/Perl/perl5/issues/23388
|
||||
CVE-2026-13221 pkg:perl exp:2026-08-15
|
||||
CVE-2026-13221 pkg:perl-base exp:2026-08-15
|
||||
CVE-2026-13221 pkg:perl-modules-5.36 exp:2026-08-15
|
||||
CVE-2026-13221 pkg:libperl5.36 exp:2026-08-15
|
||||
|
||||
# CVE-2025-7458 — SQLite integer overflow.
|
||||
# Package: libsqlite3-0.
|
||||
# Why ignored: transitive dependency of CPython's stdlib sqlite3 module. The
|
||||
# Prowler SDK does not open user-supplied SQLite databases; SQLite usage is
|
||||
# internal and bounded. No Debian bookworm fix is available.
|
||||
CVE-2025-7458 pkg:libsqlite3-0 exp:2026-07-15
|
||||
CVE-2025-7458 pkg:libsqlite3-0 exp:2026-08-15
|
||||
|
||||
# CVE-2026-43185 — Linux kernel ksmbd signedness bug.
|
||||
# Package: linux-libc-dev.
|
||||
@@ -37,7 +48,7 @@ CVE-2025-7458 pkg:libsqlite3-0 exp:2026-07-15
|
||||
# not a running kernel. Containers execute against the host kernel, so these
|
||||
# headers are inert at runtime. The upstream fix landed in kernel 7.0-rc2 and
|
||||
# has not been backported to Debian's 6.1 LTS line.
|
||||
CVE-2026-43185 pkg:linux-libc-dev exp:2026-07-15
|
||||
CVE-2026-43185 pkg:linux-libc-dev exp:2026-08-15
|
||||
|
||||
# CVE-2023-45853 — zlib MiniZip integer overflow / heap overflow in
|
||||
# zipOpenNewFileInZip4_64.
|
||||
@@ -49,8 +60,21 @@ CVE-2026-43185 pkg:linux-libc-dev exp:2026-07-15
|
||||
# zlib 1.3.1, available in Debian trixie (13); migrating the base image would
|
||||
# clear it fully.
|
||||
# Ref: https://security-tracker.debian.org/tracker/CVE-2023-45853
|
||||
CVE-2023-45853 pkg:zlib1g exp:2026-07-15
|
||||
CVE-2023-45853 pkg:zlib1g-dev exp:2026-07-15
|
||||
CVE-2023-45853 pkg:zlib1g exp:2026-08-15
|
||||
CVE-2023-45853 pkg:zlib1g-dev exp:2026-08-15
|
||||
|
||||
# CVE-2026-55200 — libssh2 out-of-bounds write in ssh2_transport_read() due to
|
||||
# an unchecked packet_length field in transport.c (heap corruption, possible RCE).
|
||||
# Package: libssh2-1.
|
||||
# Why ignored: libssh2-1 is pulled in only as a transitive dependency of libcurl4
|
||||
# (installed in the SDK Dockerfile for the networking/PowerShell stack). The
|
||||
# vulnerable path is reached exclusively when libssh2 acts as an SSH/SCP/SFTP
|
||||
# client parsing transport packets from a server. Prowler never uses libcurl's
|
||||
# SSH/SCP/SFTP transports; it talks to cloud provider HTTPS endpoints only, so the
|
||||
# affected code is unreachable at runtime. Fixed upstream in libssh2 commit
|
||||
# 97acf3df (PR #2052); no Debian bookworm fix is available yet.
|
||||
# Ref: https://security-tracker.debian.org/tracker/CVE-2026-55200
|
||||
CVE-2026-55200 pkg:libssh2-1 exp:2026-08-15
|
||||
|
||||
# --- API container image (api/Dockerfile) ---
|
||||
# The entries below are specific to the Prowler API image, which ships
|
||||
@@ -65,13 +89,13 @@ CVE-2023-45853 pkg:zlib1g-dev exp:2026-07-15
|
||||
# at runtime. The vulnerable path requires parsing attacker-controlled XML with
|
||||
# the affected interpreter, which Prowler does not do with the system Python.
|
||||
# Full mitigation also needs libexpat >= 2.8.0; no Debian bookworm fix yet.
|
||||
CVE-2026-7210 pkg:python3.11 exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11-dev exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11-minimal exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11 exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-dev exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-minimal exp:2026-07-15
|
||||
CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-07-15
|
||||
CVE-2026-7210 pkg:python3.11 exp:2026-08-15
|
||||
CVE-2026-7210 pkg:python3.11-dev exp:2026-08-15
|
||||
CVE-2026-7210 pkg:python3.11-minimal exp:2026-08-15
|
||||
CVE-2026-7210 pkg:libpython3.11 exp:2026-08-15
|
||||
CVE-2026-7210 pkg:libpython3.11-dev exp:2026-08-15
|
||||
CVE-2026-7210 pkg:libpython3.11-minimal exp:2026-08-15
|
||||
CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-08-15
|
||||
|
||||
# CVE-2026-33278 — Unbound DNSSEC validator use-after-free (DoS, possible RCE).
|
||||
# CVE-2026-42960 — Unbound DNS cache poisoning via promiscuous additional records.
|
||||
@@ -81,5 +105,5 @@ CVE-2026-7210 pkg:libpython3.11-stdlib exp:2026-07-15
|
||||
# vulnerabilities require operating a live Unbound recursive DNSSEC validator
|
||||
# that processes attacker-influenced DNS responses. Prowler never starts an
|
||||
# Unbound resolver, so neither code path is reachable. No Debian bookworm fix yet.
|
||||
CVE-2026-33278 pkg:libunbound8 exp:2026-07-15
|
||||
CVE-2026-42960 pkg:libunbound8 exp:2026-07-15
|
||||
CVE-2026-33278 pkg:libunbound8 exp:2026-08-15
|
||||
CVE-2026-42960 pkg:libunbound8 exp:2026-08-15
|
||||
|
||||
@@ -114,6 +114,7 @@ When performing these actions, ALWAYS invoke the corresponding skill FIRST:
|
||||
| Review PR requirements: template, title conventions, changelog gate | `prowler-pr` |
|
||||
| Review changelog format and conventions | `prowler-changelog` |
|
||||
| Reviewing JSON:API compliance | `jsonapi` |
|
||||
| Reviewing Prowler UI components | `prowler-ui` |
|
||||
| Reviewing compliance framework PRs | `prowler-compliance-review` |
|
||||
| Running makemigrations or pgmakemigrations | `django-migration-psql` |
|
||||
| Syncing compliance framework with upstream catalog | `prowler-compliance` |
|
||||
|
||||
+16
-2
@@ -1,12 +1,14 @@
|
||||
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/prowler"
|
||||
LABEL org.opencontainers.image.source="https://github.com/prowler-cloud/prowler"
|
||||
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
# Opt out of PowerShell telemetry (Application Insights -> dc.services.visualstudio.com)
|
||||
ENV POWERSHELL_TELEMETRY_OPTOUT=1
|
||||
|
||||
ARG TRIVY_VERSION=0.71.0
|
||||
ARG TRIVY_VERSION=0.71.2
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
@@ -95,6 +97,18 @@ RUN uv sync --locked --compile-bytecode && \
|
||||
# Install PowerShell modules
|
||||
RUN .venv/bin/python prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
USER root
|
||||
|
||||
# Remove build-only packages from the final image after Python dependencies are installed.
|
||||
RUN apt-get purge -y --auto-remove \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libzstd-dev \
|
||||
zlib1g-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
USER prowler
|
||||
|
||||
# Remove deprecated dash dependencies
|
||||
RUN pip uninstall dash-html-components -y && \
|
||||
pip uninstall dash-core-components -y
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<img align="center" alt="Prowler logo" src="https://github.com/prowler-cloud/prowler/blob/master/docs/img/prowler-logo-white.png#gh-dark-mode-only" width="50%" height="50%">
|
||||
</p>
|
||||
<p align="center">
|
||||
<b><i>Prowler</b> is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With hundreds of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
|
||||
<b><i>Prowler</b> is the Open Cloud Security Platform trusted by thousands to automate security and compliance in any cloud environment. With thousands of ready-to-use checks and compliance frameworks, Prowler delivers real-time, customizable monitoring and seamless integrations, making cloud security simple, scalable, and cost-effective for organizations of any size.
|
||||
</p>
|
||||
<p align="center">
|
||||
<b>Secure ANY cloud at AI Speed at <a href="https://prowler.com">prowler.com</i></b>
|
||||
@@ -21,7 +21,7 @@
|
||||
<a href="https://pypi.python.org/pypi/prowler/"><img alt="Python Version" src="https://img.shields.io/pypi/pyversions/prowler.svg"></a>
|
||||
<a href="https://pypistats.org/packages/prowler"><img alt="PyPI Downloads" src="https://img.shields.io/pypi/dw/prowler.svg?label=downloads"></a>
|
||||
<a href="https://hub.docker.com/r/toniblyx/prowler"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/toniblyx/prowler"></a>
|
||||
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height=19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
|
||||
<a href="https://gallery.ecr.aws/prowler-cloud/prowler"><img width="120" height="19" alt="AWS ECR Gallery" src="https://user-images.githubusercontent.com/3985464/151531396-b6535a68-c907-44eb-95a1-a09508178616.png"></a>
|
||||
<a href="https://codecov.io/gh/prowler-cloud/prowler"><img alt="Codecov coverage" src="https://codecov.io/gh/prowler-cloud/prowler/graph/badge.svg?token=OflBGsdpDl"/></a>
|
||||
<a href="https://insights.linuxfoundation.org/project/prowler-cloud-prowler"><img alt="Linux Foundation insights health score" src="https://insights.linuxfoundation.org/api/badge/health-score?project=prowler-cloud-prowler"/></a>
|
||||
</p>
|
||||
@@ -41,7 +41,7 @@
|
||||
|
||||
# Description
|
||||
|
||||
**Prowler** is the world’s most widely used _Open-Source Cloud Security Platform_ that automates security and compliance across **any cloud environment**. With hundreds of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY Cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size.
|
||||
**Prowler** is the world’s most widely used _Open-Source Cloud Security Platform_ that automates security and compliance across **any cloud environment**. With thousands of ready-to-use security checks, remediation guidance, and compliance frameworks, Prowler is built to _“Secure ANY Cloud at AI Speed”_. Prowler delivers **AI-driven**, **customizable**, and **easy-to-use** assessments, dashboards, reports, and integrations, making cloud security **simple**, **scalable**, and **cost-effective** for organizations of any size.
|
||||
|
||||
Prowler includes hundreds of built-in controls to ensure compliance with standards and frameworks, including:
|
||||
|
||||
@@ -54,16 +54,16 @@ Prowler includes hundreds of built-in controls to ensure compliance with standar
|
||||
- **National Security Standards:** ENS (Spanish National Security Scheme) and KISA ISMS-P (Korean)
|
||||
- **Custom Security Frameworks:** Tailored to your needs
|
||||
|
||||
## Prowler App / Prowler Cloud
|
||||
## Prowler Cloud & Prowler Local Server
|
||||
|
||||
Prowler App / [Prowler Cloud](https://cloud.prowler.com/) is a web-based application that simplifies running Prowler across your cloud provider accounts. It provides a user-friendly interface to visualize the results and streamline your security assessments.
|
||||
[Prowler Cloud](https://cloud.prowler.com/) and Prowler Local Server, its self-hosted open-source version, are web applications that simplify running Prowler across your cloud provider accounts. They provide a user-friendly interface to visualize the results and streamline your security assessments.
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||
|
||||
|
||||
>For more details, refer to the [Prowler App Documentation](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-app-installation)
|
||||
>For more details, refer to the [Prowler Local Server documentation](https://docs.prowler.com/getting-started/installation/prowler-app)
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
@@ -73,26 +73,45 @@ prowler <provider>
|
||||

|
||||
|
||||
|
||||
## Prowler Dashboard
|
||||
## Prowler Local Dashboard
|
||||
|
||||
```console
|
||||
prowler dashboard
|
||||
```
|
||||

|
||||

|
||||
|
||||
|
||||
## Attack Paths
|
||||
|
||||
Attack Paths automatically extends every completed AWS scan with a Neo4j graph that combines Cartography's cloud inventory with Prowler findings. The feature runs in the API worker after each scan and therefore requires:
|
||||
Attack Paths automatically extends every completed AWS scan with a graph that combines Cartography's cloud inventory with Prowler findings. The feature runs in the API worker after each scan.
|
||||
|
||||
- An accessible Neo4j instance (the Docker Compose files already ships a `neo4j` service).
|
||||
- The following environment variables so Django and Celery can connect:
|
||||
Two graph backends are supported as the long-lived sink:
|
||||
|
||||
| Variable | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `NEO4J_HOST` | Hostname used by the API containers. | `neo4j` |
|
||||
| `NEO4J_PORT` | Bolt port exposed by Neo4j. | `7687` |
|
||||
| `NEO4J_USER` / `NEO4J_PASSWORD` | Credentials with rights to create per-tenant databases. | `neo4j` / `neo4j_password` |
|
||||
- **Neo4j** (default; the Docker Compose files already ship a `neo4j` service).
|
||||
- **Amazon Neptune** (cloud-managed; opt-in).
|
||||
|
||||
Select the sink with `ATTACK_PATHS_SINK_DATABASE` (`neo4j` or `neptune`; default `neo4j`).
|
||||
|
||||
> Note: Cartography ingestion always uses a temporary Neo4j database, regardless of the configured sink. The `NEO4J_*` variables below must remain set even when `ATTACK_PATHS_SINK_DATABASE=neptune`.
|
||||
|
||||
### Neo4j sink
|
||||
|
||||
| Variable | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `NEO4J_HOST` | Hostname used by the API containers. | `neo4j` |
|
||||
| `NEO4J_PORT` | Bolt port exposed by Neo4j. | `7687` |
|
||||
| `NEO4J_USER` / `NEO4J_PASSWORD` | Credentials with rights to create per-tenant databases. | `neo4j` / `neo4j_password` |
|
||||
|
||||
### Neptune sink
|
||||
|
||||
| Variable | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `NEPTUNE_WRITER_ENDPOINT` | Bolt host for the Neptune writer instance. Required when sink is `neptune`. | _empty_ |
|
||||
| `NEPTUNE_READER_ENDPOINT` | Optional reader endpoint for read-only queries. Falls back to the writer when unset. | _empty_ |
|
||||
| `NEPTUNE_PORT` | Bolt port exposed by Neptune. | `8182` |
|
||||
| `AWS_REGION` | Region the Neptune cluster lives in. Required when sink is `neptune`. | _empty_ |
|
||||
|
||||
Neptune authenticates with SigV4 using the standard boto3 credential chain. The worker's IAM role (or `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`) supplies the credentials. There is no Neptune password variable.
|
||||
|
||||
Every AWS provider scan will enqueue an Attack Paths ingestion job automatically. Other cloud providers will be added in future iterations.
|
||||
|
||||
@@ -102,29 +121,30 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
> For the most accurate and up-to-date information about checks, services, frameworks, and categories, visit [**Prowler Hub**](https://hub.prowler.com).
|
||||
|
||||
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/compliance/) | [Categories](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/misc/#categories) | Support | Interface |
|
||||
| Provider | Checks | Services | [Compliance Frameworks](https://docs.prowler.com/user-guide/compliance/tutorials/compliance) | [Categories](https://docs.prowler.com/user-guide/cli/tutorials/misc#categories) | Support | Interface |
|
||||
|---|---|---|---|---|---|---|
|
||||
| AWS | 613 | 86 | 46 | 19 | Official | UI, API, CLI |
|
||||
| Azure | 190 | 22 | 20 | 16 | Official | UI, API, CLI |
|
||||
| GCP | 109 | 20 | 18 | 12 | Official | UI, API, CLI |
|
||||
| Kubernetes | 90 | 7 | 7 | 11 | Official | UI, API, CLI |
|
||||
| GitHub | 24 | 3 | 1 | 5 | Official | UI, API, CLI |
|
||||
| M365 | 107 | 10 | 4 | 10 | Official | UI, API, CLI |
|
||||
| OCI | 52 | 14 | 4 | 10 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 63 | 9 | 5 | 9 | Official | UI, API, CLI |
|
||||
| Cloudflare | 29 | 3 | 1 | 5 | Official | UI, API, CLI |
|
||||
| AWS | 615 | 86 | 47 | 19 | Official | UI, API, CLI |
|
||||
| Azure | 190 | 22 | 21 | 16 | Official | UI, API, CLI |
|
||||
| GCP | 109 | 20 | 19 | 12 | Official | UI, API, CLI |
|
||||
| Kubernetes | 90 | 7 | 8 | 11 | Official | UI, API, CLI |
|
||||
| GitHub | 24 | 3 | 2 | 5 | Official | UI, API, CLI |
|
||||
| M365 | 109 | 10 | 6 | 10 | Official | UI, API, CLI |
|
||||
| OCI | 52 | 14 | 5 | 10 | Official | UI, API, CLI |
|
||||
| Alibaba Cloud | 63 | 9 | 6 | 9 | Official | UI, API, CLI |
|
||||
| Cloudflare | 29 | 3 | 2 | 5 | Official | UI, API, CLI |
|
||||
| IaC | [See `trivy` docs.](https://trivy.dev/latest/docs/coverage/iac/) | N/A | N/A | N/A | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 0 | 8 | Official | UI, API, CLI |
|
||||
| MongoDB Atlas | 10 | 3 | 1 | 8 | Official | UI, API, CLI |
|
||||
| LLM | [See `promptfoo` docs.](https://www.promptfoo.dev/docs/red-team/plugins/) | N/A | N/A | N/A | Official | CLI |
|
||||
| Image | N/A | N/A | N/A | N/A | Official | CLI, API |
|
||||
| Google Workspace | 65 | 11 | 2 | 6 | Official | UI, API, CLI |
|
||||
| OpenStack | 34 | 5 | 0 | 9 | Official | UI, API, CLI |
|
||||
| Vercel | 26 | 6 | 0 | 8 | Official | UI, API, CLI |
|
||||
| Okta | 29 | 8 | 1 | 2 | Official | UI, API, CLI |
|
||||
| Linode [Contact us](https://prowler.com/contact) | 10 | 3 | 0 | 4 | Unofficial | CLI |
|
||||
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 0 | 1 | Unofficial | CLI |
|
||||
| StackIT [Contact us](https://prowler.com/contact) | 7 | 2 | 0 | 3 | Unofficial | CLI |
|
||||
| NHN | 6 | 2 | 1 | 0 | Unofficial | CLI |
|
||||
| Google Workspace | 65 | 11 | 3 | 6 | Official | UI, API, CLI |
|
||||
| OpenStack | 34 | 5 | 1 | 9 | Official | UI, API, CLI |
|
||||
| Vercel | 26 | 6 | 1 | 8 | Official | UI, API, CLI |
|
||||
| Okta | 29 | 8 | 2 | 2 | Official | UI, API, CLI |
|
||||
| Linode [Contact us](https://prowler.com/contact) | 10 | 3 | 1 | 4 | Unofficial | CLI |
|
||||
| E2E Networks [Contact us](https://prowler.com/contact) | 27 | 6 | 0 | 2 | Unofficial | CLI |
|
||||
| Scaleway [Contact us](https://prowler.com/contact) | 1 | 1 | 1 | 1 | Unofficial | CLI |
|
||||
| StackIT [Contact us](https://prowler.com/contact) | 7 | 2 | 1 | 3 | Unofficial | CLI |
|
||||
| NHN | 6 | 2 | 2 | 0 | Unofficial | CLI |
|
||||
|
||||
> [!Note]
|
||||
> The numbers in the table are updated periodically.
|
||||
@@ -140,11 +160,11 @@ Every AWS provider scan will enqueue an Attack Paths ingestion job automatically
|
||||
|
||||
# 💻 Installation
|
||||
|
||||
## Prowler App
|
||||
## Prowler Local Server
|
||||
|
||||
Prowler App offers flexible installation methods tailored to various environments:
|
||||
Prowler Local Server offers flexible installation methods tailored to various environments:
|
||||
|
||||
> For detailed instructions on using Prowler App, refer to the [Prowler App Usage Guide](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/prowler-app/).
|
||||
> For detailed instructions on using Prowler Local Server, refer to the [usage guide](https://docs.prowler.com/user-guide/tutorials/prowler-app).
|
||||
|
||||
### Docker Compose
|
||||
|
||||
@@ -177,7 +197,7 @@ docker compose up -d
|
||||
> [!WARNING]
|
||||
> 🔒 For a secure setup, the API auto-generates a unique key pair, `DJANGO_TOKEN_SIGNING_KEY` and `DJANGO_TOKEN_VERIFYING_KEY`, and stores it in `~/.config/prowler-api` (non-container) or the bound Docker volume in `_data/api` (container). Never commit or reuse static/default keys. To rotate keys, delete the stored key files and restart the API.
|
||||
|
||||
Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
|
||||
Once configured, access Prowler Local Server at http://localhost:3000. Sign up using your email and password to get started.
|
||||
|
||||
### Common Issues with Docker Pull Installation
|
||||
|
||||
@@ -249,7 +269,7 @@ pnpm run build
|
||||
pnpm start
|
||||
```
|
||||
|
||||
> Once configured, access the Prowler App at http://localhost:3000. Sign up using your email and password to get started.
|
||||
> Once configured, access Prowler Local Server at http://localhost:3000. Sign up using your email and password to get started.
|
||||
|
||||
#### Pre-commit Hooks Setup
|
||||
|
||||
@@ -267,7 +287,7 @@ Prowler CLI is available as a project in [PyPI](https://pypi.org/project/prowler
|
||||
pip install prowler
|
||||
prowler -v
|
||||
```
|
||||
>For further guidance, refer to [https://docs.prowler.com](https://docs.prowler.com/projects/prowler-open-source/en/latest/#prowler-cli-installation)
|
||||
>For further guidance, refer to [https://docs.prowler.com](https://docs.prowler.com/getting-started/installation/prowler-cli)
|
||||
|
||||
### Containers
|
||||
|
||||
@@ -287,7 +307,7 @@ The container images are available here:
|
||||
- Prowler CLI:
|
||||
- [DockerHub](https://hub.docker.com/r/prowlercloud/prowler/tags)
|
||||
- [AWS Public ECR](https://gallery.ecr.aws/prowler-cloud/prowler)
|
||||
- Prowler App:
|
||||
- Prowler Local Server:
|
||||
- [DockerHub - Prowler UI](https://hub.docker.com/r/prowlercloud/prowler-ui/tags)
|
||||
- [DockerHub - Prowler API](https://hub.docker.com/r/prowlercloud/prowler-api/tags)
|
||||
|
||||
@@ -337,17 +357,55 @@ Full configuration, per-provider authentication, and SARIF examples: [Prowler Gi
|
||||
|
||||
# ✏️ High level architecture
|
||||
|
||||
## Prowler App
|
||||
**Prowler App** is composed of four key components:
|
||||
## Prowler Local Server
|
||||
**Prowler Local Server** is composed of four key components:
|
||||
|
||||
- **Prowler UI**: A web-based interface, built with Next.js, providing a user-friendly experience for executing Prowler scans and visualizing results.
|
||||
- **Prowler API**: A backend service, developed with Django REST Framework, responsible for running Prowler scans and storing the generated results.
|
||||
- **Prowler SDK**: A Python SDK designed to extend the functionality of the Prowler CLI for advanced capabilities.
|
||||
- **Prowler MCP Server**: A Model Context Protocol server that provides AI tools for Lighthouse, the AI-powered security assistant. This is a critical dependency for Lighthouse functionality.
|
||||
|
||||

|
||||
```mermaid
|
||||
flowchart TB
|
||||
user([User / Security Team])
|
||||
cli([Prowler CLI])
|
||||
|
||||
<!-- Diagram source: docs/images/products/prowler-app-architecture.mmd — edit there, re-render at https://mermaid.live, and replace the PNG. -->
|
||||
subgraph APP["Prowler Local Server"]
|
||||
ui["Prowler UI<br/>(Next.js)"]
|
||||
api["Prowler API<br/>(Django REST Framework)"]
|
||||
worker["API Worker<br/>(Celery)"]
|
||||
beat["API Scheduler<br/>(Celery Beat)"]
|
||||
mcp["Prowler MCP Server<br/>(Lighthouse AI tools)"]
|
||||
end
|
||||
|
||||
sdk["Prowler SDK<br/>(Python)"]
|
||||
|
||||
subgraph DATA["Data Layer"]
|
||||
pg[("PostgreSQL")]
|
||||
valkey[("Valkey / Redis")]
|
||||
neo4j[("Neo4j")]
|
||||
end
|
||||
|
||||
providers["Providers"]
|
||||
|
||||
user --> ui
|
||||
user --> cli
|
||||
ui -->|REST| api
|
||||
ui -->|MCP HTTP| mcp
|
||||
mcp -->|REST| api
|
||||
api --> pg
|
||||
api --> valkey
|
||||
beat -->|enqueue jobs| valkey
|
||||
valkey -->|dispatch| worker
|
||||
worker --> pg
|
||||
worker -->|Attack Paths| neo4j
|
||||
worker -->|invokes| sdk
|
||||
cli --> sdk
|
||||
|
||||
sdk --> providers
|
||||
```
|
||||
|
||||
<!-- Diagram source: docs/images/products/prowler-app-architecture.mmd — keep this inline block, the docs page getting-started/products/prowler-app.mdx, and the .mmd file in sync. -->
|
||||
|
||||
|
||||
## Prowler CLI
|
||||
|
||||
@@ -2,6 +2,107 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [1.36.0] (Prowler v5.35.0)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `attack-paths-scan-perform` Celery tasks now use the configurable long-task time limits instead of the six-hour defaults [(#12009)](https://github.com/prowler-cloud/prowler/pull/12009)
|
||||
- Attack Paths scans handle provider deletion races cleanly, detect stale tasks after 16 hours, use backend-specific graph synchronization batches, and report exhausted Neptune write retries with the original database error [(#12019)](https://github.com/prowler-cloud/prowler/pull/12019)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- Jira integration credentials only accept bare Atlassian site names containing letters, numbers, and hyphens [(#12012)](https://github.com/prowler-cloud/prowler/pull/12012)
|
||||
- Social account linking requires a verified matching email from both the identity provider and the existing user account without sending account connection notifications [(#12013)](https://github.com/prowler-cloud/prowler/pull/12013)
|
||||
|
||||
---
|
||||
|
||||
## [1.35.0] (Prowler v5.34.0)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- `rls_transaction` now falls back directly to the primary DB for connection-level mid-query read replica failures via `execute_wrapper`, reducing non-streaming read crashes during replica recovery [(#10379)](https://github.com/prowler-cloud/prowler/pull/10379)
|
||||
- RBAC permission gates now combine permissions from every role assigned to a user in the active tenant [(#11979)](https://github.com/prowler-cloud/prowler/pull/11979)
|
||||
- `attack-paths-cleanup-stale-scans` now retries worker pings and checks recent scan activity before failing scans and removing temporary databases [(#11986)](https://github.com/prowler-cloud/prowler/pull/11986)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- User role relationship updates are limited to the active tenant to preserve role assignments in other tenants [(#11903)](https://github.com/prowler-cloud/prowler/pull/11903)
|
||||
- `api` container image removes the unused Debian `libxml2` runtime package and scopes the `CVE-2026-13221` Trivy exception to unaffected Perl 5.36 packages [(#11991)](https://github.com/prowler-cloud/prowler/pull/11991)
|
||||
|
||||
---
|
||||
|
||||
## [1.34.2] (Prowler v5.33.2)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths graph mutations now retry transient Neptune concurrency and deadline failures, while Neo4j mutations use managed transaction retries [(#11968)](https://github.com/prowler-cloud/prowler/pull/11968)
|
||||
- Attack Paths scans now use bounded child node identifiers for normalized list values in Neo4j and Neptune, preventing Neo4j RANGE index key size failures [(#11969)](https://github.com/prowler-cloud/prowler/pull/11969)
|
||||
- `scan-summary` aggregation now upserts summaries in deterministic conflict-key order, preventing PostgreSQL deadlocks during concurrent reaggregation [(#11971)](https://github.com/prowler-cloud/prowler/pull/11971)
|
||||
|
||||
---
|
||||
|
||||
## [1.34.1] (Prowler v5.33.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Session tokens are rejected after account password updates [(#11914)](https://github.com/prowler-cloud/prowler/pull/11914)
|
||||
- Jira dispatch task results now surface user-facing Jira failure messages [(#11925)](https://github.com/prowler-cloud/prowler/pull/11925)
|
||||
- AWS Attack Paths privilege escalation queries no longer fail on Neo4j with `Aggregation column contains implicit grouping expressions` [(#11939)](https://github.com/prowler-cloud/prowler/pull/11939)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- OpenAI-compatible Lighthouse provider base URLs are restricted before connection checks [(#11940)](https://github.com/prowler-cloud/prowler/pull/11940)
|
||||
- `LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS` environment variable to allow internal hosts as OpenAI-compatible Lighthouse AI base URLs [(#11942)](https://github.com/prowler-cloud/prowler/pull/11942)
|
||||
|
||||
---
|
||||
|
||||
## [1.34.0] (Prowler v5.33.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Compliance PDF reports no longer require provider credentials: findings are enriched from the provider metadata stored in the database, so reports generate even after the provider secret is deleted or its credentials become invalid [(#11845)](https://github.com/prowler-cloud/prowler/pull/11845)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Provider scans now queue behind active provider scans instead of dispatching concurrently, and resource failed-finding counters retry database conflicts with stable row locking [(#11848)](https://github.com/prowler-cloud/prowler/pull/11848)
|
||||
|
||||
---
|
||||
|
||||
## [1.33.1] (Prowler v5.32.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: Scan rows now have database defaults for `is_migrated` and `sink_backend` so `scan-perform-scheduled` inserts survive deploy skew [(#11826)](https://github.com/prowler-cloud/prowler/pull/11826)
|
||||
- Invited users now keep their invitation context when completing authentication with Google, GitHub, or SAML, so the invitation is accepted during login [(#11752)](https://github.com/prowler-cloud/prowler/pull/11752)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- User profile updates now allow users to update their own account while requiring user-management permissions to update other users in the same tenant [(#11792)](https://github.com/prowler-cloud/prowler/pull/11792)
|
||||
- Kubernetes provider credentials now reject kubeconfigs using `exec` authentication in Prowler Cloud, preventing user-supplied commands from running on Cloud workers [(#11753)](https://github.com/prowler-cloud/prowler/pull/11753)
|
||||
|
||||
---
|
||||
|
||||
## [1.33.0] (Prowler v5.32.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Timestamp precision support for `/api/v1/findings` `inserted_at` and `updated_at` filters [(#11754)](https://github.com/prowler-cloud/prowler/pull/11754)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Attack Paths: AWS Neptune is now supported as a persistent sink database, selectable via `ATTACK_PATHS_SINK_DATABASE=neptune` (default `neo4j`), Cartography's (bumped to 0.138.1) per-scan ingest database stays on Neo4j [(#11524)](https://github.com/prowler-cloud/prowler/pull/11524)
|
||||
- Attack Paths: Scan task now checks the ingest Neo4j database and configured graph sink before starting graph ingestion [(#11743)](https://github.com/prowler-cloud/prowler/pull/11743)
|
||||
- Disable PowerShell telemetry in the API container image [(#11746)](https://github.com/prowler-cloud/prowler/pull/11746)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: Provider graph cleanup now deletes Neo4j and Neptune relationships in directed batches before deleting nodes [(#11755)](https://github.com/prowler-cloud/prowler/pull/11755)
|
||||
- `scan-perform` no longer reports an error when a provider is deleted during a running scan [(#11696)](https://github.com/prowler-cloud/prowler/pull/11696)
|
||||
|
||||
---
|
||||
|
||||
## [1.32.1] (Prowler v5.31.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
+22
-2
@@ -1,11 +1,13 @@
|
||||
FROM python:3.12.13-slim-bookworm@sha256:76d4b7b6305788c6b4c6a19d6a22a3921bf802e9af4d5e1e5bd771208dba74bf AS build
|
||||
FROM python:3.12.13-slim-bookworm@sha256:8a7e7cc04fd3e2bd787f7f24e22d5d119aa590d429b50c95dfe12b3abe52f48b AS build
|
||||
|
||||
LABEL maintainer="https://github.com/prowler-cloud/api"
|
||||
|
||||
ARG POWERSHELL_VERSION=7.5.0
|
||||
ENV POWERSHELL_VERSION=${POWERSHELL_VERSION}
|
||||
# Opt out of PowerShell telemetry (Application Insights -> dc.services.visualstudio.com)
|
||||
ENV POWERSHELL_TELEMETRY_OPTOUT=1
|
||||
|
||||
ARG TRIVY_VERSION=0.71.0
|
||||
ARG TRIVY_VERSION=0.71.2
|
||||
ENV TRIVY_VERSION=${TRIVY_VERSION}
|
||||
|
||||
ARG ZIZMOR_VERSION=1.24.1
|
||||
@@ -102,6 +104,24 @@ RUN uv sync --locked --no-install-project && \
|
||||
|
||||
RUN .venv/bin/python .venv/lib/python3.12/site-packages/prowler/providers/m365/lib/powershell/m365_powershell.py
|
||||
|
||||
USER root
|
||||
|
||||
# Remove build-only packages from the final image after Python dependencies are installed.
|
||||
RUN apt-get purge -y --auto-remove \
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
libxml2-dev \
|
||||
libxmlsec1-dev \
|
||||
libxmlsec1-openssl \
|
||||
pkg-config \
|
||||
libtool \
|
||||
libxslt1-dev \
|
||||
python3-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
USER prowler
|
||||
|
||||
COPY --chown=prowler:prowler src/backend/ ./backend/
|
||||
COPY --chown=prowler:prowler docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Changelog fragments
|
||||
|
||||
Each PR adds one small file here instead of editing `CHANGELOG.md` directly, so concurrent PRs never conflict.
|
||||
|
||||
- Filename: `<slug>.<type>.md`, e.g. `my-new-check.added.md` (slug is free-form: letters, digits, `.`, `_`, `-`)
|
||||
- `<type>` is one of: `added`, `changed`, `deprecated`, `removed`, `fixed`, `security`
|
||||
- Content: one line with the changelog entry text, without the PR link and without a trailing period (the PR link is attached automatically at release time)
|
||||
- A PR adds as many fragment files as entries it needs, freely mixing types (one file per entry); same-type entries just use different slugs
|
||||
|
||||
Fragments are compiled into `CHANGELOG.md` when a release is prepared. Full conventions: `skills/prowler-changelog/SKILL.md`.
|
||||
@@ -0,0 +1 @@
|
||||
OCI provider secrets no longer require `region`; legacy `region` input is accepted for backwards compatibility but ignored before storing or scanning
|
||||
+13
-7
@@ -58,7 +58,7 @@ dependencies = [
|
||||
"matplotlib (==3.10.8)",
|
||||
"reportlab (==4.4.10)",
|
||||
"neo4j (==6.1.0)",
|
||||
"cartography (==0.135.0)",
|
||||
"cartography (==0.138.1)",
|
||||
"gevent (==25.9.1)",
|
||||
"werkzeug (==3.1.7)",
|
||||
"sqlparse (==0.5.5)",
|
||||
@@ -71,7 +71,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.33.0"
|
||||
version = "1.37.0"
|
||||
|
||||
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
|
||||
# target-version tracks this project's lowest supported Python.
|
||||
@@ -193,7 +193,7 @@ constraint-dependencies = [
|
||||
"blinker==1.9.0",
|
||||
"boto3==1.40.61",
|
||||
"botocore==1.40.61",
|
||||
"cartography==0.135.0",
|
||||
"cartography==0.138.1",
|
||||
"celery==5.6.2",
|
||||
"certifi==2026.1.4",
|
||||
"cffi==2.0.0",
|
||||
@@ -218,7 +218,6 @@ constraint-dependencies = [
|
||||
"debugpy==1.8.20",
|
||||
"decorator==5.2.1",
|
||||
"defusedxml==0.7.1",
|
||||
"detect-secrets==1.5.0",
|
||||
"dill==0.4.1",
|
||||
"distro==1.9.0",
|
||||
"dj-rest-auth==7.0.1",
|
||||
@@ -301,6 +300,7 @@ constraint-dependencies = [
|
||||
"jsonschema==4.23.0",
|
||||
"jsonschema-specifications==2025.9.1",
|
||||
"keystoneauth1==5.13.0",
|
||||
"kingfisher-bin==1.104.0",
|
||||
"kiwisolver==1.4.9",
|
||||
"knack==0.11.0",
|
||||
"kombu==5.6.2",
|
||||
@@ -447,7 +447,7 @@ constraint-dependencies = [
|
||||
"wcwidth==0.5.3",
|
||||
"websocket-client==1.9.0",
|
||||
"werkzeug==3.1.7",
|
||||
"workos==6.0.4",
|
||||
"workos==6.0.8",
|
||||
"wrapt==1.17.3",
|
||||
"xlsxwriter==3.2.9",
|
||||
"xmlsec==1.3.17",
|
||||
@@ -458,8 +458,13 @@ constraint-dependencies = [
|
||||
"zope-interface==8.2",
|
||||
"zstd==1.5.7.3"
|
||||
]
|
||||
# prowler@master needs okta==3.4.2; cartography 0.135.0 declares okta<1.0.0 for an
|
||||
# integration prowler does not import.
|
||||
# prowler@master needs okta==3.4.2, but cartography 0.138.1 requires okta<1.0.0.
|
||||
# Attack Paths does not ingest Okta today, so override the Cartography
|
||||
# dependency to the Prowler pin.
|
||||
#
|
||||
# prowler@master needs azure-mgmt-containerservice==34.1.0, but cartography
|
||||
# 0.138.1 requires azure-mgmt-containerservice>=41.0.0. Attack Paths does not
|
||||
# ingest Azure today, so override the Cartography dependency to the Prowler pin.
|
||||
#
|
||||
# prowler@master hard-pins microsoft-kiota-abstractions==1.9.2 in [project.dependencies].
|
||||
# The microsoft-kiota-http security bump to 1.9.9 (GHSA-7j59-v9qr-6fq9) requires
|
||||
@@ -475,6 +480,7 @@ constraint-dependencies = [
|
||||
# that request pyjwt[crypto] and leave cryptography (needed for RS256) only transitive.
|
||||
override-dependencies = [
|
||||
"okta==3.4.2",
|
||||
"azure-mgmt-containerservice==34.1.0",
|
||||
"microsoft-kiota-abstractions==1.9.9",
|
||||
"dulwich==1.2.5",
|
||||
"pyjwt[crypto]==2.13.0"
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from allauth.account.models import EmailAddress
|
||||
from allauth.core.exceptions import ImmediateHttpResponse
|
||||
from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
|
||||
from api.db_router import MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
@@ -9,7 +11,9 @@ from api.models import (
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.utils import accept_invitation_for_user
|
||||
from django.db import transaction
|
||||
from django.http import HttpResponseForbidden
|
||||
|
||||
|
||||
class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
@@ -20,9 +24,30 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
except User.DoesNotExist:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_invitation_token(request):
|
||||
for source_name in ("data", "POST"):
|
||||
data = getattr(request, source_name, None) or {}
|
||||
if not hasattr(data, "get"):
|
||||
continue
|
||||
invitation_token = data.get("invitation_token")
|
||||
if invitation_token:
|
||||
return invitation_token
|
||||
|
||||
wrapped_request = getattr(request, "_request", None)
|
||||
if wrapped_request and wrapped_request is not request:
|
||||
return ProwlerSocialAccountAdapter._get_invitation_token(wrapped_request)
|
||||
|
||||
return None
|
||||
|
||||
def pre_social_login(self, request, sociallogin):
|
||||
# Link existing accounts with the same email address
|
||||
email = sociallogin.account.extra_data.get("email")
|
||||
# The provider account is already bound, so no email-based linking is needed.
|
||||
if sociallogin.account.pk:
|
||||
return
|
||||
|
||||
# Prefer the normalized email populated by allauth. GitHub can return the
|
||||
# primary email separately from the profile stored in extra_data.
|
||||
email = sociallogin.user.email or sociallogin.account.extra_data.get("email")
|
||||
if sociallogin.provider.id == "saml":
|
||||
# For SAML, the asserted NameID email cannot be trusted on its own:
|
||||
# any tenant can claim any email domain in its SAML configuration. To
|
||||
@@ -63,6 +88,17 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
if email:
|
||||
existing_user = self.get_user_by_email(email)
|
||||
if existing_user:
|
||||
email_is_verified = EmailAddress.objects.filter(
|
||||
user=existing_user,
|
||||
email__iexact=email,
|
||||
verified=True,
|
||||
).exists()
|
||||
provider_verified_email = any(
|
||||
address.verified and address.email.casefold() == email.casefold()
|
||||
for address in sociallogin.email_addresses
|
||||
)
|
||||
if not email_is_verified or not provider_verified_email:
|
||||
raise ImmediateHttpResponse(HttpResponseForbidden())
|
||||
sociallogin.connect(request, existing_user)
|
||||
|
||||
def save_user(self, request, sociallogin, form=None):
|
||||
@@ -83,29 +119,38 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
user.name = social_account_name
|
||||
user.save(using=MainRouter.admin_db)
|
||||
|
||||
tenant = Tenant.objects.using(MainRouter.admin_db).create(
|
||||
name=f"{user.email.split('@')[0]} default tenant"
|
||||
)
|
||||
with rls_transaction(str(tenant.id)):
|
||||
Membership.objects.using(MainRouter.admin_db).create(
|
||||
user=user, tenant=tenant, role=Membership.RoleChoices.OWNER
|
||||
)
|
||||
role = Role.objects.using(MainRouter.admin_db).create(
|
||||
name="admin",
|
||||
tenant_id=tenant.id,
|
||||
manage_users=True,
|
||||
manage_account=True,
|
||||
manage_billing=True,
|
||||
manage_providers=True,
|
||||
manage_integrations=True,
|
||||
manage_scans=True,
|
||||
unlimited_visibility=True,
|
||||
)
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
invitation_token = self._get_invitation_token(request)
|
||||
if invitation_token:
|
||||
invitation, _ = accept_invitation_for_user(
|
||||
user=user,
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
invitation_token=invitation_token,
|
||||
)
|
||||
request.prowler_invitation_token = invitation_token
|
||||
request.prowler_invitation_tenant_id = str(invitation.tenant_id)
|
||||
else:
|
||||
tenant = Tenant.objects.using(MainRouter.admin_db).create(
|
||||
name=f"{user.email.split('@')[0]} default tenant"
|
||||
)
|
||||
with rls_transaction(str(tenant.id)):
|
||||
Membership.objects.using(MainRouter.admin_db).create(
|
||||
user=user, tenant=tenant, role=Membership.RoleChoices.OWNER
|
||||
)
|
||||
role = Role.objects.using(MainRouter.admin_db).create(
|
||||
name="admin",
|
||||
tenant_id=tenant.id,
|
||||
manage_users=True,
|
||||
manage_account=True,
|
||||
manage_billing=True,
|
||||
manage_providers=True,
|
||||
manage_integrations=True,
|
||||
manage_scans=True,
|
||||
unlimited_visibility=True,
|
||||
)
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
user=user,
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
else:
|
||||
request.session["saml_user_created"] = str(user.id)
|
||||
|
||||
|
||||
@@ -42,9 +42,6 @@ class ApiConfig(AppConfig):
|
||||
):
|
||||
self._ensure_crypto_keys()
|
||||
|
||||
# Neo4j driver is created lazily on first use (see api.attack_paths.database).
|
||||
# App init never contacts Neo4j, so a Neo4j outage cannot block API startup.
|
||||
|
||||
def _ensure_crypto_keys(self):
|
||||
"""
|
||||
Orchestrator method that ensures all required cryptographic keys are present.
|
||||
|
||||
@@ -4,10 +4,10 @@ Cypher sanitizer for custom (user-supplied) Attack Paths queries.
|
||||
Two responsibilities:
|
||||
|
||||
1. **Validation** - reject queries containing SSRF or dangerous procedure
|
||||
patterns (defense-in-depth; the primary control is ``neo4j.READ_ACCESS``).
|
||||
patterns (defense-in-depth; the primary control is `neo4j.READ_ACCESS`).
|
||||
|
||||
2. **Provider-scoped label injection** - inject a dynamic
|
||||
``_Provider_{uuid}`` label into every node pattern so the database can
|
||||
`_Provider_{uuid}` label into every node pattern so the database can
|
||||
use its native label index for provider isolation.
|
||||
|
||||
Label-injection pipeline:
|
||||
@@ -25,13 +25,13 @@ from rest_framework.exceptions import ValidationError
|
||||
from tasks.jobs.attack_paths.config import get_provider_label
|
||||
|
||||
# Step 1 - String / comment protection
|
||||
# Single combined regex: strings first, then line comments.
|
||||
# Single combined regex: strings first, then line comments
|
||||
# The regex engine finds the leftmost match, so a string like 'https://prowler.com'
|
||||
# is consumed as a string before the // inside it can match as a comment.
|
||||
# is consumed as a string before the // inside it can match as a comment
|
||||
_PROTECTED_RE = re.compile(r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\"|//[^\n]*")
|
||||
|
||||
# Step 2 - Clause splitting
|
||||
# OPTIONAL MATCH must come before MATCH to avoid partial matching.
|
||||
# `OPTIONAL MATCH` must come before `MATCH` to avoid partial matching
|
||||
_CLAUSE_RE = re.compile(
|
||||
r"\b(OPTIONAL\s+MATCH|MATCH|WHERE|RETURN|WITH|ORDER\s+BY"
|
||||
r"|SKIP|LIMIT|UNION|UNWIND|CALL)\b",
|
||||
@@ -39,10 +39,10 @@ _CLAUSE_RE = re.compile(
|
||||
)
|
||||
|
||||
# Pass A - Labeled node patterns (all segments)
|
||||
# Matches node patterns that have at least one :Label.
|
||||
# (?<!\w)\( - open paren NOT preceded by a word char (excludes function calls).
|
||||
# Group 1: optional variable + one or more :Label
|
||||
# Group 2: optional {properties} + closing paren
|
||||
# Matches node patterns that have at least one `:Label`
|
||||
# `(?<!\w)\(` - open paren NOT preceded by a word char, excludes function calls
|
||||
# Group 1: optional variable + one or more `:Label`
|
||||
# Group 2: optional `{`properties`}` + closing paren
|
||||
_LABELED_NODE_RE = re.compile(
|
||||
r"(?<!\w)\("
|
||||
r"("
|
||||
@@ -55,9 +55,9 @@ _LABELED_NODE_RE = re.compile(
|
||||
r")"
|
||||
)
|
||||
|
||||
# Pass B - Bare node patterns (MATCH segments only)
|
||||
# Matches (identifier) or (identifier {properties}) without any :Label.
|
||||
# Only applied in MATCH/OPTIONAL MATCH segments.
|
||||
# Pass B - Bare node patterns (`MATCH` segments only)
|
||||
# Matches (identifier) or (identifier {properties}) without any `:Label`
|
||||
# Only applied in `MATCH` / `OPTIONAL MATCH` segments
|
||||
_BARE_NODE_RE = re.compile(
|
||||
r"(?<!\w)\(" r"(\s*[a-zA-Z_]\w*)" r"(\s*(?:\{[^}]*\})?)" r"\s*\)"
|
||||
)
|
||||
@@ -96,6 +96,11 @@ def inject_provider_label(cypher: str, provider_id: str) -> str:
|
||||
node pattern.
|
||||
"""
|
||||
label = get_provider_label(provider_id)
|
||||
return inject_label(cypher, label)
|
||||
|
||||
|
||||
def inject_label(cypher: str, label: str) -> str:
|
||||
"""Rewrite a Cypher query to append a label to every node pattern."""
|
||||
|
||||
# Step 1: Protect strings and comments (single pass, leftmost-first)
|
||||
protected: list[str] = []
|
||||
@@ -134,9 +139,7 @@ def inject_provider_label(cypher: str, provider_id: str) -> str:
|
||||
return work
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Patterns that indicate SSRF or dangerous procedure calls
|
||||
# Defense-in-depth layer - the primary control is `neo4j.READ_ACCESS`
|
||||
|
||||
@@ -1,261 +1,33 @@
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
"""Backwards-compatible facade over the ingest and sink modules.
|
||||
|
||||
Historically this module owned a single Neo4j driver used for both the
|
||||
cartography temp database and the per-tenant sink database. The port to AWS
|
||||
Neptune split those roles: the cartography ingest (temp) database is always
|
||||
Neo4j and lives in `api.attack_paths.ingest`; the sink is configurable
|
||||
(Neo4j or Neptune) and lives in `api.attack_paths.sink`. This shim preserves
|
||||
the public API that `tasks/` and `api/v1/views.py` already depend on, and
|
||||
dispatches to the right module by database-name prefix.
|
||||
|
||||
A database name starting with `db-tmp-scan-` is a cartography temp DB and
|
||||
routes to ingest. Everything else routes to the configured sink.
|
||||
"""
|
||||
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
import neo4j # noqa: F401 - kept for tests that patch api.attack_paths.database.neo4j
|
||||
from api.attack_paths import ingest
|
||||
from api.attack_paths import sink as sink_module
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
from django.conf import (
|
||||
settings, # noqa: F401 - kept for tests that patch ...database.settings
|
||||
)
|
||||
|
||||
# Without this Celery goes crazy with Neo4j logging
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = env.int(
|
||||
"ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES", default=3
|
||||
)
|
||||
READ_QUERY_TIMEOUT_SECONDS = env.int(
|
||||
"ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30
|
||||
)
|
||||
MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250)
|
||||
# Shorter than CONN_ACQUISITION_TIMEOUT — the driver requires acquisition to be
|
||||
# the longer of the two (it may include opening a new connection).
|
||||
CONNECTION_TIMEOUT = env.int("NEO4J_CONNECTION_TIMEOUT", default=5)
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15)
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
|
||||
# Module-level process-wide driver singleton
|
||||
_driver: neo4j.Driver | None = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
# Base Neo4j functions
|
||||
|
||||
|
||||
def get_uri() -> str:
|
||||
host = settings.DATABASES["neo4j"]["HOST"]
|
||||
port = settings.DATABASES["neo4j"]["PORT"]
|
||||
return f"bolt://{host}:{port}"
|
||||
|
||||
|
||||
def init_driver() -> neo4j.Driver:
|
||||
global _driver
|
||||
if _driver is not None:
|
||||
return _driver
|
||||
|
||||
with _lock:
|
||||
if _driver is None:
|
||||
uri = get_uri()
|
||||
config = settings.DATABASES["neo4j"]
|
||||
|
||||
driver = neo4j.GraphDatabase.driver(
|
||||
uri,
|
||||
auth=(config["USER"], config["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=7200,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=50,
|
||||
)
|
||||
# Publish the singleton only after connectivity is verified so a
|
||||
# failed probe does not leave an unverified driver behind. Close the
|
||||
# driver on failure so a repeatedly-probed outage cannot leak pools.
|
||||
try:
|
||||
driver.verify_connectivity()
|
||||
except Exception:
|
||||
driver.close()
|
||||
raise
|
||||
_driver = driver
|
||||
|
||||
# Register cleanup handler (only runs once since we're inside the _driver is None block)
|
||||
atexit.register(close_driver)
|
||||
|
||||
return _driver
|
||||
|
||||
|
||||
def get_driver() -> neo4j.Driver:
|
||||
return init_driver()
|
||||
|
||||
|
||||
def close_driver() -> None: # TODO: Use it
|
||||
global _driver
|
||||
with _lock:
|
||||
if _driver is not None:
|
||||
try:
|
||||
_driver.close()
|
||||
|
||||
finally:
|
||||
_driver = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_session(
|
||||
database: str | None = None, default_access_mode: str | None = None
|
||||
) -> Iterator[RetryableSession]:
|
||||
session_wrapper: RetryableSession | None = None
|
||||
|
||||
try:
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: get_driver().session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
if (
|
||||
default_access_mode == neo4j.READ_ACCESS
|
||||
and exc.code
|
||||
and exc.code in READ_EXCEPTION_CODES
|
||||
):
|
||||
message = "Read query not allowed"
|
||||
code = READ_EXCEPTION_CODES[0]
|
||||
raise WriteQueryNotAllowedException(message=message, code=code)
|
||||
|
||||
message = exc.message if exc.message is not None else str(exc)
|
||||
|
||||
if exc.code and exc.code.startswith(CLIENT_STATEMENT_EXCEPTION_PREFIX):
|
||||
raise ClientStatementException(message=message, code=exc.code)
|
||||
|
||||
raise GraphDatabaseQueryException(message=message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
|
||||
def execute_read_query(
|
||||
database: str,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph:
|
||||
with get_session(database, default_access_mode=neo4j.READ_ACCESS) as session:
|
||||
|
||||
def _run(tx: neo4j.ManagedTransaction) -> neo4j.graph.Graph:
|
||||
result = tx.run(
|
||||
cypher, parameters or {}, timeout=READ_QUERY_TIMEOUT_SECONDS
|
||||
)
|
||||
return result.graph()
|
||||
|
||||
return session.execute_read(_run)
|
||||
|
||||
|
||||
def create_database(database: str) -> None:
|
||||
query = "CREATE DATABASE $database IF NOT EXISTS"
|
||||
parameters = {"database": database}
|
||||
|
||||
with get_session() as session:
|
||||
session.run(query, parameters)
|
||||
|
||||
|
||||
def drop_database(database: str) -> None:
|
||||
query = f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA"
|
||||
|
||||
with get_session() as session:
|
||||
session.run(query)
|
||||
|
||||
|
||||
def drop_subgraph(database: str, provider_id: str) -> int:
|
||||
"""
|
||||
Delete all nodes for a provider from the tenant database.
|
||||
|
||||
Deletes relationships then nodes in batches (not `DETACH DELETE`) so a dense
|
||||
provider's graph cannot exceed Neo4j's transaction memory limit.
|
||||
Silently returns 0 if the database doesn't exist.
|
||||
"""
|
||||
provider_label = get_provider_label(provider_id)
|
||||
deleted_nodes = 0
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
# Phase 1: delete relationships incident to provider nodes in batches.
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (:`{provider_label}`)-[r]-()
|
||||
WITH DISTINCT r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
deleted_count = result.single().get("deleted_rels_count", 0)
|
||||
|
||||
# Phase 2: delete the now relationship-free nodes in batches.
|
||||
deleted_count = 1
|
||||
while deleted_count > 0:
|
||||
result = session.run(
|
||||
f"""
|
||||
MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
""",
|
||||
{"batch_size": BATCH_SIZE},
|
||||
)
|
||||
deleted_count = result.single().get("deleted_nodes_count", 0)
|
||||
deleted_nodes += deleted_count
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == "Neo.ClientError.Database.DatabaseNotFound":
|
||||
return 0
|
||||
raise
|
||||
|
||||
return deleted_nodes
|
||||
|
||||
|
||||
def has_provider_data(database: str, provider_id: str) -> bool:
|
||||
"""
|
||||
Check if any ProviderResource node exists for this provider.
|
||||
|
||||
Returns `False` if the database doesn't exist.
|
||||
"""
|
||||
provider_label = get_provider_label(provider_id)
|
||||
query = f"MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`) RETURN 1 LIMIT 1"
|
||||
|
||||
try:
|
||||
with get_session(database, default_access_mode=neo4j.READ_ACCESS) as session:
|
||||
result = session.run(query)
|
||||
return result.single() is not None
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == "Neo.ClientError.Database.DatabaseNotFound":
|
||||
return False
|
||||
raise
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
query = "CALL db.clearQueryCaches()"
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
session.run(query)
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
logging.warning(f"Failed to clear query cache for database `{database}`: {exc}")
|
||||
|
||||
|
||||
# Neo4j functions related to Prowler + Cartography
|
||||
|
||||
|
||||
def get_database_name(entity_id: str | UUID, temporary: bool = False) -> str:
|
||||
prefix = "tmp-scan" if temporary else "tenant"
|
||||
return f"db-{prefix}-{str(entity_id).lower()}"
|
||||
TEMP_DB_PREFIX = "db-tmp-scan-"
|
||||
DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
|
||||
|
||||
|
||||
# Exceptions
|
||||
@@ -270,13 +42,190 @@ class GraphDatabaseQueryException(Exception):
|
||||
def __str__(self) -> str:
|
||||
if self.code:
|
||||
return f"{self.code}: {self.message}"
|
||||
|
||||
return self.message
|
||||
|
||||
|
||||
class NeptuneWriteRetryExhaustedException(GraphDatabaseQueryException):
|
||||
pass
|
||||
|
||||
|
||||
class WriteQueryNotAllowedException(GraphDatabaseQueryException):
|
||||
pass
|
||||
|
||||
|
||||
class ClientStatementException(GraphDatabaseQueryException):
|
||||
pass
|
||||
|
||||
|
||||
# Routing
|
||||
|
||||
|
||||
def _is_ingest_database(database: str | None) -> bool:
|
||||
return bool(database) and database.startswith(TEMP_DB_PREFIX)
|
||||
|
||||
|
||||
# Driver lifecycle
|
||||
|
||||
|
||||
def init_driver() -> Any:
|
||||
"""Initialize the configured sink backend.
|
||||
|
||||
The ingest driver (Neo4j for cartography temp DBs) stays lazy: it is
|
||||
only initialized when a temp-DB operation actually runs, which never
|
||||
happens on API pods.
|
||||
"""
|
||||
return sink_module.init()
|
||||
|
||||
|
||||
def close_driver() -> None:
|
||||
"""Close every driver held by this process."""
|
||||
sink_module.close()
|
||||
ingest.close_driver()
|
||||
|
||||
|
||||
def get_driver() -> neo4j.Driver:
|
||||
"""Return the sink backend's underlying driver.
|
||||
|
||||
Only meaningful for the Neo4j sink (where the backend has a single Neo4j
|
||||
driver). On Neptune this returns the writer driver. Kept for tests and
|
||||
legacy call-sites; prefer `get_session` for new code.
|
||||
"""
|
||||
backend = sink_module.get_backend()
|
||||
|
||||
# Neo4jSink exposes get_driver(); NeptuneSink exposes get_writer()
|
||||
if hasattr(backend, "get_driver"):
|
||||
return backend.get_driver()
|
||||
|
||||
if hasattr(backend, "get_writer"):
|
||||
return backend.get_writer()
|
||||
|
||||
raise RuntimeError("Active sink backend does not expose a driver handle")
|
||||
|
||||
|
||||
def verify_connectivity() -> None:
|
||||
"""Raise if the configured graph database is unreachable on the API read path.
|
||||
|
||||
Backend-agnostic entry point for the readiness probe: Neo4j verifies its
|
||||
driver, Neptune verifies the reader endpoint.
|
||||
"""
|
||||
sink_module.get_backend().verify_connectivity()
|
||||
|
||||
|
||||
def verify_scan_databases_available() -> None:
|
||||
"""Raise if either graph database needed by an Attack Paths scan is unavailable."""
|
||||
errors: list[str] = []
|
||||
first_error: Exception | None = None
|
||||
|
||||
try:
|
||||
ingest.get_driver().verify_connectivity()
|
||||
except Exception as exc:
|
||||
errors.append(f"ingest Neo4j: {exc}")
|
||||
first_error = exc
|
||||
|
||||
try:
|
||||
get_driver().verify_connectivity()
|
||||
except Exception as exc:
|
||||
errors.append(f"sink {settings.ATTACK_PATHS_SINK_DATABASE}: {exc}")
|
||||
if first_error is None:
|
||||
first_error = exc
|
||||
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"Attack Paths graph database unavailable before scan start: "
|
||||
+ "; ".join(errors)
|
||||
) from first_error
|
||||
|
||||
|
||||
def get_uri() -> str:
|
||||
"""Return the sink URI. Retained for backwards compatibility."""
|
||||
if settings.ATTACK_PATHS_SINK_DATABASE == "neptune":
|
||||
cfg = settings.DATABASES["neptune"]
|
||||
return f"bolt+s://{cfg['WRITER_ENDPOINT']}:{cfg['PORT']}"
|
||||
|
||||
cfg = settings.DATABASES["neo4j"]
|
||||
return f"bolt://{cfg['HOST']}:{cfg['PORT']}"
|
||||
|
||||
|
||||
def get_ingest_uri() -> str:
|
||||
"""Neo4j URI for the cartography temp (ingest) database, which is always
|
||||
Neo4j regardless of the configured sink."""
|
||||
return ingest.get_uri()
|
||||
|
||||
|
||||
# Session API
|
||||
|
||||
|
||||
def get_session(
|
||||
database: str | None = None,
|
||||
default_access_mode: str | None = None,
|
||||
) -> AbstractContextManager:
|
||||
"""Return a session against the right backend.
|
||||
|
||||
- `database` names starting with `db-tmp-scan-` always go to ingest.
|
||||
- No database name → ingest (used for CREATE / DROP DATABASE admin ops).
|
||||
- Any other name → sink.
|
||||
"""
|
||||
if _is_ingest_database(database) or database is None:
|
||||
return ingest.get_session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
)
|
||||
|
||||
return sink_module.get_backend().get_session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
)
|
||||
|
||||
|
||||
def execute_read_query(
|
||||
database: str,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph:
|
||||
"""Read-only query against the sink."""
|
||||
return sink_module.get_backend().execute_read_query(database, cypher, parameters)
|
||||
|
||||
|
||||
def create_database(database: str) -> None:
|
||||
"""Create a database. Temp DBs always land on ingest (Neo4j).
|
||||
|
||||
On the Neo4j sink, tenant DBs also route to ingest because both drivers
|
||||
connect to the same Neo4j cluster. On the Neptune sink, tenant DB creates
|
||||
are no-ops.
|
||||
"""
|
||||
if _is_ingest_database(database):
|
||||
ingest.create_database(database)
|
||||
return
|
||||
|
||||
sink_module.get_backend().create_database(database)
|
||||
|
||||
|
||||
def drop_database(database: str) -> None:
|
||||
"""Drop a database. Mirrors `create_database` routing."""
|
||||
if _is_ingest_database(database):
|
||||
ingest.drop_database(database)
|
||||
return
|
||||
|
||||
sink_module.get_backend().drop_database(database)
|
||||
|
||||
|
||||
def drop_subgraph(database: str, provider_id: str) -> int:
|
||||
return sink_module.get_backend().drop_subgraph(database, provider_id)
|
||||
|
||||
|
||||
def has_provider_data(database: str, provider_id: str) -> bool:
|
||||
return sink_module.get_backend().has_provider_data(database, provider_id)
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
if _is_ingest_database(database):
|
||||
ingest.clear_cache(database)
|
||||
return
|
||||
|
||||
sink_module.get_backend().clear_cache(database)
|
||||
|
||||
|
||||
# Name helper
|
||||
|
||||
|
||||
def get_database_name(entity_id: str | UUID, temporary: bool = False) -> str:
|
||||
prefix = "tmp-scan" if temporary else "tenant"
|
||||
return f"db-{prefix}-{str(entity_id).lower()}"
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Cartography ingest layer.
|
||||
|
||||
Public surface for the per-scan Neo4j temp database driver. Implementation
|
||||
lives in `api.attack_paths.ingest.driver`.
|
||||
"""
|
||||
|
||||
from api.attack_paths.ingest.driver import (
|
||||
clear_cache,
|
||||
close_driver,
|
||||
create_database,
|
||||
drop_database,
|
||||
get_driver,
|
||||
get_session,
|
||||
get_uri,
|
||||
init_driver,
|
||||
run_cypher,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"clear_cache",
|
||||
"close_driver",
|
||||
"create_database",
|
||||
"drop_database",
|
||||
"get_driver",
|
||||
"get_session",
|
||||
"get_uri",
|
||||
"init_driver",
|
||||
"run_cypher",
|
||||
]
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Cartography ingest driver: per-scan throw-away Neo4j database.
|
||||
|
||||
Cartography writes each scan's graph into a throw-away Neo4j database named
|
||||
`db-tmp-scan-{scan_uuid}`. This is always Neo4j, regardless of the configured
|
||||
sink: Neptune is single-database and cannot host per-scan throw-away
|
||||
databases. This module owns the Neo4j driver used for those temp DBs and the
|
||||
admin ops they need (CREATE / DROP DATABASE).
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = env.int(
|
||||
"ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES", default=3
|
||||
)
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15)
|
||||
# TCP connect timeout, ordered below the acquisition timeout so an unreachable
|
||||
# host can't pin a worker on a temp-DB op longer than this.
|
||||
CONNECTION_TIMEOUT = env.int("NEO4J_CONNECTION_TIMEOUT", default=5)
|
||||
MAX_CONNECTION_LIFETIME = env.int("NEO4J_MAX_CONNECTION_LIFETIME", default=7200)
|
||||
MAX_CONNECTION_POOL_SIZE = env.int("NEO4J_MAX_CONNECTION_POOL_SIZE", default=50)
|
||||
|
||||
_driver: neo4j.Driver | None = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def _neo4j_config() -> dict:
|
||||
return settings.DATABASES["neo4j"]
|
||||
|
||||
|
||||
def get_uri() -> str:
|
||||
"""Bolt URI for the Neo4j temp (ingest) database. Always Neo4j."""
|
||||
config = _neo4j_config()
|
||||
host = config["HOST"]
|
||||
port = config["PORT"]
|
||||
if not host or not port:
|
||||
raise RuntimeError(
|
||||
"NEO4J_HOST / NEO4J_PORT must be set to use the attack-paths "
|
||||
"temp database. Workers require Neo4j env even when the sink is Neptune."
|
||||
)
|
||||
|
||||
return f"bolt://{host}:{port}"
|
||||
|
||||
|
||||
def init_driver() -> neo4j.Driver:
|
||||
"""Initialize the temp-database Neo4j driver. Idempotent."""
|
||||
global _driver
|
||||
if _driver is not None:
|
||||
return _driver
|
||||
|
||||
with _lock:
|
||||
if _driver is None:
|
||||
config = _neo4j_config()
|
||||
_driver = neo4j.GraphDatabase.driver(
|
||||
get_uri(),
|
||||
auth=(config["USER"], config["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=MAX_CONNECTION_LIFETIME,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=MAX_CONNECTION_POOL_SIZE,
|
||||
)
|
||||
# Best-effort connectivity check: a Neo4j that is down at boot must
|
||||
# not crash the worker. The driver reconnects lazily on first use.
|
||||
try:
|
||||
_driver.verify_connectivity()
|
||||
|
||||
except Exception:
|
||||
logging.warning(
|
||||
"Neo4j temp-database unreachable at init; continuing with a "
|
||||
"lazily-reconnecting driver",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
atexit.register(close_driver)
|
||||
|
||||
return _driver
|
||||
|
||||
|
||||
def get_driver() -> neo4j.Driver:
|
||||
return init_driver()
|
||||
|
||||
|
||||
def close_driver() -> None:
|
||||
global _driver
|
||||
with _lock:
|
||||
if _driver is not None:
|
||||
try:
|
||||
_driver.close()
|
||||
finally:
|
||||
_driver = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_session(
|
||||
database: str | None = None,
|
||||
default_access_mode: str | None = None,
|
||||
) -> Iterator[RetryableSession]:
|
||||
"""Session against the Neo4j temp-database cluster. Used for temp DB sessions
|
||||
and for admin operations (CREATE / DROP DATABASE) when `database` is None."""
|
||||
from api.attack_paths.database import (
|
||||
ClientStatementException,
|
||||
GraphDatabaseQueryException,
|
||||
WriteQueryNotAllowedException,
|
||||
)
|
||||
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
|
||||
session_wrapper: RetryableSession | None = None
|
||||
try:
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: get_driver().session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
if (
|
||||
default_access_mode == neo4j.READ_ACCESS
|
||||
and exc.code
|
||||
and exc.code in READ_EXCEPTION_CODES
|
||||
):
|
||||
raise WriteQueryNotAllowedException(
|
||||
message="Read query not allowed", code=READ_EXCEPTION_CODES[0]
|
||||
)
|
||||
|
||||
message = exc.message if exc.message is not None else str(exc)
|
||||
if exc.code and exc.code.startswith(CLIENT_STATEMENT_EXCEPTION_PREFIX):
|
||||
raise ClientStatementException(message=message, code=exc.code)
|
||||
raise GraphDatabaseQueryException(message=message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
|
||||
def create_database(database: str) -> None:
|
||||
"""Create a database on the Neo4j cluster. Used for temp scan DBs."""
|
||||
with get_session() as session:
|
||||
session.run("CREATE DATABASE $database IF NOT EXISTS", {"database": database})
|
||||
|
||||
|
||||
def drop_database(database: str) -> None:
|
||||
"""Drop a database on the Neo4j cluster. Used for temp scan DBs."""
|
||||
with get_session() as session:
|
||||
session.run(f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA")
|
||||
|
||||
|
||||
def clear_cache(database: str) -> None:
|
||||
"""Best-effort cache clear for a Neo4j database."""
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
|
||||
try:
|
||||
with get_session(database) as session:
|
||||
session.run("CALL db.clearQueryCaches()")
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
logging.warning(f"Failed to clear query cache for database `{database}`: {exc}")
|
||||
|
||||
|
||||
def run_cypher(
|
||||
database: str | None,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
"""Execute Cypher directly without the context manager. Thin helper."""
|
||||
with get_session(database) as session:
|
||||
return session.run(cypher, parameters or {})
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,12 +1,14 @@
|
||||
from api.attack_paths.queries.aws import AWS_QUERIES
|
||||
|
||||
# TODO: drop after Neptune cutover
|
||||
from api.attack_paths.queries.aws_deprecated import AWS_DEPRECATED_QUERIES
|
||||
from api.attack_paths.queries.types import AttackPathsQueryDefinition
|
||||
|
||||
# Query definitions organized by provider
|
||||
# Query definitions for scans synced with the current schema.
|
||||
_QUERY_DEFINITIONS: dict[str, list[AttackPathsQueryDefinition]] = {
|
||||
"aws": AWS_QUERIES,
|
||||
}
|
||||
|
||||
# Flat lookup by query ID for O(1) access
|
||||
_QUERIES_BY_ID: dict[str, AttackPathsQueryDefinition] = {
|
||||
definition.id: definition
|
||||
for definitions in _QUERY_DEFINITIONS.values()
|
||||
@@ -14,11 +16,45 @@ _QUERIES_BY_ID: dict[str, AttackPathsQueryDefinition] = {
|
||||
}
|
||||
|
||||
|
||||
def get_queries_for_provider(provider: str) -> list[AttackPathsQueryDefinition]:
|
||||
"""Get all attack path queries for a specific provider."""
|
||||
return _QUERY_DEFINITIONS.get(provider, [])
|
||||
# TODO: drop after Neptune cutover
|
||||
#
|
||||
# Query definitions for pre-cutover scans (`AttackPathsScan.is_migrated=False`)
|
||||
# whose graph data was written under the previous schema. Both maps expose the
|
||||
# same query IDs so the API contract is identical regardless of which set is
|
||||
# routed to.
|
||||
_DEPRECATED_QUERY_DEFINITIONS: dict[str, list[AttackPathsQueryDefinition]] = {
|
||||
"aws": AWS_DEPRECATED_QUERIES,
|
||||
}
|
||||
|
||||
_DEPRECATED_QUERIES_BY_ID: dict[str, AttackPathsQueryDefinition] = {
|
||||
definition.id: definition
|
||||
for definitions in _DEPRECATED_QUERY_DEFINITIONS.values()
|
||||
for definition in definitions
|
||||
}
|
||||
|
||||
|
||||
def get_query_by_id(query_id: str) -> AttackPathsQueryDefinition | None:
|
||||
"""Get a specific attack path query by its ID."""
|
||||
return _QUERIES_BY_ID.get(query_id)
|
||||
def get_queries_for_provider(
|
||||
provider: str,
|
||||
is_migrated: bool = True,
|
||||
) -> list[AttackPathsQueryDefinition]:
|
||||
"""Get all attack path queries for a provider.
|
||||
|
||||
`is_migrated` selects the catalog: True for scans synced with the current
|
||||
schema, False for pre-cutover scans still using the legacy graph shape.
|
||||
# TODO: drop the `is_migrated` parameter after Neptune cutover
|
||||
"""
|
||||
catalog = _QUERY_DEFINITIONS if is_migrated else _DEPRECATED_QUERY_DEFINITIONS
|
||||
return catalog.get(provider, [])
|
||||
|
||||
|
||||
def get_query_by_id(
|
||||
query_id: str,
|
||||
is_migrated: bool = True,
|
||||
) -> AttackPathsQueryDefinition | None:
|
||||
"""Get a specific attack path query by ID.
|
||||
|
||||
`is_migrated` selects the catalog (see `get_queries_for_provider`).
|
||||
# TODO: drop the `is_migrated` parameter after Neptune cutover
|
||||
"""
|
||||
by_id = _QUERIES_BY_ID if is_migrated else _DEPRECATED_QUERIES_BY_ID
|
||||
return by_id.get(query_id)
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
@@ -8,18 +10,44 @@ import neo4j.exceptions
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RetryExhaustedError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
retry_context: str,
|
||||
method_name: str,
|
||||
attempts: int,
|
||||
elapsed_seconds: float,
|
||||
last_error: Exception,
|
||||
) -> None:
|
||||
self.retry_context = retry_context
|
||||
self.method_name = method_name
|
||||
self.attempts = attempts
|
||||
self.elapsed_seconds = elapsed_seconds
|
||||
self.last_error = last_error
|
||||
last_message = getattr(last_error, "message", None) or str(last_error)
|
||||
super().__init__(
|
||||
f"{retry_context} {method_name} failed after {attempts} attempts over "
|
||||
f"{elapsed_seconds:.3f}s. Last error: {last_message}"
|
||||
)
|
||||
|
||||
|
||||
class RetryableSession:
|
||||
"""
|
||||
Wrapper around `neo4j.Session` that retries `neo4j.exceptions.ServiceUnavailable` errors.
|
||||
"""
|
||||
"""Wrapper around ``neo4j.Session`` with a refreshable retry policy."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: Callable[[], neo4j.Session],
|
||||
max_retries: int,
|
||||
retry_if: Callable[[Exception], bool] | None = None,
|
||||
initial_retry_delay_seconds: float = 0,
|
||||
retry_context: str | None = None,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._max_retries = max(0, max_retries)
|
||||
self._retry_if = retry_if
|
||||
self._initial_retry_delay_seconds = max(0.0, initial_retry_delay_seconds)
|
||||
self._retry_context = retry_context
|
||||
self._session = self._session_factory()
|
||||
|
||||
def close(self) -> None:
|
||||
@@ -50,30 +78,75 @@ class RetryableSession:
|
||||
def _call_with_retry(self, method_name: str, *args: Any, **kwargs: Any) -> Any:
|
||||
attempt = 0
|
||||
last_exc: Exception | None = None
|
||||
started_at = time.monotonic()
|
||||
|
||||
while attempt <= self._max_retries:
|
||||
try:
|
||||
method = getattr(self._session, method_name)
|
||||
return method(*args, **kwargs)
|
||||
|
||||
except (
|
||||
BrokenPipeError,
|
||||
ConnectionResetError,
|
||||
neo4j.exceptions.ServiceUnavailable,
|
||||
) as exc: # pragma: no cover - depends on infra
|
||||
except Exception as exc:
|
||||
if not self._should_retry(exc):
|
||||
raise
|
||||
|
||||
last_exc = exc
|
||||
attempt += 1
|
||||
|
||||
if attempt > self._max_retries:
|
||||
if self._retry_context is not None:
|
||||
raise RetryExhaustedError(
|
||||
retry_context=self._retry_context,
|
||||
method_name=method_name,
|
||||
attempts=attempt,
|
||||
elapsed_seconds=time.monotonic() - started_at,
|
||||
last_error=exc,
|
||||
) from exc
|
||||
raise
|
||||
|
||||
logger.warning(
|
||||
f"Neo4j session {method_name} failed with {type(exc).__name__} ({attempt}/{self._max_retries} attempts). Retrying..."
|
||||
)
|
||||
delay = self._retry_delay(attempt)
|
||||
if self._retry_context is not None:
|
||||
error_message = getattr(exc, "message", None) or str(exc)
|
||||
logger.warning(
|
||||
"%s %s failed with %s: %s; retry %s/%s in %.3fs",
|
||||
self._retry_context,
|
||||
method_name,
|
||||
type(exc).__name__,
|
||||
error_message,
|
||||
attempt,
|
||||
self._max_retries,
|
||||
delay,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Graph session %s failed with %s; retry %s/%s in %.3fs",
|
||||
method_name,
|
||||
type(exc).__name__,
|
||||
attempt,
|
||||
self._max_retries,
|
||||
delay,
|
||||
)
|
||||
self._refresh_session()
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
|
||||
raise last_exc if last_exc else RuntimeError("Unexpected retry loop exit")
|
||||
|
||||
def _should_retry(self, exc: Exception) -> bool:
|
||||
if isinstance(
|
||||
exc,
|
||||
(
|
||||
BrokenPipeError,
|
||||
ConnectionResetError,
|
||||
neo4j.exceptions.ServiceUnavailable,
|
||||
),
|
||||
):
|
||||
return True
|
||||
return self._retry_if(exc) if self._retry_if else False
|
||||
|
||||
def _retry_delay(self, attempt: int) -> float:
|
||||
max_delay = self._initial_retry_delay_seconds * (2**attempt)
|
||||
return random.uniform(max_delay / 2, max_delay) if max_delay else 0
|
||||
|
||||
def _refresh_session(self) -> None:
|
||||
if self._session is not None:
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Attack-paths sink database layer.
|
||||
|
||||
The sink is the persistent store where attack-paths graphs live after a scan
|
||||
finishes. Currently selectable between Neo4j (OSS / local dev default) and
|
||||
AWS Neptune (hosted dev/staging/prod). Backend is picked by the
|
||||
`ATTACK_PATHS_SINK_DATABASE` setting at process init.
|
||||
|
||||
This package exposes the public factory API; the implementation lives in
|
||||
`api.attack_paths.sink.factory`.
|
||||
"""
|
||||
|
||||
from api.attack_paths.sink.factory import (
|
||||
SinkBackend,
|
||||
close,
|
||||
get_backend,
|
||||
get_backend_for_name,
|
||||
get_backend_for_scan,
|
||||
init,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SinkBackend",
|
||||
"close",
|
||||
"get_backend",
|
||||
"get_backend_for_name",
|
||||
"get_backend_for_scan",
|
||||
"init",
|
||||
]
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Protocol every sink backend must implement."""
|
||||
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any, Protocol
|
||||
|
||||
import neo4j
|
||||
|
||||
|
||||
class SinkDatabase(Protocol):
|
||||
"""Contract for the persistent attack-paths graph store.
|
||||
|
||||
The `database` argument is an opaque identifier passed through from the
|
||||
legacy `database.py` API surface. On Neo4j it is the per-tenant database
|
||||
name (e.g. `db-tenant-{uuid}`). On Neptune it is ignored (the cluster
|
||||
has a single graph, and isolation is label-based).
|
||||
"""
|
||||
|
||||
sync_batch_size: int
|
||||
|
||||
def init(self) -> None: ...
|
||||
|
||||
def close(self) -> None: ...
|
||||
|
||||
def verify_connectivity(self) -> None:
|
||||
"""Raise if the backend the API read path uses is unreachable.
|
||||
|
||||
Neo4j verifies its single driver. Neptune verifies the reader
|
||||
driver (the endpoint the API serves reads from); on single-endpoint
|
||||
clusters the reader aliases the writer, so that path is covered too.
|
||||
Used by the readiness probe; must not block longer than the caller's
|
||||
probe budget.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_session(
|
||||
self,
|
||||
database: str | None = None,
|
||||
default_access_mode: str | None = None,
|
||||
) -> AbstractContextManager: ...
|
||||
|
||||
def execute_read_query(
|
||||
self,
|
||||
database: str,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph: ...
|
||||
|
||||
def create_database(self, database: str) -> None: ...
|
||||
|
||||
def drop_database(self, database: str) -> None: ...
|
||||
|
||||
def drop_subgraph(self, database: str, provider_id: str) -> int: ...
|
||||
|
||||
def has_provider_data(self, database: str, provider_id: str) -> bool: ...
|
||||
|
||||
def clear_cache(self, database: str) -> None: ...
|
||||
|
||||
def ensure_sync_indexes(self, database: str) -> None:
|
||||
"""Create any index needed for the sync write path.
|
||||
|
||||
Called once at the start of each provider sync; must be idempotent.
|
||||
Neo4j creates a `_provider_element_id` index on `_ProviderResource`;
|
||||
Neptune is a no-op (its `~id` lookup needs no index).
|
||||
"""
|
||||
...
|
||||
|
||||
def write_nodes(
|
||||
self,
|
||||
database: str,
|
||||
labels: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Upsert a batch of nodes into the sink.
|
||||
|
||||
`labels` is a pre-rendered Cypher label string ready to drop after
|
||||
the node variable (e.g. `` `AWSUser`:`_ProviderResource`:`_Tenant_x` ``).
|
||||
Each row carries `provider_element_id` and `props`.
|
||||
"""
|
||||
...
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
database: str,
|
||||
rel_type: str,
|
||||
provider_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Upsert a batch of relationships into the sink.
|
||||
|
||||
Each row carries `start_element_id`, `end_element_id`,
|
||||
`provider_element_id` and `props`. `rel_type` is the relationship
|
||||
type (already a valid Cypher identifier).
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Shared batched deletion helpers for sink backends."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES = {
|
||||
"outgoing relationship": """
|
||||
MATCH (n:`{provider_label}`)-[r]->()
|
||||
WITH r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
"incoming relationship": """
|
||||
MATCH (n:`{provider_label}`)<-[r]-()
|
||||
WITH r LIMIT $batch_size
|
||||
DELETE r
|
||||
RETURN COUNT(r) AS deleted_rels_count
|
||||
""",
|
||||
}
|
||||
|
||||
NODE_DELETE_QUERY_TEMPLATE = """
|
||||
MATCH (n:{provider_resource_label}:`{provider_label}`)
|
||||
WITH n LIMIT $batch_size
|
||||
DELETE n
|
||||
RETURN COUNT(n) AS deleted_nodes_count
|
||||
"""
|
||||
|
||||
|
||||
def delete_batches(
|
||||
*,
|
||||
session: Any,
|
||||
logger: logging.Logger,
|
||||
log_target: str,
|
||||
provider_id: str,
|
||||
query: str,
|
||||
phase: str,
|
||||
count_key: str,
|
||||
total_key: str,
|
||||
deleted_key: str,
|
||||
initial_total: int,
|
||||
batch_size: int,
|
||||
drop_t0: float,
|
||||
) -> tuple[int, int]:
|
||||
def delete_batch(tx: Any) -> int:
|
||||
record = tx.run(query, {"batch_size": batch_size}).single()
|
||||
return (record[count_key] if record else 0) or 0
|
||||
|
||||
deleted_total = initial_total
|
||||
batches = 0
|
||||
while True:
|
||||
logger.info(
|
||||
"Deleting %s batch from %s "
|
||||
"(provider=%s, batch=%s, total_%s=%s, elapsed=%.3fs)",
|
||||
phase,
|
||||
log_target,
|
||||
provider_id,
|
||||
batches + 1,
|
||||
total_key,
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
deleted = session.execute_write(delete_batch)
|
||||
if deleted == 0:
|
||||
return deleted_total, batches
|
||||
|
||||
batches += 1
|
||||
deleted_total += deleted
|
||||
logger.info(
|
||||
"Deleted %s batch from %s "
|
||||
"(provider=%s, batch=%s, %s=%s, total_%s=%s, elapsed=%.3fs)",
|
||||
phase,
|
||||
log_target,
|
||||
provider_id,
|
||||
batches,
|
||||
deleted_key,
|
||||
deleted,
|
||||
total_key,
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Sink backend factory and process-wide handle cache.
|
||||
|
||||
Picks the active backend from `settings.ATTACK_PATHS_SINK_DATABASE` at first
|
||||
use, holds the active backend plus any secondary backends needed to serve
|
||||
scans written under the previous configuration, and tears them all down on
|
||||
process shutdown. Imported via `from api.attack_paths import sink as
|
||||
sink_module`.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from enum import StrEnum, auto
|
||||
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.models import AttackPathsScan
|
||||
from django.conf import settings
|
||||
|
||||
# Backend names
|
||||
|
||||
|
||||
class SinkBackend(StrEnum):
|
||||
NEO4J = auto()
|
||||
NEPTUNE = auto()
|
||||
|
||||
|
||||
# Backend cache
|
||||
|
||||
_backend: SinkDatabase | None = None
|
||||
_secondary_backends: dict[SinkBackend, SinkDatabase] = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def _resolve_setting() -> SinkBackend:
|
||||
raw = settings.ATTACK_PATHS_SINK_DATABASE.lower()
|
||||
try:
|
||||
return SinkBackend(raw)
|
||||
|
||||
except ValueError:
|
||||
valid = sorted(b.value for b in SinkBackend)
|
||||
raise RuntimeError(
|
||||
f"ATTACK_PATHS_SINK_DATABASE must be one of {valid}; got {raw!r}"
|
||||
)
|
||||
|
||||
|
||||
def _build_backend(name: SinkBackend) -> SinkDatabase:
|
||||
if name is SinkBackend.NEO4J:
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
return Neo4jSink()
|
||||
|
||||
if name is SinkBackend.NEPTUNE:
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
return NeptuneSink()
|
||||
|
||||
raise RuntimeError(f"Unknown sink backend {name!r}")
|
||||
|
||||
|
||||
# Lifecycle
|
||||
|
||||
|
||||
def init(name: SinkBackend | str | None = None) -> SinkDatabase:
|
||||
"""Initialize the configured sink backend. Idempotent."""
|
||||
global _backend
|
||||
if _backend is not None:
|
||||
return _backend
|
||||
|
||||
with _lock:
|
||||
if _backend is None:
|
||||
resolved = SinkBackend(name) if name else _resolve_setting()
|
||||
backend = _build_backend(resolved)
|
||||
backend.init()
|
||||
_backend = backend
|
||||
|
||||
return _backend
|
||||
|
||||
|
||||
def close() -> None:
|
||||
"""Close the active backend and every cached secondary backend."""
|
||||
global _backend
|
||||
with _lock:
|
||||
backends = [
|
||||
b for b in (_backend, *_secondary_backends.values()) if b is not None
|
||||
]
|
||||
_backend = None
|
||||
_secondary_backends.clear()
|
||||
|
||||
for backend in backends:
|
||||
try:
|
||||
backend.close()
|
||||
|
||||
except Exception: # pragma: no cover - best-effort
|
||||
pass
|
||||
|
||||
|
||||
def get_backend() -> SinkDatabase:
|
||||
"""Return the active sink. Initializes on first call."""
|
||||
return init()
|
||||
|
||||
|
||||
# Per-scan routing
|
||||
|
||||
|
||||
def get_backend_for_scan(scan: AttackPathsScan) -> SinkDatabase:
|
||||
"""Route reads by the sink that stores this scan's graph."""
|
||||
raw_backend = getattr(scan, "sink_backend", SinkBackend.NEO4J.value)
|
||||
if not isinstance(raw_backend, str):
|
||||
raw_backend = SinkBackend.NEO4J.value
|
||||
return get_backend_for_name(raw_backend)
|
||||
|
||||
|
||||
def get_backend_for_name(name: SinkBackend | str) -> SinkDatabase:
|
||||
"""Return the backend named by persisted scan metadata."""
|
||||
resolved = SinkBackend(name)
|
||||
if resolved is _resolve_setting():
|
||||
return get_backend()
|
||||
|
||||
return _build_backend_cached(resolved)
|
||||
|
||||
|
||||
def _build_backend_cached(name: SinkBackend) -> SinkDatabase:
|
||||
# TODO: drop after Neptune cutover
|
||||
# Needed only during cutover to serve Neo4j-written scans from a Neptune-
|
||||
# configured API pod (and vice versa). Once every scan is on Neptune,
|
||||
# `get_backend_for_scan` becomes a one-liner returning `get_backend()`.
|
||||
if name in _secondary_backends:
|
||||
return _secondary_backends[name]
|
||||
|
||||
with _lock:
|
||||
if name not in _secondary_backends:
|
||||
backend = _build_backend(name)
|
||||
backend.init()
|
||||
_secondary_backends[name] = backend
|
||||
|
||||
return _secondary_backends[name]
|
||||
@@ -0,0 +1,419 @@
|
||||
"""Neo4j sink implementation.
|
||||
|
||||
Owns a Neo4j driver independent from the staging driver. On OSS and local dev
|
||||
this is the only sink; on hosted deployments it runs only as a legacy read
|
||||
path while phase-1 drains tenant DBs.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from typing import Any
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.attack_paths.sink.drop import (
|
||||
NODE_DELETE_QUERY_TEMPLATE,
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
|
||||
delete_batches,
|
||||
)
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = env.int(
|
||||
"ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES", default=3
|
||||
)
|
||||
READ_QUERY_TIMEOUT_SECONDS = env.int(
|
||||
"ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30
|
||||
)
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEO4J_CONN_ACQUISITION_TIMEOUT", default=15)
|
||||
# TCP connect timeout, ordered below the acquisition timeout so an unreachable
|
||||
# host can't pin a request or the readiness probe longer than this.
|
||||
CONNECTION_TIMEOUT = env.int("NEO4J_CONNECTION_TIMEOUT", default=5)
|
||||
MAX_CONNECTION_LIFETIME = env.int("NEO4J_MAX_CONNECTION_LIFETIME", default=7200)
|
||||
MAX_CONNECTION_POOL_SIZE = env.int("NEO4J_MAX_CONNECTION_POOL_SIZE", default=50)
|
||||
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
|
||||
|
||||
|
||||
class Neo4jSink(SinkDatabase):
|
||||
"""Neo4j-backed sink. Multi-database cluster; tenant isolation is physical."""
|
||||
|
||||
sync_batch_size = env.int("ATTACK_PATHS_NEO4J_SYNC_BATCH_SIZE", default=1000)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._driver: neo4j.Driver | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._atexit_registered = False
|
||||
|
||||
# Driver
|
||||
|
||||
def _config(self) -> dict:
|
||||
return settings.DATABASES["neo4j"]
|
||||
|
||||
def _uri(self) -> str:
|
||||
cfg = self._config()
|
||||
host = cfg["HOST"]
|
||||
port = cfg["PORT"]
|
||||
if not host or not port:
|
||||
raise RuntimeError(
|
||||
"NEO4J_HOST / NEO4J_PORT must be set when ATTACK_PATHS_SINK_DATABASE=neo4j"
|
||||
)
|
||||
return f"bolt://{host}:{port}"
|
||||
|
||||
def init(self) -> neo4j.Driver:
|
||||
if self._driver is not None:
|
||||
return self._driver
|
||||
with self._lock:
|
||||
if self._driver is None:
|
||||
cfg = self._config()
|
||||
self._driver = neo4j.GraphDatabase.driver(
|
||||
self._uri(),
|
||||
auth=(cfg["USER"], cfg["PASSWORD"]),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=MAX_CONNECTION_LIFETIME,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=MAX_CONNECTION_POOL_SIZE,
|
||||
)
|
||||
# Eager connectivity check is best-effort:
|
||||
# A Neo4j that is down at boot must not crash the process, same degradation model as Postgres
|
||||
# The driver reconnects lazily on first use
|
||||
# /health/ready surfaces the outage until it recovers
|
||||
try:
|
||||
self._driver.verify_connectivity()
|
||||
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Neo4j sink unreachable at init; continuing with a lazily-reconnecting driver",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if not self._atexit_registered:
|
||||
atexit.register(self.close)
|
||||
self._atexit_registered = True
|
||||
return self._driver
|
||||
|
||||
def _get_driver(self) -> neo4j.Driver:
|
||||
return self.init()
|
||||
|
||||
def verify_connectivity(self) -> None:
|
||||
self._get_driver().verify_connectivity()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
if self._driver is not None:
|
||||
try:
|
||||
self._driver.close()
|
||||
finally:
|
||||
self._driver = None
|
||||
|
||||
# Sessions
|
||||
|
||||
@contextmanager
|
||||
def get_session(
|
||||
self,
|
||||
database: str | None = None,
|
||||
default_access_mode: str | None = None,
|
||||
) -> Iterator[RetryableSession]:
|
||||
from api.attack_paths.database import (
|
||||
ClientStatementException,
|
||||
GraphDatabaseQueryException,
|
||||
WriteQueryNotAllowedException,
|
||||
)
|
||||
|
||||
session_wrapper: RetryableSession | None = None
|
||||
try:
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: self._get_driver().session(
|
||||
database=database, default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
if (
|
||||
default_access_mode == neo4j.READ_ACCESS
|
||||
and exc.code
|
||||
and exc.code in READ_EXCEPTION_CODES
|
||||
):
|
||||
raise WriteQueryNotAllowedException(
|
||||
message="Read query not allowed", code=READ_EXCEPTION_CODES[0]
|
||||
)
|
||||
|
||||
message = exc.message if exc.message is not None else str(exc)
|
||||
if exc.code and exc.code.startswith(CLIENT_STATEMENT_EXCEPTION_PREFIX):
|
||||
raise ClientStatementException(message=message, code=exc.code)
|
||||
raise GraphDatabaseQueryException(message=message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
# Operations
|
||||
|
||||
def execute_read_query(
|
||||
self,
|
||||
database: str,
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph:
|
||||
with self.get_session(
|
||||
database, default_access_mode=neo4j.READ_ACCESS
|
||||
) as session:
|
||||
|
||||
def _run(tx: neo4j.ManagedTransaction) -> neo4j.graph.Graph:
|
||||
result = tx.run(
|
||||
cypher, parameters or {}, timeout=READ_QUERY_TIMEOUT_SECONDS
|
||||
)
|
||||
return result.graph()
|
||||
|
||||
return session.execute_read(_run)
|
||||
|
||||
def create_database(self, database: str) -> None:
|
||||
with self.get_session() as session:
|
||||
session.run(
|
||||
"CREATE DATABASE $database IF NOT EXISTS", {"database": database}
|
||||
)
|
||||
|
||||
def drop_database(self, database: str) -> None:
|
||||
with self.get_session() as session:
|
||||
session.run(f"DROP DATABASE `{database}` IF EXISTS DESTROY DATA")
|
||||
|
||||
def drop_subgraph(self, database: str, provider_id: str) -> int:
|
||||
"""Delete all nodes for a provider from a tenant database, batched.
|
||||
|
||||
Deletes relationships then nodes in batches (not `DETACH DELETE`) so a
|
||||
dense provider's graph cannot exceed Neo4j's transaction memory limit.
|
||||
Silently returns 0 if the database doesn't exist.
|
||||
"""
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
GRAPH_MUTATION_BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
deleted_nodes = deleted_relationships = 0
|
||||
relationship_batches = node_batches = 0
|
||||
drop_t0 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"Dropping provider graph from Neo4j sink database %s "
|
||||
"(provider=%s, provider_label=%s)",
|
||||
database,
|
||||
provider_id,
|
||||
provider_label,
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
"Opening Neo4j sink session for provider graph drop "
|
||||
"(database=%s, provider=%s)",
|
||||
database,
|
||||
provider_id,
|
||||
)
|
||||
with self.get_session(database) as session:
|
||||
logger.info(
|
||||
"Opened Neo4j sink session for provider graph drop "
|
||||
"(database=%s, provider=%s)",
|
||||
database,
|
||||
provider_id,
|
||||
)
|
||||
log_target = f"Neo4j sink database {database}"
|
||||
for (
|
||||
phase,
|
||||
query_template,
|
||||
) in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
|
||||
deleted_relationships, phase_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target=log_target,
|
||||
provider_id=provider_id,
|
||||
query=query_template.format(provider_label=provider_label),
|
||||
phase=phase,
|
||||
count_key="deleted_rels_count",
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=GRAPH_MUTATION_BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
|
||||
deleted_nodes, node_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target=log_target,
|
||||
provider_id=provider_id,
|
||||
query=NODE_DELETE_QUERY_TEMPLATE.format(
|
||||
provider_label=provider_label,
|
||||
provider_resource_label=PROVIDER_RESOURCE_LABEL,
|
||||
),
|
||||
phase="node",
|
||||
count_key="deleted_nodes_count",
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=GRAPH_MUTATION_BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == DATABASE_NOT_FOUND_CODE:
|
||||
logger.info(
|
||||
"Skipped provider graph drop from Neo4j sink database %s "
|
||||
"(provider=%s, reason=database_not_found, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
return 0
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"Finished dropping provider graph from Neo4j sink database %s "
|
||||
"(provider=%s, relationship_batches=%s, deleted_rels=%s, "
|
||||
"node_batches=%s, deleted_nodes=%s, elapsed=%.3fs)",
|
||||
database,
|
||||
provider_id,
|
||||
relationship_batches,
|
||||
deleted_relationships,
|
||||
node_batches,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
return deleted_nodes
|
||||
|
||||
def has_provider_data(self, database: str, provider_id: str) -> bool:
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
query = (
|
||||
f"MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`) RETURN 1 LIMIT 1"
|
||||
)
|
||||
try:
|
||||
with self.get_session(
|
||||
database, default_access_mode=neo4j.READ_ACCESS
|
||||
) as session:
|
||||
result = session.run(query)
|
||||
return result.single() is not None
|
||||
|
||||
except GraphDatabaseQueryException as exc:
|
||||
if exc.code == DATABASE_NOT_FOUND_CODE:
|
||||
return False
|
||||
raise
|
||||
|
||||
def clear_cache(self, database: str) -> None:
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
|
||||
try:
|
||||
with self.get_session(database) as session:
|
||||
session.run("CALL db.clearQueryCaches()")
|
||||
except GraphDatabaseQueryException as exc:
|
||||
logger.warning(
|
||||
f"Failed to clear query cache for database `{database}`: {exc}"
|
||||
)
|
||||
|
||||
# Sync write path
|
||||
|
||||
def ensure_sync_indexes(self, database: str) -> None:
|
||||
"""Create the `_provider_element_id` lookup index on `_ProviderResource`.
|
||||
|
||||
Every synced node carries the `_ProviderResource` label, so a single
|
||||
index covers both node-upserts and relationship endpoint MATCHes.
|
||||
Without this index the rel sync degrades to a label scan per row and
|
||||
large provider syncs become unworkable.
|
||||
"""
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_ELEMENT_ID_PROPERTY,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
)
|
||||
|
||||
query = (
|
||||
f"CREATE INDEX provider_element_id_idx IF NOT EXISTS "
|
||||
f"FOR (n:`{PROVIDER_RESOURCE_LABEL}`) "
|
||||
f"ON (n.`{PROVIDER_ELEMENT_ID_PROPERTY}`)"
|
||||
)
|
||||
with self.get_session(database) as session:
|
||||
session.execute_write(lambda tx: tx.run(query).consume())
|
||||
|
||||
def write_nodes(
|
||||
self,
|
||||
database: str,
|
||||
labels: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_ELEMENT_ID_PROPERTY,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
)
|
||||
|
||||
query = f"""
|
||||
UNWIND $rows AS row
|
||||
MERGE (n:`{PROVIDER_RESOURCE_LABEL}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.provider_element_id}})
|
||||
SET n:{labels}
|
||||
SET n += row.props
|
||||
"""
|
||||
with self.get_session(database) as session:
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
database: str,
|
||||
rel_type: str,
|
||||
provider_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_ELEMENT_ID_PROPERTY,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
query = f"""
|
||||
UNWIND $rows AS row
|
||||
MATCH (s:`{PROVIDER_RESOURCE_LABEL}`:`{provider_label}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.start_element_id}})
|
||||
MATCH (t:`{PROVIDER_RESOURCE_LABEL}`:`{provider_label}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.end_element_id}})
|
||||
MERGE (s)-[r:`{rel_type}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.provider_element_id}}]->(t)
|
||||
SET r += row.props
|
||||
"""
|
||||
with self.get_session(database) as session:
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
# For compatibility with test harnesses that patch the concrete driver
|
||||
def get_driver(self) -> neo4j.Driver:
|
||||
return self._get_driver()
|
||||
|
||||
|
||||
# Helper for tests / external callers that want a writer session specifically
|
||||
def get_read_session(
|
||||
sink: Neo4jSink, database: str
|
||||
) -> AbstractContextManager[RetryableSession]:
|
||||
return sink.get_session(database, default_access_mode=neo4j.READ_ACCESS)
|
||||
@@ -0,0 +1,519 @@
|
||||
"""AWS Neptune sink implementation.
|
||||
|
||||
Dual Bolt drivers: one against the writer endpoint for workers, one against
|
||||
the reader endpoint for the API read path. If `NEPTUNE_READER_ENDPOINT` is
|
||||
unset the reader falls back to the writer driver so single-node clusters work.
|
||||
|
||||
Neptune is single-database. The `database` argument on the SinkDatabase
|
||||
protocol is ignored; tenant / provider isolation is enforced by labels that
|
||||
the sync step already writes on every node (see tasks/jobs/attack_paths/sync.py).
|
||||
|
||||
SigV4 auth lives at the bottom of this file as `neptune_auth_provider`. The
|
||||
neo4j driver invokes the returned callable on each token refresh.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import neo4j
|
||||
import neo4j.exceptions
|
||||
from api.attack_paths.retryable_session import RetryableSession, RetryExhaustedError
|
||||
from api.attack_paths.sink.base import SinkDatabase
|
||||
from api.attack_paths.sink.drop import (
|
||||
NODE_DELETE_QUERY_TEMPLATE,
|
||||
RELATIONSHIP_DELETE_QUERY_TEMPLATES,
|
||||
delete_batches,
|
||||
)
|
||||
from botocore.auth import SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
from botocore.session import Session as BotoSession
|
||||
from config.env import env
|
||||
from django.conf import settings
|
||||
from neo4j.auth_management import AuthManagers, ExpiringAuth
|
||||
|
||||
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
||||
logging.getLogger("neo4j").propagate = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SERVICE_UNAVAILABLE_MAX_RETRIES = env.int(
|
||||
"ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES", default=3
|
||||
)
|
||||
READ_QUERY_TIMEOUT_SECONDS = env.int(
|
||||
"ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30
|
||||
)
|
||||
# Neptune serverless cold-start can be >30s; give the driver room
|
||||
CONN_ACQUISITION_TIMEOUT = env.int("NEPTUNE_CONN_ACQUISITION_TIMEOUT", default=60)
|
||||
# TCP connect timeout, ordered below the acquisition timeout so an unreachable
|
||||
# endpoint can't pin a request or the readiness probe longer than this. Kept
|
||||
# generous: cold-start delays query execution, not the socket connect.
|
||||
CONNECTION_TIMEOUT = env.int("NEPTUNE_CONNECTION_TIMEOUT", default=10)
|
||||
# Roll connections hourly so SigV4 rotations and cert refreshes don't strand long-lived pool entries
|
||||
MAX_CONNECTION_LIFETIME = env.int("NEPTUNE_MAX_CONNECTION_LIFETIME", default=3600)
|
||||
MAX_CONNECTION_POOL_SIZE = env.int("NEPTUNE_MAX_CONNECTION_POOL_SIZE", default=50)
|
||||
NEPTUNE_WRITE_RETRY_DELAY_SECONDS = 2
|
||||
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
RETRYABLE_WRITE_ERROR_FRAGMENTS = (
|
||||
"Operation failed due to conflicting concurrent operations",
|
||||
"Operation terminated (deadline exceeded)",
|
||||
)
|
||||
|
||||
# Refresh 60s before the 5-minute SigV4 window closes
|
||||
SIGV4_TOKEN_LIFETIME_MINUTES = 4
|
||||
|
||||
|
||||
def _is_retryable_write_error(exc: Exception) -> bool:
|
||||
if not isinstance(exc, neo4j.exceptions.Neo4jError):
|
||||
return False
|
||||
message = exc.message or ""
|
||||
return any(fragment in message for fragment in RETRYABLE_WRITE_ERROR_FRAGMENTS)
|
||||
|
||||
|
||||
class NeptuneSink(SinkDatabase):
|
||||
"""Neptune-backed sink. Single database; isolation is label-based."""
|
||||
|
||||
sync_batch_size = env.int("ATTACK_PATHS_NEPTUNE_SYNC_BATCH_SIZE", default=500)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._writer: neo4j.Driver | None = None
|
||||
self._reader: neo4j.Driver | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._atexit_registered = False
|
||||
|
||||
# Config
|
||||
|
||||
def _config(self) -> dict:
|
||||
return settings.DATABASES["neptune"]
|
||||
|
||||
def _bolt_uri(self, endpoint: str, port: str) -> str:
|
||||
return f"bolt+s://{endpoint}:{port}"
|
||||
|
||||
def _https_url(self, endpoint: str, port: str) -> str:
|
||||
return f"https://{endpoint}:{port}"
|
||||
|
||||
def _build_driver(self, endpoint: str) -> neo4j.Driver:
|
||||
cfg = self._config()
|
||||
port = cfg["PORT"]
|
||||
region = cfg["REGION"]
|
||||
if not endpoint or not region:
|
||||
raise RuntimeError(
|
||||
"NEPTUNE_WRITER_ENDPOINT and AWS_REGION must be set when "
|
||||
"ATTACK_PATHS_SINK_DATABASE=neptune"
|
||||
)
|
||||
return neo4j.GraphDatabase.driver(
|
||||
self._bolt_uri(endpoint, port),
|
||||
auth=AuthManagers.bearer(
|
||||
neptune_auth_provider(region, self._https_url(endpoint, port))
|
||||
),
|
||||
keep_alive=True,
|
||||
max_connection_lifetime=MAX_CONNECTION_LIFETIME,
|
||||
connection_timeout=CONNECTION_TIMEOUT,
|
||||
connection_acquisition_timeout=CONN_ACQUISITION_TIMEOUT,
|
||||
max_connection_pool_size=MAX_CONNECTION_POOL_SIZE,
|
||||
max_transaction_retry_time=0,
|
||||
)
|
||||
|
||||
# Lifecycle
|
||||
|
||||
def init(self) -> None:
|
||||
if self._writer is not None:
|
||||
return
|
||||
with self._lock:
|
||||
if self._writer is None:
|
||||
cfg = self._config()
|
||||
writer_endpoint = cfg["WRITER_ENDPOINT"]
|
||||
reader_endpoint = cfg["READER_ENDPOINT"] or writer_endpoint
|
||||
|
||||
# Eager connectivity checks are best-effort
|
||||
# A Neptune that is down at boot must not crash the process, same degradation model as Postgres
|
||||
# Drivers reconnect lazily on first use
|
||||
# /health/ready surfaces the outage until it recovers
|
||||
self._writer = self._build_driver(writer_endpoint)
|
||||
self._verify_best_effort(self._writer, "writer")
|
||||
|
||||
if reader_endpoint == writer_endpoint:
|
||||
self._reader = self._writer
|
||||
|
||||
else:
|
||||
self._reader = self._build_driver(reader_endpoint)
|
||||
self._verify_best_effort(self._reader, "reader")
|
||||
|
||||
if not self._atexit_registered:
|
||||
atexit.register(self.close)
|
||||
self._atexit_registered = True
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
# `Driver.close()` is idempotent, so closing the same driver twice
|
||||
# (when reader aliases writer on single-endpoint configs) is safe
|
||||
for driver in (self._reader, self._writer):
|
||||
if driver is None:
|
||||
continue
|
||||
try:
|
||||
driver.close()
|
||||
except Exception: # pragma: no cover - best-effort
|
||||
pass
|
||||
self._writer = None
|
||||
self._reader = None
|
||||
|
||||
# Sessions
|
||||
|
||||
def _get_writer(self) -> neo4j.Driver:
|
||||
self.init()
|
||||
assert self._writer is not None
|
||||
return self._writer
|
||||
|
||||
def _get_reader(self) -> neo4j.Driver:
|
||||
self.init()
|
||||
assert self._reader is not None
|
||||
return self._reader
|
||||
|
||||
@staticmethod
|
||||
def _verify_best_effort(driver: neo4j.Driver, role: str) -> None:
|
||||
try:
|
||||
driver.verify_connectivity()
|
||||
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Neptune %s endpoint unreachable at init; continuing with a lazily-reconnecting driver",
|
||||
role,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def verify_connectivity(self) -> None:
|
||||
# The API read path uses the reader driver
|
||||
# On single-endpoint clusters it aliases the writer, so this also covers the writer
|
||||
# A writer-only outage is a workers' concern (no HTTP probe there) and deliberately does not fail API readiness
|
||||
self._get_reader().verify_connectivity()
|
||||
|
||||
@contextmanager
|
||||
def get_session(
|
||||
self,
|
||||
database: str | None = None, # noqa: ARG002 - ignored on Neptune
|
||||
default_access_mode: str | None = None,
|
||||
) -> Iterator[RetryableSession]:
|
||||
from api.attack_paths.database import (
|
||||
ClientStatementException,
|
||||
GraphDatabaseQueryException,
|
||||
NeptuneWriteRetryExhaustedException,
|
||||
WriteQueryNotAllowedException,
|
||||
)
|
||||
|
||||
driver = (
|
||||
self._get_reader()
|
||||
if default_access_mode == neo4j.READ_ACCESS
|
||||
else self._get_writer()
|
||||
)
|
||||
|
||||
session_wrapper: RetryableSession | None = None
|
||||
try:
|
||||
is_write_session = default_access_mode != neo4j.READ_ACCESS
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: driver.session(
|
||||
default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
retry_if=_is_retryable_write_error if is_write_session else None,
|
||||
initial_retry_delay_seconds=(
|
||||
NEPTUNE_WRITE_RETRY_DELAY_SECONDS if is_write_session else 0
|
||||
),
|
||||
retry_context="Neptune write" if is_write_session else None,
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
except RetryExhaustedError as exc:
|
||||
last_error = exc.last_error
|
||||
raise NeptuneWriteRetryExhaustedException(
|
||||
message=str(exc),
|
||||
code=getattr(last_error, "code", None),
|
||||
) from last_error
|
||||
|
||||
except neo4j.exceptions.Neo4jError as exc:
|
||||
if (
|
||||
default_access_mode == neo4j.READ_ACCESS
|
||||
and exc.code
|
||||
and exc.code in READ_EXCEPTION_CODES
|
||||
):
|
||||
raise WriteQueryNotAllowedException(
|
||||
message="Read query not allowed", code=READ_EXCEPTION_CODES[0]
|
||||
)
|
||||
|
||||
message = exc.message if exc.message is not None else str(exc)
|
||||
if exc.code and exc.code.startswith(CLIENT_STATEMENT_EXCEPTION_PREFIX):
|
||||
raise ClientStatementException(message=message, code=exc.code)
|
||||
raise GraphDatabaseQueryException(message=message, code=exc.code)
|
||||
|
||||
finally:
|
||||
if session_wrapper is not None:
|
||||
session_wrapper.close()
|
||||
|
||||
# Operations
|
||||
|
||||
def execute_read_query(
|
||||
self,
|
||||
database: str, # noqa: ARG002 - ignored on Neptune
|
||||
cypher: str,
|
||||
parameters: dict[str, Any] | None = None,
|
||||
) -> neo4j.graph.Graph:
|
||||
with self.get_session(default_access_mode=neo4j.READ_ACCESS) as session:
|
||||
|
||||
def _run(tx: neo4j.ManagedTransaction) -> neo4j.graph.Graph:
|
||||
result = tx.run(
|
||||
cypher, parameters or {}, timeout=READ_QUERY_TIMEOUT_SECONDS
|
||||
)
|
||||
return result.graph()
|
||||
|
||||
return session.execute_read(_run)
|
||||
|
||||
def create_database(self, database: str) -> None: # noqa: ARG002
|
||||
# Neptune clusters are single-database; there is nothing to create.
|
||||
return None
|
||||
|
||||
def drop_database(self, database: str) -> None: # noqa: ARG002
|
||||
# Neptune clusters are single-database; there is nothing to drop.
|
||||
return None
|
||||
|
||||
def drop_subgraph(self, database: str, provider_id: str) -> int: # noqa: ARG002
|
||||
"""Delete a provider's subgraph in two bounded phases.
|
||||
|
||||
Neptune write transactions are capped at ~2 minutes. A naive
|
||||
`DETACH DELETE` on a label-scanned batch grows unbounded with graph
|
||||
density (one node can drag thousands of relationships into the same
|
||||
transaction). Instead:
|
||||
|
||||
1. Delete relationships incident to provider nodes, one fixed-size
|
||||
batch per transaction.
|
||||
2. Delete the now-orphaned nodes, one fixed-size batch per transaction.
|
||||
|
||||
Each transaction does work proportional to `batch_size`, never to the
|
||||
graph's branching factor.
|
||||
"""
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
GRAPH_MUTATION_BATCH_SIZE,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
deleted_relationships = 0
|
||||
relationship_batches = 0
|
||||
node_batches = 0
|
||||
drop_t0 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"Dropping provider graph from Neptune sink "
|
||||
"(provider=%s, provider_label=%s)",
|
||||
provider_id,
|
||||
provider_label,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Opening Neptune writer session for provider graph drop (provider=%s)",
|
||||
provider_id,
|
||||
)
|
||||
with self.get_session() as session:
|
||||
logger.info(
|
||||
"Opened Neptune writer session for provider graph drop (provider=%s)",
|
||||
provider_id,
|
||||
)
|
||||
for phase, query_template in RELATIONSHIP_DELETE_QUERY_TEMPLATES.items():
|
||||
deleted_relationships, phase_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target="Neptune sink",
|
||||
provider_id=provider_id,
|
||||
query=query_template.format(provider_label=provider_label),
|
||||
phase=phase,
|
||||
count_key="deleted_rels_count",
|
||||
total_key="rels",
|
||||
deleted_key="deleted_rels",
|
||||
initial_total=deleted_relationships,
|
||||
batch_size=GRAPH_MUTATION_BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
relationship_batches += phase_batches
|
||||
|
||||
deleted_nodes, node_batches = delete_batches(
|
||||
session=session,
|
||||
logger=logger,
|
||||
log_target="Neptune sink",
|
||||
provider_id=provider_id,
|
||||
query=NODE_DELETE_QUERY_TEMPLATE.format(
|
||||
provider_label=provider_label,
|
||||
provider_resource_label=PROVIDER_RESOURCE_LABEL,
|
||||
),
|
||||
phase="node",
|
||||
count_key="deleted_nodes_count",
|
||||
total_key="nodes",
|
||||
deleted_key="deleted_nodes",
|
||||
initial_total=0,
|
||||
batch_size=GRAPH_MUTATION_BATCH_SIZE,
|
||||
drop_t0=drop_t0,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Finished dropping provider graph from Neptune sink "
|
||||
"(provider=%s, relationship_batches=%s, deleted_rels=%s, "
|
||||
"node_batches=%s, deleted_nodes=%s, elapsed=%.3fs)",
|
||||
provider_id,
|
||||
relationship_batches,
|
||||
deleted_relationships,
|
||||
node_batches,
|
||||
deleted_nodes,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
return deleted_nodes
|
||||
|
||||
def has_provider_data(self, database: str, provider_id: str) -> bool: # noqa: ARG002
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
get_provider_label,
|
||||
)
|
||||
|
||||
provider_label = get_provider_label(provider_id)
|
||||
query = (
|
||||
f"MATCH (n:{PROVIDER_RESOURCE_LABEL}:`{provider_label}`) RETURN 1 LIMIT 1"
|
||||
)
|
||||
with self.get_session(default_access_mode=neo4j.READ_ACCESS) as session:
|
||||
result = session.run(query)
|
||||
return result.single() is not None
|
||||
|
||||
def clear_cache(self, database: str) -> None: # noqa: ARG002
|
||||
# Neptune has no user-facing cache-clear procedure; no-op.
|
||||
return None
|
||||
|
||||
# Sync write path
|
||||
|
||||
def ensure_sync_indexes(self, database: str) -> None: # noqa: ARG002
|
||||
# Neptune routes node and relationship lookups through `~id`, which is the cluster's primary key
|
||||
# No additional index is needed or supported
|
||||
return None
|
||||
|
||||
def write_nodes(
|
||||
self,
|
||||
database: str, # noqa: ARG002
|
||||
labels: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
PROVIDER_ELEMENT_ID_PROPERTY,
|
||||
PROVIDER_RESOURCE_LABEL,
|
||||
)
|
||||
|
||||
# MERGE on `~id` is the documented and engine-optimized idempotent
|
||||
# upsert pattern for Neptune openCypher. The label inside the MERGE
|
||||
# matters: Neptune assigns a default `vertex` label to any node
|
||||
# created without an explicit one, so we pin `_ProviderResource`
|
||||
# (which every synced node carries anyway) at MERGE-time. Additional
|
||||
# labels are added after
|
||||
#
|
||||
# We also write `_provider_element_id` as a regular property so
|
||||
# non-sync code (drop_subgraph, query helpers) keeps a stable contract
|
||||
# that doesn't know about `~id`
|
||||
query = f"""
|
||||
UNWIND $rows AS row
|
||||
MERGE (n:`{PROVIDER_RESOURCE_LABEL}` {{`~id`: row.provider_element_id}})
|
||||
SET n:{labels}
|
||||
SET n += row.props
|
||||
SET n.`{PROVIDER_ELEMENT_ID_PROPERTY}` = row.provider_element_id
|
||||
"""
|
||||
with self.get_session() as session:
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
database: str, # noqa: ARG002
|
||||
rel_type: str,
|
||||
provider_id: str, # noqa: ARG002 - encoded in start/end `~id` already
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
from tasks.jobs.attack_paths.config import PROVIDER_ELEMENT_ID_PROPERTY
|
||||
|
||||
# `id(n) = $value` is Neptune's parameterized fast path; both endpoint
|
||||
# MATCHes resolve in O(1) via the system `~id`, so per-row work stays
|
||||
# bounded regardless of batch size
|
||||
query = f"""
|
||||
UNWIND $rows AS row
|
||||
MATCH (s) WHERE id(s) = row.start_element_id
|
||||
MATCH (e) WHERE id(e) = row.end_element_id
|
||||
MERGE (s)-[r:`{rel_type}` {{`{PROVIDER_ELEMENT_ID_PROPERTY}`: row.provider_element_id}}]->(e)
|
||||
SET r += row.props
|
||||
"""
|
||||
with self.get_session() as session:
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
# Test helpers
|
||||
|
||||
def get_writer(self) -> neo4j.Driver:
|
||||
return self._get_writer()
|
||||
|
||||
def get_reader(self) -> neo4j.Driver:
|
||||
return self._get_reader()
|
||||
|
||||
|
||||
# SigV4 auth provider
|
||||
|
||||
|
||||
class _NeptuneAuthToken(neo4j.Auth):
|
||||
"""Neo4j Auth backed by a SigV4-signed GET to `/opencypher`."""
|
||||
|
||||
def __init__(self, region: str, url: str) -> None:
|
||||
session = BotoSession()
|
||||
credentials = session.get_credentials()
|
||||
if credentials is None:
|
||||
raise RuntimeError(
|
||||
"No AWS credentials available for Neptune SigV4 signing. "
|
||||
"Ensure the boto3 credential chain can resolve."
|
||||
)
|
||||
credentials = credentials.get_frozen_credentials()
|
||||
|
||||
request = AWSRequest(method="GET", url=url + "/opencypher")
|
||||
# SigV4 canonical Host must carry the real `host:port`
|
||||
# Neptune runs on a non-default port (8182), so `.hostname` would drop it and break signing
|
||||
request.headers.add_header("Host", urlsplit(url).netloc)
|
||||
SigV4Auth(credentials, "neptune-db", region).add_auth(request)
|
||||
|
||||
auth_obj = {
|
||||
header: request.headers[header]
|
||||
for header in (
|
||||
"Authorization",
|
||||
"X-Amz-Date",
|
||||
"X-Amz-Security-Token",
|
||||
"Host",
|
||||
)
|
||||
if header in request.headers
|
||||
}
|
||||
auth_obj["HttpMethod"] = "GET"
|
||||
|
||||
super().__init__("basic", "username", json.dumps(auth_obj))
|
||||
|
||||
|
||||
def neptune_auth_provider(region: str, https_url: str) -> Callable[[], ExpiringAuth]:
|
||||
"""Return a callable the neo4j driver can invoke to refresh credentials."""
|
||||
|
||||
def _provider() -> ExpiringAuth:
|
||||
token = _NeptuneAuthToken(region, https_url)
|
||||
expires_at = (
|
||||
datetime.datetime.now(datetime.UTC)
|
||||
+ datetime.timedelta(minutes=SIGV4_TOKEN_LIFETIME_MINUTES)
|
||||
).timestamp()
|
||||
return ExpiringAuth(auth=token, expires_at=expires_at)
|
||||
|
||||
return _provider
|
||||
@@ -5,6 +5,7 @@ from typing import Any
|
||||
import neo4j
|
||||
from api.attack_paths import AttackPathsQueryDefinition
|
||||
from api.attack_paths import database as graph_database
|
||||
from api.attack_paths import sink as sink_module
|
||||
from api.attack_paths.cypher_sanitizer import (
|
||||
inject_provider_label,
|
||||
validate_custom_query,
|
||||
@@ -14,7 +15,9 @@ from api.attack_paths.queries.schema import (
|
||||
RAW_SCHEMA_URL,
|
||||
get_cartography_schema_query,
|
||||
)
|
||||
from api.models import AttackPathsScan
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
from rest_framework.exceptions import APIException, PermissionDenied, ValidationError
|
||||
from tasks.jobs.attack_paths.config import (
|
||||
INTERNAL_LABELS,
|
||||
@@ -26,6 +29,10 @@ from tasks.jobs.attack_paths.config import (
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
|
||||
def _custom_query_timeout_ms() -> int:
|
||||
return env.int("ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS", default=30) * 1000
|
||||
|
||||
|
||||
# Predefined query helpers
|
||||
|
||||
|
||||
@@ -102,13 +109,13 @@ def execute_query(
|
||||
definition: AttackPathsQueryDefinition,
|
||||
parameters: dict[str, Any],
|
||||
provider_id: str,
|
||||
scan: AttackPathsScan,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
graph = graph_database.execute_read_query(
|
||||
database=database_name,
|
||||
cypher=definition.cypher,
|
||||
parameters=parameters,
|
||||
)
|
||||
# TODO: drop after Neptune cutover
|
||||
# Route reads by the scan row's recorded sink, not by current settings.
|
||||
backend = sink_module.get_backend_for_scan(scan)
|
||||
graph = backend.execute_read_query(database_name, definition.cypher, parameters)
|
||||
return _serialize_graph(graph, provider_id)
|
||||
|
||||
except graph_database.WriteQueryNotAllowedException:
|
||||
@@ -142,22 +149,31 @@ def execute_custom_query(
|
||||
database_name: str,
|
||||
cypher: str,
|
||||
provider_id: str,
|
||||
scan: AttackPathsScan,
|
||||
) -> dict[str, Any]:
|
||||
# Defense-in-depth for custom queries:
|
||||
# 1. neo4j.READ_ACCESS — prevents mutations at the driver level
|
||||
# 2. inject_provider_label() — regex-based label injection scopes node patterns
|
||||
# 3. _serialize_graph() — post-query filter drops nodes without the provider label
|
||||
# 1. `neo4j.READ_ACCESS` - prevents mutations at the driver level
|
||||
# 2. `inject_provider_label()` - regex-based label injection scopes node patterns
|
||||
# 3. `_serialize_graph()` - post-query filter drops nodes without the provider label
|
||||
# 4. `USING QUERY:TIMEOUTMILLISECONDS` on Neptune - server-side runaway cutoff
|
||||
#
|
||||
# Layer 2 is best-effort (regex can't fully parse Cypher);
|
||||
# layer 3 is the safety net that guarantees provider isolation.
|
||||
validate_custom_query(cypher)
|
||||
cypher = inject_provider_label(cypher, provider_id)
|
||||
|
||||
# TODO: drop after Neptune cutover
|
||||
backend = sink_module.get_backend_for_scan(scan)
|
||||
|
||||
# Neptune enforces a cluster-level query timeout; prepending the hint
|
||||
# makes the limit explicit and matches the client-side read timeout.
|
||||
# Applies only when the scan's graph lives in Neptune.
|
||||
if getattr(scan, "sink_backend", None) == "neptune":
|
||||
timeout_ms = _custom_query_timeout_ms()
|
||||
cypher = f"USING QUERY:TIMEOUTMILLISECONDS {timeout_ms}\n{cypher}"
|
||||
|
||||
try:
|
||||
graph = graph_database.execute_read_query(
|
||||
database=database_name,
|
||||
cypher=cypher,
|
||||
)
|
||||
graph = backend.execute_read_query(database_name, cypher, None)
|
||||
serialized = _serialize_graph(graph, provider_id)
|
||||
return _truncate_graph(serialized)
|
||||
|
||||
@@ -180,10 +196,11 @@ def execute_custom_query(
|
||||
|
||||
|
||||
def get_cartography_schema(
|
||||
database_name: str, provider_id: str
|
||||
database_name: str, provider_id: str, scan: AttackPathsScan
|
||||
) -> dict[str, str] | None:
|
||||
try:
|
||||
with graph_database.get_session(
|
||||
backend = sink_module.get_backend_for_scan(scan)
|
||||
with backend.get_session(
|
||||
database_name, default_access_mode=neo4j.READ_ACCESS
|
||||
) as session:
|
||||
result = session.run(get_cartography_schema_query(provider_id))
|
||||
|
||||
+273
-50
@@ -2,7 +2,7 @@ import re
|
||||
import secrets
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from api.db_router import (
|
||||
@@ -48,6 +48,140 @@ REPLICA_MAX_ATTEMPTS = env.int("POSTGRES_REPLICA_MAX_ATTEMPTS", default=3)
|
||||
REPLICA_RETRY_BASE_DELAY = env.float("POSTGRES_REPLICA_RETRY_BASE_DELAY", default=0.5)
|
||||
|
||||
SET_CONFIG_QUERY = "SELECT set_config(%s, %s::text, TRUE);"
|
||||
SET_TRANSACTION_READ_ONLY_QUERY = "SET TRANSACTION READ ONLY;"
|
||||
|
||||
REPLICA_CONNECTION_SQLSTATE_PREFIXES = ("08",)
|
||||
REPLICA_CONNECTION_SQLSTATES = {"57P01", "57P02", "57P03"}
|
||||
REPLICA_NON_FAILOVER_SQLSTATES = {"57014", "40001", "40P01"}
|
||||
REPLICA_CONNECTION_ERROR_MESSAGES = (
|
||||
"ssl syscall",
|
||||
"eof detected",
|
||||
"server closed the connection",
|
||||
"connection already closed",
|
||||
"connection not open",
|
||||
"could not connect to server",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"connection timed out",
|
||||
"lost synchronization",
|
||||
"terminating connection",
|
||||
"database system is starting up",
|
||||
"database system is shutting down",
|
||||
"database system is in recovery mode",
|
||||
)
|
||||
REPLICA_NON_FAILOVER_ERROR_MESSAGES = (
|
||||
"canceling statement due to user request",
|
||||
"deadlock detected",
|
||||
"could not serialize access",
|
||||
)
|
||||
|
||||
|
||||
def _iter_exception_chain(error: BaseException):
|
||||
seen = set()
|
||||
pending = [error]
|
||||
while pending:
|
||||
current = pending.pop(0)
|
||||
if current is None or id(current) in seen:
|
||||
continue
|
||||
seen.add(id(current))
|
||||
yield current
|
||||
|
||||
cause = getattr(current, "__cause__", None)
|
||||
context = getattr(current, "__context__", None)
|
||||
if cause is not None:
|
||||
pending.append(cause)
|
||||
if context is not None:
|
||||
pending.append(context)
|
||||
for arg in getattr(current, "args", ()):
|
||||
if isinstance(arg, BaseException):
|
||||
pending.append(arg)
|
||||
|
||||
|
||||
def _get_exception_sqlstate(error: BaseException) -> str | None:
|
||||
for attr in ("pgcode", "sqlstate"):
|
||||
sqlstate = getattr(error, attr, None)
|
||||
if sqlstate:
|
||||
return sqlstate
|
||||
|
||||
diag = getattr(error, "diag", None)
|
||||
if diag is not None:
|
||||
sqlstate = getattr(diag, "sqlstate", None)
|
||||
if sqlstate:
|
||||
return sqlstate
|
||||
return None
|
||||
|
||||
|
||||
def _is_replica_connection_failure(error: BaseException) -> bool:
|
||||
"""
|
||||
Return True only for replica failures where retrying on primary is safe.
|
||||
|
||||
Query cancellations, serialization failures, and deadlocks should surface to
|
||||
callers because replaying them can hide real query or concurrency problems.
|
||||
"""
|
||||
messages = []
|
||||
sqlstates = set()
|
||||
|
||||
for chained_error in _iter_exception_chain(error):
|
||||
sqlstate = _get_exception_sqlstate(chained_error)
|
||||
if sqlstate:
|
||||
sqlstates.add(sqlstate)
|
||||
messages.append(str(chained_error).lower())
|
||||
|
||||
if sqlstates & REPLICA_NON_FAILOVER_SQLSTATES:
|
||||
return False
|
||||
if any(
|
||||
sqlstate.startswith(REPLICA_CONNECTION_SQLSTATE_PREFIXES)
|
||||
or sqlstate in REPLICA_CONNECTION_SQLSTATES
|
||||
for sqlstate in sqlstates
|
||||
):
|
||||
return True
|
||||
|
||||
message = " ".join(messages)
|
||||
if any(marker in message for marker in REPLICA_NON_FAILOVER_ERROR_MESSAGES):
|
||||
return False
|
||||
|
||||
return any(marker in message for marker in REPLICA_CONNECTION_ERROR_MESSAGES)
|
||||
|
||||
|
||||
def _strip_leading_sql_comments(sql: str) -> str:
|
||||
if not isinstance(sql, str):
|
||||
return ""
|
||||
|
||||
sql_text = sql.lstrip()
|
||||
while True:
|
||||
if sql_text.startswith("--"):
|
||||
newline_index = sql_text.find("\n")
|
||||
if newline_index == -1:
|
||||
return ""
|
||||
sql_text = sql_text[newline_index + 1 :].lstrip()
|
||||
continue
|
||||
|
||||
if sql_text.startswith("/*"):
|
||||
comment_end_index = sql_text.find("*/", 2)
|
||||
if comment_end_index == -1:
|
||||
return ""
|
||||
sql_text = sql_text[comment_end_index + 2 :].lstrip()
|
||||
continue
|
||||
|
||||
return sql_text
|
||||
|
||||
|
||||
def _is_safe_primary_replay(sql: str, many: bool) -> bool:
|
||||
if many:
|
||||
return False
|
||||
|
||||
sql_text = _strip_leading_sql_comments(sql)
|
||||
if not re.match(r"(?is)^SELECT\b", sql_text):
|
||||
return False
|
||||
|
||||
return not any(
|
||||
re.search(pattern, sql_text, re.IGNORECASE | re.DOTALL)
|
||||
for pattern in (
|
||||
r"\bINTO\b",
|
||||
r"\bFOR\s+(?:NO\s+KEY\s+)?UPDATE\b",
|
||||
r"\bFOR\s+(?:KEY\s+)?SHARE\b",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -77,14 +211,36 @@ def rls_transaction(
|
||||
retry_on_replica: bool = True,
|
||||
):
|
||||
"""
|
||||
Creates a new database transaction setting the given configuration value for Postgres RLS. It validates the
|
||||
if the value is a valid UUID.
|
||||
Context manager that opens an RLS-scoped database transaction.
|
||||
|
||||
Sets a Postgres configuration variable (``set_config``) so that Row-Level
|
||||
Security policies can filter by tenant. When *using* points to a read
|
||||
replica and *retry_on_replica* is True, replica failures are handled in two
|
||||
places:
|
||||
|
||||
1. **Pre-yield** (connection-setup failures): the function retries
|
||||
up to ``REPLICA_MAX_ATTEMPTS`` times on the replica, then falls
|
||||
back to the primary DB.
|
||||
2. **Post-yield** (mid-query failures): an ``execute_wrapper``
|
||||
intercepts connection-level ``OperationalError`` during
|
||||
``cursor.execute()`` calls and falls back directly to the primary DB
|
||||
for single ``SELECT`` statements. The primary fallback transaction is
|
||||
read-only, and unsafe statements keep raising the original error.
|
||||
The wrapper swaps the inner cursor so ``fetchall()`` / ``fetchone()``
|
||||
read from the new connection transparently.
|
||||
|
||||
Limitation: server-side cursors (``.iterator()``) fetch rows via
|
||||
``fetchmany()``, which the wrapper does not intercept. Call sites
|
||||
that iterate large result sets with ``.iterator()`` on the replica
|
||||
should add their own retry logic.
|
||||
|
||||
Args:
|
||||
value (str): Database configuration parameter value.
|
||||
parameter (str): Database configuration parameter name, by default is 'api.tenant_id'.
|
||||
using (str | None): Optional database alias to run the transaction against. Defaults to the
|
||||
active read alias (if any) or Django's default connection.
|
||||
value: Database configuration parameter value (must be a valid UUID).
|
||||
parameter: Database configuration parameter name.
|
||||
using: Optional database alias. Defaults to the active read
|
||||
alias or Django's default connection.
|
||||
retry_on_replica: Whether replica setup failures can retry and
|
||||
connection-level mid-query failures can fall back to primary.
|
||||
"""
|
||||
requested_alias = using or get_read_db_alias()
|
||||
db_alias = requested_alias or DEFAULT_DB_ALIAS
|
||||
@@ -92,54 +248,121 @@ def rls_transaction(
|
||||
db_alias = DEFAULT_DB_ALIAS
|
||||
|
||||
alias = db_alias
|
||||
is_replica = READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS
|
||||
max_attempts = REPLICA_MAX_ATTEMPTS if is_replica and retry_on_replica else 1
|
||||
is_replica = bool(READ_REPLICA_ALIAS and alias == READ_REPLICA_ALIAS)
|
||||
can_failover = is_replica and retry_on_replica
|
||||
replica_alias = alias # captured before the loop mutates alias
|
||||
max_attempts = (REPLICA_MAX_ATTEMPTS + 1) if can_failover else 1
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
router_token = None
|
||||
yielded_cursor = False
|
||||
# State shared between the generator and the _query_failover closure.
|
||||
# The fallback transaction.atomic() is registered into fallback_stack
|
||||
# via enter_context so its __exit__ runs when the outer with-ExitStack
|
||||
# block exits, with the right exc_info. No manual __enter__/__exit__.
|
||||
_fallback = {"succeeded": False, "token": None, "caller_exited_cleanly": False}
|
||||
|
||||
# On final attempt, fallback to primary
|
||||
if attempt == max_attempts and is_replica:
|
||||
logger.warning(
|
||||
f"RLS transaction failed after {attempt - 1} attempts on replica, "
|
||||
f"falling back to primary DB"
|
||||
)
|
||||
alias = DEFAULT_DB_ALIAS
|
||||
with ExitStack() as fallback_stack:
|
||||
|
||||
conn = connections[alias]
|
||||
try:
|
||||
if alias != DEFAULT_DB_ALIAS:
|
||||
router_token = set_read_db_alias(alias)
|
||||
def _query_failover(execute, sql, params, many, context):
|
||||
"""execute_wrapper: replay failed replica queries on the primary DB."""
|
||||
try:
|
||||
return execute(sql, params, many, context)
|
||||
except OperationalError as err:
|
||||
if not _is_replica_connection_failure(err):
|
||||
raise
|
||||
if not _is_safe_primary_replay(sql, many):
|
||||
raise
|
||||
|
||||
with transaction.atomic(using=alias):
|
||||
with conn.cursor() as cursor:
|
||||
try:
|
||||
# just in case the value is a UUID object
|
||||
uuid.UUID(str(value))
|
||||
except ValueError:
|
||||
raise ValidationError("Must be a valid UUID")
|
||||
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
|
||||
yielded_cursor = True
|
||||
yield cursor
|
||||
return
|
||||
except OperationalError as e:
|
||||
if yielded_cursor:
|
||||
raise
|
||||
# If on primary or max attempts reached, raise
|
||||
if not is_replica or attempt == max_attempts:
|
||||
raise
|
||||
try:
|
||||
connections[replica_alias].close()
|
||||
except Exception:
|
||||
pass # Best-effort; connection may already be dead
|
||||
|
||||
# Retry with exponential backoff
|
||||
delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1))
|
||||
logger.info(
|
||||
f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), "
|
||||
f"retrying in {delay}s. Error: {e}"
|
||||
)
|
||||
time.sleep(delay)
|
||||
finally:
|
||||
if router_token is not None:
|
||||
reset_read_db_alias(router_token)
|
||||
logger.warning(
|
||||
"Mid-query replica connection failure, falling back to primary DB"
|
||||
)
|
||||
primary = connections[DEFAULT_DB_ALIAS]
|
||||
primary.ensure_connection()
|
||||
fallback_stack.enter_context(transaction.atomic(using=DEFAULT_DB_ALIAS))
|
||||
|
||||
fallback_cursor = primary.cursor()
|
||||
fallback_stack.callback(fallback_cursor.close)
|
||||
fallback_cursor.execute(SET_TRANSACTION_READ_ONLY_QUERY)
|
||||
fallback_cursor.execute(SET_CONFIG_QUERY, [parameter, value])
|
||||
_fallback["token"] = set_read_db_alias(DEFAULT_DB_ALIAS)
|
||||
|
||||
fallback_cursor.execute(sql, params)
|
||||
|
||||
context["cursor"].db = primary
|
||||
context["cursor"].cursor = fallback_cursor.cursor
|
||||
_fallback["succeeded"] = True
|
||||
return None
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
router_token = None
|
||||
yielded_cursor = False
|
||||
|
||||
# On final attempt, fall back to primary
|
||||
if attempt == max_attempts and can_failover:
|
||||
if attempt > 1:
|
||||
logger.warning(
|
||||
f"RLS transaction failed after {attempt - 1} attempts on replica, "
|
||||
f"falling back to primary DB"
|
||||
)
|
||||
alias = DEFAULT_DB_ALIAS
|
||||
|
||||
conn = connections[alias]
|
||||
try:
|
||||
if alias != DEFAULT_DB_ALIAS:
|
||||
router_token = set_read_db_alias(alias)
|
||||
|
||||
with transaction.atomic(using=alias):
|
||||
with conn.cursor() as cursor:
|
||||
try:
|
||||
uuid.UUID(str(value))
|
||||
except ValueError:
|
||||
raise ValidationError("Must be a valid UUID")
|
||||
cursor.execute(SET_CONFIG_QUERY, [parameter, value])
|
||||
|
||||
wrapper_cm = (
|
||||
conn.execute_wrapper(_query_failover)
|
||||
if can_failover and alias == replica_alias
|
||||
else nullcontext()
|
||||
)
|
||||
with wrapper_cm:
|
||||
yielded_cursor = True
|
||||
yield cursor
|
||||
_fallback["caller_exited_cleanly"] = True
|
||||
return
|
||||
except OperationalError as e:
|
||||
if yielded_cursor:
|
||||
if _fallback["succeeded"] and _fallback["caller_exited_cleanly"]:
|
||||
# Caller's queries succeeded on primary via failover.
|
||||
# This error is transaction.atomic() cleanup on the
|
||||
# dead replica connection, suppress it.
|
||||
return
|
||||
raise
|
||||
|
||||
if not can_failover or attempt == max_attempts:
|
||||
raise
|
||||
|
||||
try:
|
||||
connections[alias].close()
|
||||
except Exception:
|
||||
pass # Best-effort; connection may already be dead
|
||||
|
||||
# Retry with exponential backoff
|
||||
delay = REPLICA_RETRY_BASE_DELAY * (2 ** (attempt - 1))
|
||||
logger.info(
|
||||
f"RLS transaction failed on replica (attempt {attempt}/{max_attempts}), "
|
||||
f"retrying in {delay}s. Error: {e}"
|
||||
)
|
||||
time.sleep(delay)
|
||||
finally:
|
||||
if _fallback["token"] is not None:
|
||||
reset_read_db_alias(_fallback["token"])
|
||||
_fallback["token"] = None
|
||||
|
||||
if router_token is not None:
|
||||
reset_read_db_alias(router_token)
|
||||
|
||||
|
||||
class CustomUserManager(BaseUserManager):
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import uuid
|
||||
from functools import wraps
|
||||
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY, rls_transaction
|
||||
from api.exceptions import ProviderDeletedException
|
||||
from api.models import Provider, Scan
|
||||
from api.models import Membership, Provider, Scan, Tenant
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.db import DatabaseError, connection, transaction
|
||||
from django.db import DEFAULT_DB_ALIAS, DatabaseError, connection, transaction
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
|
||||
|
||||
@@ -75,9 +76,11 @@ def handle_provider_deletion(func):
|
||||
"""
|
||||
Decorator that raises `ProviderDeletedException` if provider was deleted during execution.
|
||||
|
||||
Catches `ObjectDoesNotExist` and `DatabaseError` (including `IntegrityError`), checks if
|
||||
provider still exists, and raises `ProviderDeletedException` if not. Otherwise,
|
||||
re-raises original exception.
|
||||
Catches `ObjectDoesNotExist`, `DatabaseError` (including `IntegrityError`), and
|
||||
`GraphDatabaseQueryException`, checks if provider still exists, and raises
|
||||
`ProviderDeletedException` if not. Graph database errors also check whether the
|
||||
tenant still exists and has memberships. Otherwise, re-raises the original
|
||||
exception.
|
||||
|
||||
Requires `tenant_id` and `provider_id` in kwargs.
|
||||
|
||||
@@ -92,11 +95,16 @@ def handle_provider_deletion(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except (ObjectDoesNotExist, DatabaseError):
|
||||
except (ObjectDoesNotExist, DatabaseError, GraphDatabaseQueryException) as exc:
|
||||
tenant_id = kwargs.get("tenant_id")
|
||||
provider_id = kwargs.get("provider_id")
|
||||
database_alias = (
|
||||
DEFAULT_DB_ALIAS
|
||||
if isinstance(exc, GraphDatabaseQueryException)
|
||||
else READ_REPLICA_ALIAS
|
||||
)
|
||||
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
with rls_transaction(tenant_id, using=database_alias):
|
||||
if provider_id is None:
|
||||
scan_id = kwargs.get("scan_id")
|
||||
if scan_id is None:
|
||||
@@ -113,6 +121,13 @@ def handle_provider_deletion(func):
|
||||
raise ProviderDeletedException(
|
||||
f"Provider '{provider_id}' was deleted during the scan"
|
||||
) from None
|
||||
if isinstance(exc, GraphDatabaseQueryException) and (
|
||||
not Tenant.objects.filter(pk=tenant_id).exists()
|
||||
or not Membership.objects.filter(tenant_id=tenant_id).exists()
|
||||
):
|
||||
raise ProviderDeletedException(
|
||||
f"Tenant '{tenant_id}' was deleted during the scan"
|
||||
) from None
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
|
||||
+206
-59
@@ -67,6 +67,7 @@ from django_filters.rest_framework import (
|
||||
)
|
||||
from rest_framework_json_api.django_filters.backends import DjangoFilterBackend
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
from uuid6 import UUID
|
||||
|
||||
|
||||
class CustomDjangoFilterBackend(DjangoFilterBackend):
|
||||
@@ -672,35 +673,32 @@ class LatestResourceFilter(ProviderRelationshipFilterSet):
|
||||
return queryset.filter(tags__text_search=value)
|
||||
|
||||
|
||||
class FindingFilter(CommonFindingFilters):
|
||||
FINDING_BASE_FILTER_FIELDS = {
|
||||
"id": ["exact", "in"],
|
||||
"uid": ["exact", "in"],
|
||||
"scan": ["exact", "in"],
|
||||
"delta": ["exact", "in"],
|
||||
"status": ["exact", "in"],
|
||||
"severity": ["exact", "in"],
|
||||
"impact": ["exact", "in"],
|
||||
"check_id": ["exact", "in", "icontains"],
|
||||
}
|
||||
|
||||
|
||||
class BaseFindingFilter(CommonFindingFilters):
|
||||
DATE_FILTER_FIELDS = ()
|
||||
DATE_FILTER_NAMES = ()
|
||||
DATE_RANGE_HELP_TEXT = (
|
||||
f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days."
|
||||
)
|
||||
DATE_FILTER_REQUIRED_DETAIL = "At least one date filter is required."
|
||||
|
||||
scan = UUIDFilter(method="filter_scan_id")
|
||||
scan__in = UUIDInFilter(method="filter_scan_id_in")
|
||||
|
||||
inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__gte = DateFilter(
|
||||
method="filter_inserted_at_gte",
|
||||
help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
)
|
||||
inserted_at__lte = DateFilter(
|
||||
method="filter_inserted_at_lte",
|
||||
help_text=f"Maximum date range is {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = Finding
|
||||
fields = {
|
||||
"id": ["exact", "in"],
|
||||
"uid": ["exact", "in"],
|
||||
"scan": ["exact", "in"],
|
||||
"delta": ["exact", "in"],
|
||||
"status": ["exact", "in"],
|
||||
"severity": ["exact", "in"],
|
||||
"impact": ["exact", "in"],
|
||||
"check_id": ["exact", "in", "icontains"],
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
"updated_at": ["gte", "lte"],
|
||||
}
|
||||
fields = FINDING_BASE_FILTER_FIELDS
|
||||
filter_overrides = {
|
||||
FindingDeltaEnumField: {
|
||||
"filter_class": CharFilter,
|
||||
@@ -723,17 +721,13 @@ class FindingFilter(CommonFindingFilters):
|
||||
return queryset.filter(resource_services__contains=[value])
|
||||
|
||||
def filter_queryset(self, queryset):
|
||||
if not (self.data.get("scan") or self.data.get("scan__in")) and not (
|
||||
self.data.get("inserted_at")
|
||||
or self.data.get("inserted_at__date")
|
||||
or self.data.get("inserted_at__gte")
|
||||
or self.data.get("inserted_at__lte")
|
||||
if not (self.data.get("scan") or self.data.get("scan__in")) and not any(
|
||||
self.data.get(filter_name) for filter_name in self.DATE_FILTER_NAMES
|
||||
):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], "
|
||||
"or filter[inserted_at.lte].",
|
||||
"detail": self.DATE_FILTER_REQUIRED_DETAIL,
|
||||
"status": 400,
|
||||
"source": {"pointer": "/data/attributes/inserted_at"},
|
||||
"code": "required",
|
||||
@@ -742,31 +736,42 @@ class FindingFilter(CommonFindingFilters):
|
||||
)
|
||||
|
||||
cleaned = self.form.cleaned_data
|
||||
exact_date = cleaned.get("inserted_at") or cleaned.get("inserted_at__date")
|
||||
gte_date = cleaned.get("inserted_at__gte") or exact_date
|
||||
lte_date = cleaned.get("inserted_at__lte") or exact_date
|
||||
|
||||
if gte_date is None:
|
||||
gte_date = datetime.now(UTC).date()
|
||||
if lte_date is None:
|
||||
lte_date = datetime.now(UTC).date()
|
||||
|
||||
if abs(lte_date - gte_date) > timedelta(
|
||||
days=settings.FINDINGS_MAX_DAYS_IN_RANGE
|
||||
):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
"status": 400,
|
||||
"source": {"pointer": "/data/attributes/inserted_at"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
for field_name in self.DATE_FILTER_FIELDS:
|
||||
self.validate_datetime_filter_range(cleaned, field_name)
|
||||
|
||||
return super().filter_queryset(queryset)
|
||||
|
||||
def validate_datetime_filter_range(self, cleaned, field_name):
|
||||
exact_value = cleaned.get(field_name) or cleaned.get(f"{field_name}__date")
|
||||
gte_value = cleaned.get(f"{field_name}__gte") or exact_value
|
||||
lte_value = cleaned.get(f"{field_name}__lte") or exact_value
|
||||
|
||||
if not (exact_value or gte_value or lte_value):
|
||||
return
|
||||
|
||||
default_value = datetime.now(UTC).date()
|
||||
gte_value = gte_value or default_value
|
||||
lte_value = lte_value or default_value
|
||||
|
||||
gte_datetime = self.filter_value_to_datetime(gte_value, field_name)
|
||||
lte_datetime = self.filter_value_to_datetime(lte_value, field_name)
|
||||
|
||||
if abs(lte_datetime - gte_datetime) <= timedelta(
|
||||
days=settings.FINDINGS_MAX_DAYS_IN_RANGE
|
||||
):
|
||||
return
|
||||
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.",
|
||||
"status": 400,
|
||||
"source": {"pointer": f"/data/attributes/{field_name}"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Convert filter values to UUIDv7 values for use with partitioning
|
||||
def filter_scan_id(self, queryset, name, value):
|
||||
try:
|
||||
@@ -824,27 +829,169 @@ class FindingFilter(CommonFindingFilters):
|
||||
datetime_value = self.maybe_date_to_datetime(value)
|
||||
start = uuid7_start(datetime_to_uuid7(datetime_value))
|
||||
end = uuid7_start(datetime_to_uuid7(datetime_value + timedelta(days=1)))
|
||||
|
||||
return queryset.filter(id__gte=start, id__lt=end)
|
||||
|
||||
def filter_inserted_at_gte(self, queryset, name, value):
|
||||
datetime_value = self.maybe_date_to_datetime(value)
|
||||
start = uuid7_start(datetime_to_uuid7(datetime_value))
|
||||
|
||||
return queryset.filter(id__gte=start)
|
||||
|
||||
def filter_inserted_at_lte(self, queryset, name, value):
|
||||
datetime_value = self.maybe_date_to_datetime(value)
|
||||
end = uuid7_start(datetime_to_uuid7(datetime_value + timedelta(days=1)))
|
||||
|
||||
return queryset.filter(id__lt=end)
|
||||
|
||||
@staticmethod
|
||||
def maybe_date_to_datetime(value):
|
||||
dt = value
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if isinstance(value, date):
|
||||
dt = datetime.combine(value, datetime.min.time(), tzinfo=UTC)
|
||||
return dt
|
||||
return datetime.combine(value, datetime.min.time(), tzinfo=UTC)
|
||||
if isinstance(value, str):
|
||||
return parse(value)
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def filter_value_to_datetime(cls, value, field_name):
|
||||
try:
|
||||
datetime_value = cls.maybe_date_to_datetime(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
raise ValidationError(
|
||||
[
|
||||
{
|
||||
"detail": "Enter a valid date or datetime.",
|
||||
"status": 400,
|
||||
"source": {"pointer": f"/data/attributes/{field_name}"},
|
||||
"code": "invalid",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
if datetime_value.tzinfo is None:
|
||||
return datetime_value.replace(tzinfo=UTC)
|
||||
return datetime_value.astimezone(UTC)
|
||||
|
||||
|
||||
class FindingFilter(BaseFindingFilter):
|
||||
DATE_FILTER_FIELDS = ("inserted_at", "updated_at")
|
||||
DATE_FILTER_NAMES = (
|
||||
"inserted_at",
|
||||
"inserted_at__date",
|
||||
"inserted_at__gte",
|
||||
"inserted_at__lte",
|
||||
"updated_at",
|
||||
"updated_at__date",
|
||||
"updated_at__gte",
|
||||
"updated_at__lte",
|
||||
)
|
||||
DATE_FILTER_REQUIRED_DETAIL = (
|
||||
"At least one date filter is required: filter[inserted_at], filter[updated_at], "
|
||||
"filter[inserted_at.gte], filter[updated_at.gte], filter[inserted_at.lte], "
|
||||
"or filter[updated_at.lte]."
|
||||
)
|
||||
|
||||
inserted_at = CharFilter(method="filter_inserted_at")
|
||||
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__gte = CharFilter(
|
||||
method="filter_inserted_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
inserted_at__lte = CharFilter(
|
||||
method="filter_inserted_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
updated_at = CharFilter(method="filter_updated_at")
|
||||
updated_at__date = DateFilter(method="filter_updated_at", lookup_expr="date")
|
||||
updated_at__gte = CharFilter(
|
||||
method="filter_updated_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
updated_at__lte = CharFilter(
|
||||
method="filter_updated_at",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
|
||||
class Meta(BaseFindingFilter.Meta):
|
||||
fields = FINDING_BASE_FILTER_FIELDS | {
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
"updated_at": ["date", "gte", "lte"],
|
||||
}
|
||||
|
||||
def filter_inserted_at(self, queryset, name, value):
|
||||
start, end = self.filter_value_to_datetime_bounds(value, "inserted_at")
|
||||
|
||||
if name.endswith("__gte"):
|
||||
return queryset.filter(id__gte=self.datetime_to_uuid7_boundary(start))
|
||||
if name.endswith("__lte"):
|
||||
return queryset.filter(id__lt=self.datetime_to_uuid7_boundary(end))
|
||||
|
||||
return queryset.filter(
|
||||
id__gte=self.datetime_to_uuid7_boundary(start),
|
||||
id__lt=self.datetime_to_uuid7_boundary(end),
|
||||
)
|
||||
|
||||
def filter_updated_at(self, queryset, name, value):
|
||||
start, end = self.filter_value_to_datetime_bounds(value, "updated_at")
|
||||
|
||||
if name.endswith("__gte"):
|
||||
return queryset.filter(updated_at__gte=start)
|
||||
if name.endswith("__lte"):
|
||||
return queryset.filter(updated_at__lt=end)
|
||||
|
||||
return queryset.filter(updated_at__gte=start, updated_at__lt=end)
|
||||
|
||||
@classmethod
|
||||
def filter_value_to_datetime_bounds(cls, value, field_name):
|
||||
start = cls.filter_value_to_datetime(value, field_name)
|
||||
if cls.is_date_filter_value(value):
|
||||
return start, start + timedelta(days=1)
|
||||
return start, start + timedelta(milliseconds=1)
|
||||
|
||||
@staticmethod
|
||||
def datetime_to_uuid7_boundary(datetime_value):
|
||||
timestamp_ms = int(datetime_value.timestamp() * 1000) & 0xFFFFFFFFFFFF
|
||||
uuid_int = timestamp_ms << 80
|
||||
uuid_int |= 0x7 << 76
|
||||
uuid_int |= 0x2 << 62
|
||||
return UUID(int=uuid_int)
|
||||
|
||||
@staticmethod
|
||||
def is_date_filter_value(value):
|
||||
if isinstance(value, datetime):
|
||||
return False
|
||||
if isinstance(value, date):
|
||||
return True
|
||||
return isinstance(value, str) and len(value.strip()) == 10
|
||||
|
||||
|
||||
class FindingMetadataFilter(BaseFindingFilter):
|
||||
DATE_FILTER_FIELDS = ("inserted_at",)
|
||||
DATE_FILTER_NAMES = (
|
||||
"inserted_at",
|
||||
"inserted_at__date",
|
||||
"inserted_at__gte",
|
||||
"inserted_at__lte",
|
||||
)
|
||||
DATE_FILTER_REQUIRED_DETAIL = (
|
||||
"At least one date filter is required: filter[inserted_at], filter[inserted_at.gte], "
|
||||
"or filter[inserted_at.lte]."
|
||||
)
|
||||
|
||||
inserted_at = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__date = DateFilter(method="filter_inserted_at", lookup_expr="date")
|
||||
inserted_at__gte = DateFilter(
|
||||
method="filter_inserted_at_gte",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
inserted_at__lte = DateFilter(
|
||||
method="filter_inserted_at_lte",
|
||||
help_text=BaseFindingFilter.DATE_RANGE_HELP_TEXT,
|
||||
)
|
||||
|
||||
class Meta(BaseFindingFilter.Meta):
|
||||
fields = FINDING_BASE_FILTER_FIELDS | {
|
||||
"inserted_at": ["date", "gte", "lte"],
|
||||
}
|
||||
|
||||
|
||||
class LatestFindingFilter(CommonFindingFilters):
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
Format (draft-inadarei-api-health-check-06).
|
||||
|
||||
Liveness reports only process status. Readiness verifies that PostgreSQL,
|
||||
Valkey and Neo4j are reachable and returns per-dependency detail when any
|
||||
of them is unreachable.
|
||||
Valkey and the attack-paths graph store (Neo4j or Neptune, per
|
||||
``ATTACK_PATHS_SINK_DATABASE``) are reachable and returns per-dependency
|
||||
detail when any of them is unreachable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,6 +12,8 @@ from __future__ import annotations
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FuturesTimeoutError
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
@@ -37,9 +40,28 @@ STATUS_FAIL = "fail"
|
||||
STATUS_WARN = "warn"
|
||||
|
||||
# Short socket timeout so a stuck Valkey cannot stall the probe.
|
||||
# Neo4j inherits its driver-level ``connection_acquisition_timeout``.
|
||||
VALKEY_PROBE_TIMEOUT_SECONDS = 2
|
||||
|
||||
# Probe-scoped budget for the graph database.
|
||||
# ``Driver.verify_connectivity()`` takes no timeout; its only bound is the
|
||||
# driver-level ``connection_acquisition_timeout`` (60s on Neptune). The
|
||||
# probe needs its own budget, independent of the workload driver, so a
|
||||
# graph-database outage cannot pin a worker thread (and the readiness lock)
|
||||
# for a minute.
|
||||
GRAPH_DB_PROBE_TIMEOUT_SECONDS = 5
|
||||
|
||||
# Bounded pool that enforces ``GRAPH_DB_PROBE_TIMEOUT_SECONDS``. If the
|
||||
# graph database is unreachable the probe call blocks until the driver's
|
||||
# own acquisition timeout fires; we abandon the future after the budget and
|
||||
# report ``fail``. Orphaned tasks are capped by ``max_workers`` plus the 3s
|
||||
# readiness cache plus the per-IP throttle, so they cannot pile up: worst
|
||||
# case during a graph-database outage is every readiness call failing fast
|
||||
# in ``GRAPH_DB_PROBE_TIMEOUT_SECONDS`` with at most 2 background threads
|
||||
# stuck for <= the driver acquisition timeout.
|
||||
_graph_db_probe_executor = ThreadPoolExecutor(
|
||||
max_workers=2, thread_name_prefix="health-graph-db-probe"
|
||||
)
|
||||
|
||||
# Brief cache window so high-frequency probes (ALB target groups, scrapers)
|
||||
# do not stampede the actual dependency checks.
|
||||
CACHE_CONTROL_HEADER = "max-age=3, must-revalidate"
|
||||
@@ -109,11 +131,24 @@ def _probe_valkey() -> None:
|
||||
client.close()
|
||||
|
||||
|
||||
def _probe_neo4j() -> None:
|
||||
# Lazy import: avoids pulling attack_paths into the boot import graph.
|
||||
from api.attack_paths.database import get_driver
|
||||
def _graph_db_component_id() -> str:
|
||||
"""Return the active graph database name for the ``componentId`` field."""
|
||||
return settings.ATTACK_PATHS_SINK_DATABASE.strip().lower()
|
||||
|
||||
get_driver().verify_connectivity()
|
||||
|
||||
def _probe_graph_db() -> None:
|
||||
# Lazy import: avoids pulling attack_paths into the boot import graph
|
||||
from api.attack_paths.database import verify_connectivity
|
||||
|
||||
future = _graph_db_probe_executor.submit(verify_connectivity)
|
||||
try:
|
||||
future.result(timeout=GRAPH_DB_PROBE_TIMEOUT_SECONDS)
|
||||
except FuturesTimeoutError as exc:
|
||||
# Do not wait for the abandoned task; it ends when the driver's own acquisition timeout fires
|
||||
future.cancel()
|
||||
raise TimeoutError(
|
||||
f"graph-db probe exceeded {GRAPH_DB_PROBE_TIMEOUT_SECONDS}s"
|
||||
) from exc
|
||||
|
||||
|
||||
def _build_check_entry(
|
||||
@@ -176,14 +211,18 @@ def _readiness_payload() -> tuple[dict[str, Any], int]:
|
||||
):
|
||||
return snapshot[1], snapshot[2]
|
||||
|
||||
graph_db_component_id = _graph_db_component_id()
|
||||
|
||||
postgres_result, postgres_ms = _measure("postgres", _probe_postgres)
|
||||
valkey_result, valkey_ms = _measure("valkey", _probe_valkey)
|
||||
neo4j_result, neo4j_ms = _measure("neo4j", _probe_neo4j)
|
||||
graph_db_result, graph_db_ms = _measure(graph_db_component_id, _probe_graph_db)
|
||||
|
||||
entries = [
|
||||
_build_check_entry("postgres", "datastore", postgres_result, postgres_ms),
|
||||
_build_check_entry("valkey", "datastore", valkey_result, valkey_ms),
|
||||
_build_check_entry("neo4j", "datastore", neo4j_result, neo4j_ms),
|
||||
_build_check_entry(
|
||||
graph_db_component_id, "datastore", graph_db_result, graph_db_ms
|
||||
),
|
||||
]
|
||||
overall = _aggregate_status(entries)
|
||||
|
||||
@@ -191,7 +230,7 @@ def _readiness_payload() -> tuple[dict[str, Any], int]:
|
||||
payload["checks"] = {
|
||||
"postgres:responseTime": [entries[0]],
|
||||
"valkey:responseTime": [entries[1]],
|
||||
"neo4j:responseTime": [entries[2]],
|
||||
"graphdb:responseTime": [entries[2]],
|
||||
}
|
||||
|
||||
http_status = (
|
||||
@@ -233,10 +272,10 @@ class LivenessView(APIView):
|
||||
class ReadinessView(APIView):
|
||||
"""Readiness probe.
|
||||
|
||||
Returns 200 when PostgreSQL, Valkey and Neo4j all respond, or 503 with
|
||||
per-dependency detail when any of them is unreachable. Per-IP throttle
|
||||
plus the short in-process result cache cap the real dependency hits
|
||||
regardless of inbound traffic shape.
|
||||
Returns 200 when PostgreSQL, Valkey and the attack-paths graph store
|
||||
all respond, or 503 with per-dependency detail when any of them is
|
||||
unreachable. Per-IP throttle plus the short in-process result cache cap
|
||||
the real dependency hits regardless of inbound traffic shape.
|
||||
"""
|
||||
|
||||
authentication_classes: list = []
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0095_reconcile_orphan_tasks_periodic_task"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="attackpathsscan",
|
||||
name="is_migrated",
|
||||
field=models.BooleanField(default=False),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="attackpathsscan",
|
||||
name="sink_backend",
|
||||
field=models.CharField(
|
||||
choices=[("neo4j", "Neo4j"), ("neptune", "Neptune")],
|
||||
default="neo4j",
|
||||
max_length=16,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,25 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0096_attack_paths_scan_is_migrated"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="attackpathsscan",
|
||||
name="is_migrated",
|
||||
field=models.BooleanField(db_default=False, default=False),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="attackpathsscan",
|
||||
name="sink_backend",
|
||||
field=models.CharField(
|
||||
choices=[("neo4j", "Neo4j"), ("neptune", "Neptune")],
|
||||
db_default="neo4j",
|
||||
default="neo4j",
|
||||
max_length=16,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -757,6 +757,10 @@ class Scan(RowLevelSecurityProtectedModel):
|
||||
|
||||
|
||||
class AttackPathsScan(RowLevelSecurityProtectedModel):
|
||||
class SinkBackendChoices(models.TextChoices):
|
||||
NEO4J = "neo4j", "Neo4j"
|
||||
NEPTUNE = "neptune", "Neptune"
|
||||
|
||||
objects = ActiveProviderManager()
|
||||
all_objects = models.Manager()
|
||||
|
||||
@@ -805,6 +809,19 @@ class AttackPathsScan(RowLevelSecurityProtectedModel):
|
||||
)
|
||||
ingestion_exceptions = models.JSONField(default=dict, null=True, blank=True)
|
||||
|
||||
# True when the scan was synced with the current schema (list-typed
|
||||
# properties materialised as child item nodes). False for pre-cutover scans
|
||||
# still using the previous graph shape. Query catalog selection uses this
|
||||
# flag; physical read routing uses sink_backend below.
|
||||
# TODO: drop after Neptune cutover
|
||||
is_migrated = models.BooleanField(default=False, db_default=False)
|
||||
sink_backend = models.CharField(
|
||||
choices=SinkBackendChoices.choices,
|
||||
db_default=SinkBackendChoices.NEO4J,
|
||||
default=SinkBackendChoices.NEO4J,
|
||||
max_length=16,
|
||||
)
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "attack_paths_scans"
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ class HasPermissions(BasePermission):
|
||||
if not tenant_id:
|
||||
return False
|
||||
|
||||
user_roles = (
|
||||
user_roles = list(
|
||||
User.objects.using(MainRouter.admin_db)
|
||||
.get(id=request.user.id)
|
||||
.roles.using(MainRouter.admin_db)
|
||||
@@ -43,11 +43,10 @@ class HasPermissions(BasePermission):
|
||||
if not user_roles:
|
||||
return False
|
||||
|
||||
for perm in required_permissions:
|
||||
if not getattr(user_roles[0], perm.value, False):
|
||||
return False
|
||||
|
||||
return True
|
||||
return all(
|
||||
any(getattr(role, permission.value, False) for role in user_roles)
|
||||
for permission in required_permissions
|
||||
)
|
||||
|
||||
|
||||
def get_role(user: User, tenant_id: str) -> Role:
|
||||
|
||||
+1602
-314
File diff suppressed because it is too large
Load Diff
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
|
||||
class SSEChannelManager(DefaultChannelManager):
|
||||
"""Connect `django-eventstream` to the platform's SSE viewsets."""
|
||||
|
||||
def get_channels_for_request(self, request: Request, view_kwargs: dict) -> set[str]: # noqa: vulture
|
||||
def get_channels_for_request(self, request: Request, view_kwargs: dict) -> set[str]:
|
||||
"""Return the request's channels scoped to the active JWT tenant.
|
||||
|
||||
Args:
|
||||
@@ -30,6 +30,7 @@ class SSEChannelManager(DefaultChannelManager):
|
||||
The subset of `request.sse_channels` whose embedded tenant
|
||||
matches the active request tenant.
|
||||
"""
|
||||
_ = view_kwargs
|
||||
try:
|
||||
request_tenant_id = UUID(str(getattr(request, "tenant_id", None)))
|
||||
except (TypeError, ValueError):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
@@ -8,6 +9,23 @@ from conftest import TEST_PASSWORD, get_api_tokens, get_authorization_header
|
||||
from django.urls import reverse
|
||||
from drf_simple_apikey.crypto import get_crypto
|
||||
from rest_framework.test import APIClient
|
||||
from rest_framework_simplejwt.token_blacklist.models import (
|
||||
BlacklistedToken,
|
||||
OutstandingToken,
|
||||
)
|
||||
|
||||
PASSWORD_CHANGE_PASSWORD = "InitialSecret123@"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def password_change_user(tenants_fixture):
|
||||
user = User.objects.create_user(
|
||||
name="password_change_user",
|
||||
email=f"password-change-{uuid4()}@prowler.com",
|
||||
password=PASSWORD_CHANGE_PASSWORD,
|
||||
)
|
||||
Membership.objects.create(user=user, tenant=tenants_fixture[0])
|
||||
return user
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -103,6 +121,120 @@ def test_refresh_token(create_test_user, tenants_fixture):
|
||||
assert new_refresh_response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_password_change_invalidates_existing_tokens(password_change_user):
|
||||
client = APIClient()
|
||||
new_password = "ChangedSecret123@"
|
||||
|
||||
access_token, refresh_token = get_api_tokens(
|
||||
client, password_change_user.email, PASSWORD_CHANGE_PASSWORD
|
||||
)
|
||||
auth_headers = get_authorization_header(access_token)
|
||||
outstanding_token_ids = list(
|
||||
OutstandingToken.objects.filter(user=password_change_user).values_list(
|
||||
"id", flat=True
|
||||
)
|
||||
)
|
||||
assert outstanding_token_ids
|
||||
assert not BlacklistedToken.objects.filter(
|
||||
token_id__in=outstanding_token_ids
|
||||
).exists()
|
||||
|
||||
password_change_payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(password_change_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
}
|
||||
}
|
||||
password_change_response = client.patch(
|
||||
reverse("user-detail", kwargs={"pk": password_change_user.id}),
|
||||
data=json.dumps(password_change_payload),
|
||||
headers=auth_headers,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
assert password_change_response.status_code == 200, password_change_response.json()
|
||||
assert BlacklistedToken.objects.filter(
|
||||
token_id__in=outstanding_token_ids
|
||||
).count() == len(outstanding_token_ids)
|
||||
|
||||
old_access_response = client.get(reverse("user-me"), headers=auth_headers)
|
||||
assert old_access_response.status_code == 401
|
||||
|
||||
old_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "tokens-refresh",
|
||||
"attributes": {"refresh": refresh_token},
|
||||
}
|
||||
},
|
||||
format="vnd.api+json",
|
||||
)
|
||||
assert old_refresh_response.status_code == 400
|
||||
|
||||
new_access_token, _ = get_api_tokens(
|
||||
client, password_change_user.email, new_password
|
||||
)
|
||||
new_access_response = client.get(
|
||||
reverse("user-me"), headers=get_authorization_header(new_access_token)
|
||||
)
|
||||
assert new_access_response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_password_change_invalidates_rotated_refresh_token(
|
||||
password_change_user,
|
||||
):
|
||||
client = APIClient()
|
||||
new_password = "ChangedSecret123@"
|
||||
|
||||
access_token, refresh_token = get_api_tokens(
|
||||
client, password_change_user.email, PASSWORD_CHANGE_PASSWORD
|
||||
)
|
||||
rotated_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "tokens-refresh",
|
||||
"attributes": {"refresh": refresh_token},
|
||||
}
|
||||
},
|
||||
format="vnd.api+json",
|
||||
)
|
||||
assert rotated_refresh_response.status_code == 200
|
||||
rotated_refresh_token = rotated_refresh_response.json()["data"]["attributes"][
|
||||
"refresh"
|
||||
]
|
||||
|
||||
password_change_payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(password_change_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
}
|
||||
}
|
||||
password_change_response = client.patch(
|
||||
reverse("user-detail", kwargs={"pk": password_change_user.id}),
|
||||
data=json.dumps(password_change_payload),
|
||||
headers=get_authorization_header(access_token),
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
assert password_change_response.status_code == 200, password_change_response.json()
|
||||
|
||||
old_rotated_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "tokens-refresh",
|
||||
"attributes": {"refresh": rotated_refresh_token},
|
||||
}
|
||||
},
|
||||
format="vnd.api+json",
|
||||
)
|
||||
assert old_rotated_refresh_response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_user_me_when_inviting_users(create_test_user, tenants_fixture, roles_fixture):
|
||||
client = APIClient()
|
||||
@@ -187,8 +319,9 @@ def test_user_me_when_inviting_users(create_test_user, tenants_fixture, roles_fi
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestTokenSwitchTenant:
|
||||
def test_switch_tenant_with_valid_token(self, tenants_fixture, providers_fixture):
|
||||
def test_switch_tenant_with_valid_token(self, tenants_fixture, aws_provider):
|
||||
client = APIClient()
|
||||
assert aws_provider
|
||||
|
||||
test_user = "test_email@prowler.com"
|
||||
test_password = "Test_password1@"
|
||||
@@ -1396,13 +1529,14 @@ class TestAPIKeyMultiTenantWorkflows:
|
||||
assert me_response2.json()["data"]["id"] == str(user.id)
|
||||
|
||||
def test_api_key_cannot_access_different_tenant_resources(
|
||||
self, tenants_fixture, providers_fixture
|
||||
self, tenants_fixture, aws_provider
|
||||
):
|
||||
"""API key from one tenant cannot access resources from another tenant.
|
||||
|
||||
Verifies RLS enforcement after authentication ensures tenant isolation.
|
||||
"""
|
||||
client = APIClient()
|
||||
assert aws_provider
|
||||
|
||||
user1 = User.objects.create_user(
|
||||
name="tenant1_user",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user