Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eaee2622a4 | ||
|
|
9b184d1d45 | ||
|
|
c1d18040b7 | ||
|
|
e59b391dea | ||
|
|
8bf926d4e7 | ||
|
|
0c9654db4b | ||
|
|
31261d78f3 | ||
|
|
e8779953cd | ||
|
|
7345e051cf | ||
|
|
c2b0135e35 | ||
|
|
1c4d8e3e75 | ||
|
|
b3562a800f | ||
|
|
cea4244db8 | ||
|
|
52f2da90f6 | ||
|
|
d1d6825159 | ||
|
|
548dd0f35a | ||
|
|
c11fdb9d6d | ||
|
|
9f3a0534a8 | ||
|
|
44b7afdb25 | ||
|
|
fd19a9a048 | ||
|
|
1da1da54d1 | ||
|
|
c123dc3788 | ||
|
|
3517cb331a | ||
|
|
fa365eb106 | ||
|
|
4526b91d3b | ||
|
|
76a2d7bfe6 | ||
|
|
ac3f289de6 | ||
|
|
838f82b255 | ||
|
|
5e00c4bfcf | ||
|
|
f0ae56b8ea | ||
|
|
4484c2f192 | ||
|
|
e6bbcb8043 | ||
|
|
ad04e69c35 | ||
|
|
6cae37174c | ||
|
|
aa6de57430 | ||
|
|
a48aa37f87 | ||
|
|
3cc8f86780 | ||
|
|
4cb02a0ead | ||
|
|
221c558cee | ||
|
|
81c3152ebb | ||
|
|
855e9a043e | ||
|
|
7b5d724bb7 | ||
|
|
441f2a3c48 | ||
|
|
efb86bb7ab | ||
|
|
398a0a484f | ||
|
|
55924d8150 | ||
|
|
0cf6f2f83e | ||
|
|
cf18093261 | ||
|
|
1850e209e6 | ||
|
|
2e37188c9f |
@@ -145,19 +145,20 @@ DJANGO_BROKER_VISIBILITY_TIMEOUT=86400
|
||||
DJANGO_SENTRY_DSN=
|
||||
DJANGO_THROTTLE_TOKEN_OBTAIN=50/minute
|
||||
|
||||
# Sentry for the web app (server + browser). Empty/unset UI_SENTRY_DSN ⇒
|
||||
# Sentry disabled, zero egress. SENTRY_RELEASE (unprefixed) feeds the web app's
|
||||
# server/edge SDKs.
|
||||
# Sentry for the web app (server + browser). The UI_SENTRY_* values load only
|
||||
# when UI_SENTRY_ENABLE="true"; without it they are ignored (default off, zero
|
||||
# egress). The deprecated NEXT_PUBLIC_SENTRY_DSN still activates Sentry without
|
||||
# the flag. SENTRY_RELEASE (unprefixed) feeds the web app's server/edge SDKs.
|
||||
UI_SENTRY_DSN=
|
||||
UI_SENTRY_ENVIRONMENT=local
|
||||
SENTRY_RELEASE=local
|
||||
# Reserved runtime public config (registered now; no UI consumer yet)
|
||||
# POSTHOG_KEY=
|
||||
# POSTHOG_HOST=
|
||||
# UI_POSTHOG_KEY=
|
||||
# UI_POSTHOG_HOST=
|
||||
# REO_DEV_CLIENT_ID=
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.32.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.33.2
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
# v5
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/api"
|
||||
# schedule:
|
||||
# interval: "daily"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# - "component/api"
|
||||
|
||||
# Dependabot version updates disabled - migrated to Renovate - 2026/07/02
|
||||
# - package-ecosystem: "github-actions"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "github_actions"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# - package-ecosystem: "npm"
|
||||
# directory: "/ui"
|
||||
# schedule:
|
||||
# interval: "daily"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "npm"
|
||||
# - "component/ui"
|
||||
|
||||
# Dependabot version updates disabled - migrated to Renovate - 2026/07/02
|
||||
# - package-ecosystem: "docker"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "docker"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# - package-ecosystem: "pre-commit"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 25
|
||||
# target-branch: master
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pre-commit"
|
||||
# cooldown:
|
||||
# default-days: 7
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/04/15
|
||||
# v4.6
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v4.6
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# - "v4"
|
||||
|
||||
# - package-ecosystem: "github-actions"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v4.6
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "github_actions"
|
||||
# - "v4"
|
||||
|
||||
# - package-ecosystem: "docker"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "weekly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v4.6
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "docker"
|
||||
# - "v4"
|
||||
|
||||
# Dependabot Updates are temporary disabled - 2025/03/19
|
||||
# v3
|
||||
# - package-ecosystem: "pip"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v3
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "pip"
|
||||
# - "v3"
|
||||
|
||||
# - package-ecosystem: "github-actions"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "monthly"
|
||||
# open-pull-requests-limit: 10
|
||||
# target-branch: v3
|
||||
# labels:
|
||||
# - "dependencies"
|
||||
# - "github_actions"
|
||||
# - "v3"
|
||||
@@ -18,8 +18,8 @@ Please add a detailed description of how to review this PR.
|
||||
|
||||
<summary><b>Community Checklist</b></summary>
|
||||
|
||||
- [ ] This feature/issue is listed in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
|
||||
- [ ] Is it assigned to me, if not, request it via the issue/feature in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](goto.prowler.com/slack)
|
||||
- [ ] This feature/issue is listed in the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
|
||||
- [ ] Is it assigned to me, if not, request it via the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](https://goto.prowler.com/slack)
|
||||
|
||||
</details>
|
||||
|
||||
@@ -28,7 +28,7 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Review if code is being documented following this specification https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings
|
||||
- [ ] Review if backport is needed.
|
||||
- [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable.
|
||||
- [ ] Ensure a changelog fragment is added under [prowler/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/prowler/changelog.d), if applicable.
|
||||
|
||||
#### SDK/CLI
|
||||
- Are there new checks included in this PR? Yes / No
|
||||
@@ -40,7 +40,7 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Table (640px > X < 1024px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Desktop (X > 1024px)
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable.
|
||||
- [ ] Ensure a changelog fragment is added under [ui/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/ui/changelog.d), if applicable.
|
||||
|
||||
#### API
|
||||
- [ ] All issue/task requirements work as expected on the API
|
||||
@@ -50,7 +50,11 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Any other relevant evidence of the implementation (if applicable)
|
||||
- [ ] Verify if API specs need to be regenerated.
|
||||
- [ ] Check if version updates are required (e.g., specs, uv, etc.).
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable.
|
||||
- [ ] Ensure a changelog fragment is added under [api/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/api/changelog.d), if applicable.
|
||||
|
||||
#### MCP Server
|
||||
- [ ] All issue/task requirements work as expected on the MCP Server
|
||||
- [ ] Ensure a changelog fragment is added under [mcp_server/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/mcp_server/changelog.d), if applicable.
|
||||
|
||||
### License
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rename changelog fragments to their PR number before running towncrier.
|
||||
|
||||
For every <slug>.<type>.md in <component_dir>/changelog.d/, find the commit that
|
||||
added it, resolve its PR via the GitHub API (falling back to the squash-commit
|
||||
subject), and `git mv` it to <PR>.<type>.md so towncrier renders the PR link.
|
||||
Unresolvable fragments become +<slug>.<type>.md orphans (rendered without link).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
FRAGMENT_RE = re.compile(
|
||||
r"^(?P<slug>[A-Za-z0-9][A-Za-z0-9._-]*?)"
|
||||
r"\.(?P<type>added|changed|deprecated|removed|fixed|security)"
|
||||
r"(?:\.(?P<counter>[0-9]+))?\.md$"
|
||||
)
|
||||
SUBJECT_PR_RE = re.compile(r" \(#([0-9]+)\)$")
|
||||
IGNORED_FILES = {".gitkeep", "README.md"}
|
||||
API_TIMEOUT_SECONDS = 10
|
||||
|
||||
|
||||
def git(*args: str) -> str:
|
||||
result = subprocess.run(["git", *args], check=True, capture_output=True, text=True)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def find_adding_commit(path: str) -> str | None:
|
||||
"""Find the commit that added a file, following renames.
|
||||
|
||||
Falls back to a plain (no --follow) lookup: rename detection can lose the
|
||||
add event for degenerate content (e.g. files identical to many others).
|
||||
"""
|
||||
sha = git("log", "--follow", "--diff-filter=A", "--format=%H", "-1", "--", path)
|
||||
if not sha:
|
||||
sha = git("log", "--diff-filter=A", "--format=%H", "-1", "--", path)
|
||||
return sha or None
|
||||
|
||||
|
||||
def pr_from_api(repo: str, sha: str) -> int | None:
|
||||
"""Resolve the PR associated with a commit via the GitHub API.
|
||||
|
||||
Returns None on any network/API failure so the caller can fall back to
|
||||
parsing the squash-commit subject.
|
||||
"""
|
||||
url = f"https://api.github.com/repos/{repo}/commits/{sha}/pulls"
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"User-Agent": "prowler-changelog-attribution",
|
||||
}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
request = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=API_TIMEOUT_SECONDS) as response:
|
||||
pulls = json.load(response)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
return None
|
||||
if isinstance(pulls, list) and pulls:
|
||||
return pulls[0].get("number")
|
||||
return None
|
||||
|
||||
|
||||
def pr_from_subject(sha: str) -> int | None:
|
||||
subject = git("log", "-1", "--format=%s", sha)
|
||||
match = SUBJECT_PR_RE.search(subject)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def unique_destination(directory: str, base_name: str, fragment_type: str) -> str:
|
||||
"""Return a non-colliding fragment path, appending a numeric counter if needed."""
|
||||
candidate = os.path.join(directory, f"{base_name}.{fragment_type}.md")
|
||||
counter = 0
|
||||
while os.path.exists(candidate):
|
||||
counter += 1
|
||||
candidate = os.path.join(directory, f"{base_name}.{fragment_type}.{counter}.md")
|
||||
return candidate
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("component_dir", help="Component directory, e.g. prowler")
|
||||
parser.add_argument("--repo", default="prowler-cloud/prowler")
|
||||
parser.add_argument(
|
||||
"--no-api",
|
||||
action="store_true",
|
||||
help="Skip the GitHub API and resolve PRs from commit subjects only",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
fragments_dir = os.path.join(args.component_dir, "changelog.d")
|
||||
if not os.path.isdir(fragments_dir):
|
||||
print(f"::error::Fragments directory not found: {fragments_dir}")
|
||||
return 1
|
||||
|
||||
malformed = []
|
||||
to_process = []
|
||||
for name in sorted(os.listdir(fragments_dir)):
|
||||
if name in IGNORED_FILES or name.startswith("+"):
|
||||
continue
|
||||
match = FRAGMENT_RE.match(name)
|
||||
if not match:
|
||||
malformed.append(name)
|
||||
continue
|
||||
if match.group("slug").isdigit():
|
||||
continue
|
||||
to_process.append((name, match))
|
||||
|
||||
if malformed:
|
||||
for name in malformed:
|
||||
print(
|
||||
f"::error::Malformed fragment filename in {fragments_dir}: {name} "
|
||||
"(expected <slug>.<type>.md with type one of added|changed|"
|
||||
"deprecated|removed|fixed|security)"
|
||||
)
|
||||
return 1
|
||||
|
||||
for name, match in to_process:
|
||||
slug, fragment_type = match.group("slug"), match.group("type")
|
||||
|
||||
path = os.path.join(fragments_dir, name)
|
||||
sha = find_adding_commit(path)
|
||||
pr_number = None
|
||||
if sha:
|
||||
if not args.no_api:
|
||||
pr_number = pr_from_api(args.repo, sha)
|
||||
if pr_number is None:
|
||||
pr_number = pr_from_subject(sha)
|
||||
|
||||
if pr_number is not None:
|
||||
destination = unique_destination(
|
||||
fragments_dir, str(pr_number), fragment_type
|
||||
)
|
||||
else:
|
||||
destination = unique_destination(fragments_dir, f"+{slug}", fragment_type)
|
||||
print(
|
||||
f"::warning::Could not resolve a PR for {path}; renamed to "
|
||||
f"{os.path.basename(destination)} (entry will render without a PR link)"
|
||||
)
|
||||
git("mv", path, destination)
|
||||
print(f"{path} -> {destination}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,13 @@
|
||||
{% set category_order = definitions.keys() %}
|
||||
{% for section, _ in sections.items() %}
|
||||
{% for category in category_order if category in sections[section] %}
|
||||
### {{ definitions[category]['name'] }}
|
||||
|
||||
{% for text, values in sections[section][category].items() -%}
|
||||
- {{ text }}{% if values %} {{ values|join(', ') }}{% endif %}{{ "\n" }}
|
||||
{%- endfor %}
|
||||
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
---
|
||||
{{ "\n" }}
|
||||
@@ -62,6 +62,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -9,6 +9,7 @@ on:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-codeql.yml'
|
||||
- '.github/codeql/api-codeql-config.yml'
|
||||
- '!api/CHANGELOG.md'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
@@ -17,6 +18,7 @@ on:
|
||||
- 'api/**'
|
||||
- '.github/workflows/api-codeql.yml'
|
||||
- '.github/codeql/api-codeql-config.yml'
|
||||
- '!api/CHANGELOG.md'
|
||||
schedule:
|
||||
- cron: '00 12 * * *'
|
||||
|
||||
|
||||
@@ -108,6 +108,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
|
||||
@@ -77,6 +77,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -111,6 +111,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -0,0 +1,527 @@
|
||||
name: 'Tools: Compile Changelogs'
|
||||
|
||||
run-name: 'Compile changelogs for Prowler ${{ inputs.prowler_version }}'
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
prowler_version:
|
||||
description: 'Prowler version being released (e.g., 5.31.0)'
|
||||
required: true
|
||||
type: string
|
||||
target_branch:
|
||||
description: 'Branch to compile on (master for minor releases, v5.X for patches)'
|
||||
required: true
|
||||
type: string
|
||||
sdk_version:
|
||||
description: 'SDK version override (empty = mirrors prowler_version; "skip" = hold this component back)'
|
||||
required: false
|
||||
type: string
|
||||
api_version:
|
||||
description: 'API version override (empty = auto-derive 1.<prowler_minor + 1>.<prowler_patch>; "skip" = hold back)'
|
||||
required: false
|
||||
type: string
|
||||
ui_version:
|
||||
description: 'UI version override (empty = auto-derive 1.<prowler_minor>.<prowler_patch>; "skip" = hold back)'
|
||||
required: false
|
||||
type: string
|
||||
mcp_version:
|
||||
description: 'MCP Server version override (empty = auto-derive from pending fragment types; "skip" = hold back)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ inputs.prowler_version }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ inputs.prowler_version }}
|
||||
TARGET_BRANCH: ${{ inputs.target_branch }}
|
||||
SDK_VERSION: ${{ inputs.sdk_version }}
|
||||
API_VERSION: ${{ inputs.api_version }}
|
||||
UI_VERSION: ${{ inputs.ui_version }}
|
||||
MCP_VERSION: ${{ inputs.mcp_version }}
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
compile-changelogs:
|
||||
if: github.event_name == 'workflow_dispatch' && github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Block outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ inputs.target_branch }}
|
||||
fetch-depth: 0 # PR attribution resolves each fragment's adding commit from history
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install towncrier
|
||||
run: pip install --no-cache-dir towncrier==25.8.0
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.name 'prowler-bot'
|
||||
git config --global user.email '179230569+prowler-bot@users.noreply.github.com'
|
||||
|
||||
- name: Validate version inputs
|
||||
run: |
|
||||
if [[ ! "$PROWLER_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::Invalid prowler_version syntax: '$PROWLER_VERSION' (must be N.N.N)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$TARGET_BRANCH" != "master" ] && [[ ! "$TARGET_BRANCH" =~ ^v[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::Invalid target_branch syntax: '$TARGET_BRANCH' (must be 'master' or vN.N, e.g. v5.31)"
|
||||
exit 1
|
||||
fi
|
||||
IFS=. read -r prowler_major prowler_minor prowler_patch <<< "$PROWLER_VERSION"
|
||||
prowler_major=$((10#$prowler_major))
|
||||
prowler_minor=$((10#$prowler_minor))
|
||||
prowler_patch=$((10#$prowler_patch))
|
||||
if [ "$prowler_patch" -eq 0 ]; then
|
||||
if [ "$TARGET_BRANCH" != "master" ]; then
|
||||
echo "::error::target_branch must be 'master' for Prowler ${PROWLER_VERSION}; got '${TARGET_BRANCH}'"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
expected_target_branch="v${prowler_major}.${prowler_minor}"
|
||||
if [ "$TARGET_BRANCH" != "$expected_target_branch" ]; then
|
||||
echo "::error::target_branch must be '${expected_target_branch}' for Prowler ${PROWLER_VERSION}; got '${TARGET_BRANCH}'"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
for pair in "sdk_version:$SDK_VERSION" "api_version:$API_VERSION" "ui_version:$UI_VERSION" "mcp_version:$MCP_VERSION"; do
|
||||
input_name="${pair%%:*}"
|
||||
input_value="${pair#*:}"
|
||||
if [ -n "$input_value" ] && [ "$input_value" != "skip" ] && [[ ! "$input_value" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::Invalid $input_name syntax: '$input_value' (must be N.N.N, empty for auto-derivation, or 'skip')"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Compile changelogs
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
component_input() {
|
||||
case "$1" in
|
||||
prowler) echo "$SDK_VERSION" ;;
|
||||
api) echo "$API_VERSION" ;;
|
||||
ui) echo "$UI_VERSION" ;;
|
||||
mcp_server) echo "$MCP_VERSION" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
version_key() {
|
||||
local version="$1"
|
||||
local major minor patch
|
||||
IFS=. read -r major minor patch <<< "$version"
|
||||
printf '%06d.%06d.%06d' "$((10#$major))" "$((10#$minor))" "$((10#$patch))"
|
||||
}
|
||||
|
||||
pending_fragments() {
|
||||
find "$1/changelog.d" -maxdepth 1 -type f ! -name '.gitkeep' ! -name 'README.md' | sort
|
||||
}
|
||||
|
||||
# The component's last released version is the first stamped heading
|
||||
# of its CHANGELOG.md, the same source prepare-release.yml greps.
|
||||
latest_released_version() {
|
||||
grep -m1 -E '^## \[v?[0-9]+\.[0-9]+\.[0-9]+\]' "$1/CHANGELOG.md" | sed -E 's/^## \[v?([0-9]+\.[0-9]+\.[0-9]+)\].*/\1/'
|
||||
}
|
||||
|
||||
has_removed_fragments() {
|
||||
echo "$1" | grep -qE '\.removed(\.[0-9]+)?\.md$'
|
||||
}
|
||||
|
||||
# Resolve every component's effective version before compiling
|
||||
# anything, so a wrong input cannot leave the tree half-compiled.
|
||||
# Empty input = auto-derive (latest released version + semver bump
|
||||
# from the pending fragment types). 'skip' = hold the component back.
|
||||
errors=0
|
||||
compiling=""
|
||||
for component in prowler api ui mcp_server; do
|
||||
input=$(component_input "$component")
|
||||
fragments=$(pending_fragments "$component")
|
||||
|
||||
if [ "$input" = "skip" ]; then
|
||||
if [ -n "$fragments" ]; then
|
||||
echo "::warning::${component}: held back by request; these pending fragments stay for a future release:"
|
||||
echo "$fragments"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ -n "$input" ] && [ -z "$fragments" ]; then
|
||||
echo "::error::${component}: version input '$input' provided but ${component}/changelog.d/ has no pending fragments (wrong input?)"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
if [ -z "$fragments" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
removed_fragments=false
|
||||
if has_removed_fragments "$fragments"; then
|
||||
removed_fragments=true
|
||||
fi
|
||||
current=$(latest_released_version "$component")
|
||||
if [ -z "$current" ]; then
|
||||
echo "::error::${component}: could not read the latest released version from ${component}/CHANGELOG.md; restore the released heading before compiling"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ -n "$input" ]; then
|
||||
effective="$input"
|
||||
mode="explicit"
|
||||
current_key=$(version_key "$current")
|
||||
effective_key=$(version_key "$effective")
|
||||
if [[ "$effective_key" < "$current_key" || "$effective_key" == "$current_key" ]]; then
|
||||
echo "::error::${component}: explicit version '${effective}' must be greater than the latest released version (${current})"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
else
|
||||
if [ "$removed_fragments" = "true" ]; then
|
||||
echo "::error::${component}: pending 'removed' fragments imply a major bump (breaking change); provide its version input explicitly"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
# SDK, UI, and API versions are deterministic mirrors of the
|
||||
# Prowler version (the scheme bump-version.yml codifies): the SDK
|
||||
# mirrors it directly, the UI tracks 1.<minor>.<patch>, and the
|
||||
# API is the independent 1.<minor + 1>.<patch> stream. Only the
|
||||
# MCP Server has its own cadence, derived from fragment types.
|
||||
IFS=. read -r _ prowler_minor prowler_patch <<< "$PROWLER_VERSION"
|
||||
prowler_minor=$((10#$prowler_minor))
|
||||
prowler_patch=$((10#$prowler_patch))
|
||||
case "$component" in
|
||||
prowler) effective="$PROWLER_VERSION" ;;
|
||||
ui) effective="1.${prowler_minor}.${prowler_patch}" ;;
|
||||
api) effective="1.$((prowler_minor + 1)).${prowler_patch}" ;;
|
||||
mcp_server)
|
||||
IFS=. read -r major minor patch <<< "$current"
|
||||
major=$((10#$major))
|
||||
minor=$((10#$minor))
|
||||
patch=$((10#$patch))
|
||||
# Prowler patch releases (vN.N target) are maintenance
|
||||
# releases, so the MCP Server bumps patch regardless of
|
||||
# fragment types; a deliberate exception needs the explicit
|
||||
# version input.
|
||||
if [ "$TARGET_BRANCH" != "master" ]; then
|
||||
effective="${major}.${minor}.$((patch + 1))"
|
||||
if echo "$fragments" | grep -qE '\.(added|deprecated)(\.[0-9]+)?\.md$'; then
|
||||
echo "::warning::${component}: 'added'/'deprecated' fragments are shipping in a Prowler patch; auto-derived a patch bump (${current} -> ${effective}), pass the version input to override"
|
||||
fi
|
||||
elif echo "$fragments" | grep -qE '\.(added|changed|deprecated)(\.[0-9]+)?\.md$'; then
|
||||
effective="${major}.$((minor + 1)).0"
|
||||
else
|
||||
effective="${major}.${minor}.$((patch + 1))"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
current_key=$(version_key "$current")
|
||||
effective_key=$(version_key "$effective")
|
||||
if [[ "$effective_key" < "$current_key" || "$effective_key" == "$current_key" ]]; then
|
||||
echo "::error::${component}: auto-derived version '${effective}' is not greater than the latest released version (${current}); check prowler_version or pass the version input explicitly"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
mode="auto"
|
||||
echo "::notice::${component}: version auto-derived ${current} -> ${effective}"
|
||||
fi
|
||||
|
||||
if [ "$removed_fragments" = "true" ]; then
|
||||
IFS=. read -r current_major _ <<< "$current"
|
||||
current_major=$((10#$current_major))
|
||||
IFS=. read -r effective_major effective_minor effective_patch <<< "$effective"
|
||||
effective_major=$((10#$effective_major))
|
||||
effective_minor=$((10#$effective_minor))
|
||||
effective_patch=$((10#$effective_patch))
|
||||
if [ "$effective_major" -le "$current_major" ] || [ "$effective_minor" -ne 0 ] || [ "$effective_patch" -ne 0 ]; then
|
||||
echo "::error::${component}: removed fragments require a major component release (${current} -> X.0.0 with X > ${current_major}); got ${effective}"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Without the marker the build would insert the new block above the
|
||||
# file header instead of below it.
|
||||
if ! grep -q '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md"; then
|
||||
echo "::error::${component}/CHANGELOG.md is missing the '<!-- changelog: release notes start -->' marker; restore it after the intro line before compiling"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
# A hand-written UNRELEASED block means someone followed the old
|
||||
# convention; its entries would be left out of the compiled block
|
||||
# and out of the release notes extraction.
|
||||
if grep -q '(Prowler UNRELEASED)' "$component/CHANGELOG.md"; then
|
||||
echo "::error::${component}/CHANGELOG.md contains a hand-written '(Prowler UNRELEASED)' block; convert its entries to fragments in ${component}/changelog.d/ and delete the block before compiling"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "${effective} ${mode}" > "${RUNNER_TEMP}/version-${component}.txt"
|
||||
compiling="${compiling}${component} "
|
||||
done
|
||||
if [ "$errors" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$compiling" ]; then
|
||||
echo "::error::Nothing to compile: no component has pending fragments to release"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body_file="${RUNNER_TEMP}/compile-changelogs-pr-body.md"
|
||||
{
|
||||
echo "### Description"
|
||||
echo ""
|
||||
echo "Compiles the pending changelog fragments into the per-component \`CHANGELOG.md\` files for Prowler v${PROWLER_VERSION}, replacing the manual stamping PR."
|
||||
echo ""
|
||||
echo "| Component | Version | Fragments consumed |"
|
||||
echo "|---|---|---|"
|
||||
} > "$body_file"
|
||||
|
||||
compiled_components=""
|
||||
for component in prowler api ui mcp_server; do
|
||||
if [ ! -f "${RUNNER_TEMP}/version-${component}.txt" ]; then
|
||||
echo "Skipping ${component} (no pending fragments or held back)"
|
||||
echo "| \`${component}\` | - | 0 |" >> "$body_file"
|
||||
continue
|
||||
fi
|
||||
read -r version mode < "${RUNNER_TEMP}/version-${component}.txt"
|
||||
version_label="$version"
|
||||
if [ "$mode" = "auto" ]; then
|
||||
version_label="${version} (auto)"
|
||||
fi
|
||||
|
||||
count=$(pending_fragments "$component" | wc -l | tr -d ' ')
|
||||
echo "Compiling ${component} ${version} (${count} fragments, ${mode} version)..."
|
||||
|
||||
# Captured before attribution renames them: these original paths are
|
||||
# what the forward-sync deletes on master (backports copy fragments
|
||||
# verbatim, so filenames match across branches).
|
||||
pending_fragments "$component" > "${RUNNER_TEMP}/consumed-${component}.txt"
|
||||
pre_lines=$(wc -l < "$component/CHANGELOG.md")
|
||||
|
||||
# Attribution must run before the build: towncrier renders the
|
||||
# first dotted segment of each filename as the PR number.
|
||||
python .github/scripts/changelog_attribution.py "$component"
|
||||
towncrier build --config "$component/towncrier.toml" --version "$version" --name "Prowler v${PROWLER_VERSION}" --yes
|
||||
|
||||
# The build only inserts lines right after the marker, so the new
|
||||
# stamped block is exactly the added lines following it. Captured
|
||||
# for the forward-sync to master.
|
||||
post_lines=$(wc -l < "$component/CHANGELOG.md")
|
||||
delta=$((post_lines - pre_lines))
|
||||
marker_line=$(grep -n -m1 '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md" | cut -d: -f1)
|
||||
sed -n "$((marker_line + 1)),$((marker_line + delta))p" "$component/CHANGELOG.md" > "${RUNNER_TEMP}/block-${component}.md"
|
||||
|
||||
compiled_components="${compiled_components}${component} "
|
||||
echo "| \`${component}\` | ${version_label} | ${count} |" >> "$body_file"
|
||||
done
|
||||
echo "COMPILED_COMPONENTS=${compiled_components}" >> "$GITHUB_ENV"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "Review that no pending fragment was dropped (the diff must delete every consumed fragment) and that each new version block is correct, then squash-merge."
|
||||
echo ""
|
||||
echo "### License"
|
||||
echo ""
|
||||
echo "By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license."
|
||||
} >> "$body_file"
|
||||
|
||||
echo "PR_BODY_FILE=${body_file}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Create compile PR
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
commit-message: 'chore(changelog): v${{ env.PROWLER_VERSION }}'
|
||||
branch: compile-changelogs-${{ env.PROWLER_VERSION }}
|
||||
base: ${{ env.TARGET_BRANCH }}
|
||||
title: 'chore(changelog): v${{ env.PROWLER_VERSION }}'
|
||||
body-path: ${{ env.PR_BODY_FILE }}
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
labels: |
|
||||
no-changelog
|
||||
skip-sync
|
||||
|
||||
# Patch compiles (target_branch = v5.X) leave master holding the consumed
|
||||
# fragments and missing the new version block. This applies the equivalent
|
||||
# change to master: insert the same stamped blocks under the marker and
|
||||
# delete the consumed fragments, so the next minor compile cannot
|
||||
# re-release entries that already shipped in the patch.
|
||||
- name: Apply forward-sync to master
|
||||
if: env.TARGET_BRANCH != 'master'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
git checkout -B master origin/master
|
||||
|
||||
version_key() {
|
||||
local version="$1"
|
||||
local major minor patch
|
||||
IFS=. read -r major minor patch <<< "$version"
|
||||
printf '%06d.%06d.%06d' "$((10#$major))" "$((10#$minor))" "$((10#$patch))"
|
||||
}
|
||||
|
||||
release_from_heading() {
|
||||
local heading="$1"
|
||||
echo "$heading" | sed -E 's/^## \[[^]]+\] \(Prowler v?([0-9]+\.[0-9]+\.[0-9]+)\).*/\1/'
|
||||
}
|
||||
|
||||
insert_changelog_block_ordered() {
|
||||
local component="$1"
|
||||
local block_file="$2"
|
||||
local changelog="${component}/CHANGELOG.md"
|
||||
local incoming_heading incoming_release incoming_key
|
||||
local marker_line insertion_line duplicate_line total_lines
|
||||
local line heading existing_release existing_key
|
||||
|
||||
marker_line=$(grep -n -m1 '^<!-- changelog: release notes start -->$' "$changelog" | cut -d: -f1)
|
||||
incoming_heading=$(grep -m1 -E '^## \[[^]]+\] \(Prowler v?[0-9]+\.[0-9]+\.[0-9]+\)' "$block_file" || true)
|
||||
if [ -z "$incoming_heading" ]; then
|
||||
echo "::error::${block_file} does not contain a stamped Prowler release heading"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
incoming_release=$(release_from_heading "$incoming_heading")
|
||||
incoming_key=$(version_key "$incoming_release")
|
||||
insertion_line=""
|
||||
duplicate_line=""
|
||||
|
||||
while IFS=: read -r line heading; do
|
||||
existing_release=$(release_from_heading "$heading")
|
||||
existing_key=$(version_key "$existing_release")
|
||||
if [[ "$incoming_key" == "$existing_key" ]]; then
|
||||
duplicate_line="$line"
|
||||
break
|
||||
fi
|
||||
if [[ "$incoming_key" > "$existing_key" ]]; then
|
||||
insertion_line="$line"
|
||||
break
|
||||
fi
|
||||
done < <(grep -n -E '^## \[[^]]+\] \(Prowler v?[0-9]+\.[0-9]+\.[0-9]+\)' "$changelog" || true)
|
||||
|
||||
if [ -n "$duplicate_line" ]; then
|
||||
echo "::error::${changelog} already contains a block for Prowler v${incoming_release} at line ${duplicate_line}; refusing to insert a duplicate"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$insertion_line" ]; then
|
||||
insertion_line=$(($(wc -l < "$changelog") + 1))
|
||||
fi
|
||||
if [ "$insertion_line" -le "$marker_line" ]; then
|
||||
insertion_line=$((marker_line + 1))
|
||||
fi
|
||||
|
||||
# The captured block window can be off by one blank line on either
|
||||
# end (towncrier re-emits the blank after the marker), so strip the
|
||||
# outer blank lines and pad exactly one on each side: the block
|
||||
# must never glue to the marker above or the next heading below.
|
||||
awk '
|
||||
/[^[:space:]]/ { for (i = 0; i < pending; i++) print ""; pending = 0; print; started = 1; next }
|
||||
started { pending++ }
|
||||
' "$block_file" > "${RUNNER_TEMP}/block-normalized.md"
|
||||
|
||||
total_lines=$(wc -l < "$changelog")
|
||||
{
|
||||
head -n "$((insertion_line - 1))" "$changelog"
|
||||
if [ "$insertion_line" -gt 1 ] && [ -n "$(sed -n "$((insertion_line - 1))p" "$changelog")" ]; then
|
||||
echo ""
|
||||
fi
|
||||
cat "${RUNNER_TEMP}/block-normalized.md"
|
||||
if [ "$insertion_line" -le "$total_lines" ]; then
|
||||
echo ""
|
||||
fi
|
||||
tail -n +"$insertion_line" "$changelog"
|
||||
} > "${RUNNER_TEMP}/changelog.tmp"
|
||||
mv "${RUNNER_TEMP}/changelog.tmp" "$changelog"
|
||||
|
||||
echo "::notice::Inserted ${component} changelog block for Prowler v${incoming_release} at line ${insertion_line}"
|
||||
}
|
||||
|
||||
sync_body="${RUNNER_TEMP}/forward-sync-pr-body.md"
|
||||
{
|
||||
echo "### Description"
|
||||
echo ""
|
||||
echo "Forward-syncs the v${PROWLER_VERSION} compiled changelogs from \`${TARGET_BRANCH}\` to \`master\`: inserts the same stamped version blocks under the insertion marker and deletes the consumed fragments, so the next minor compile cannot re-release entries that already shipped in this patch. Opened automatically by the same run that opened the compile PR; review and squash-merge after it."
|
||||
echo ""
|
||||
echo "| Component | Fragments deleted on master | Skipped (only on ${TARGET_BRANCH}) |"
|
||||
echo "|---|---|---|"
|
||||
} > "$sync_body"
|
||||
|
||||
for component in $COMPILED_COMPONENTS; do
|
||||
block_file="${RUNNER_TEMP}/block-${component}.md"
|
||||
consumed_file="${RUNNER_TEMP}/consumed-${component}.txt"
|
||||
|
||||
if ! grep -qm1 '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md"; then
|
||||
echo "::error::${component}/CHANGELOG.md on master is missing the insertion marker; cannot forward-sync"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
deleted=0
|
||||
skipped=0
|
||||
while IFS= read -r fragment; do
|
||||
if [ -z "$fragment" ]; then
|
||||
continue
|
||||
fi
|
||||
if [ -f "$fragment" ]; then
|
||||
git rm -q "$fragment"
|
||||
deleted=$((deleted + 1))
|
||||
else
|
||||
echo "::notice::${fragment} does not exist on master (change landed only on ${TARGET_BRANCH}); skipping its deletion"
|
||||
skipped=$((skipped + 1))
|
||||
fi
|
||||
done < "$consumed_file"
|
||||
|
||||
insert_changelog_block_ordered "$component" "$block_file"
|
||||
|
||||
echo "| \`${component}\` | ${deleted} | ${skipped} |" >> "$sync_body"
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### License"
|
||||
echo ""
|
||||
echo "By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license."
|
||||
} >> "$sync_body"
|
||||
|
||||
echo "SYNC_BODY_FILE=${sync_body}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Create forward-sync PR
|
||||
if: env.TARGET_BRANCH != 'master'
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
commit-message: 'chore(changelog): v${{ env.PROWLER_VERSION }} forward-sync to master'
|
||||
branch: forward-sync-changelogs-${{ env.PROWLER_VERSION }}
|
||||
base: master
|
||||
title: 'chore(changelog): v${{ env.PROWLER_VERSION }} forward-sync to master'
|
||||
body-path: ${{ env.SYNC_BODY_FILE }}
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
labels: |
|
||||
no-changelog
|
||||
skip-sync
|
||||
@@ -102,6 +102,7 @@ jobs:
|
||||
files_ignore: |
|
||||
mcp_server/README.md
|
||||
mcp_server/CHANGELOG.md
|
||||
mcp_server/changelog.d/**
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -19,6 +19,60 @@ concurrency:
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test-changelog-attribution:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: git fetch --depth=1 origin "${BASE_REF}"
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
.github/scripts/changelog_attribution.py
|
||||
.github/workflows/pr-check-changelog.yml
|
||||
.github/workflows/compile-changelogs.yml
|
||||
.github/towncrier/template.md.jinja
|
||||
*/towncrier.toml
|
||||
tests/github/**
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Test changelog attribution
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: |
|
||||
python3 -m pip install --user --disable-pip-version-check pytest==9.0.3 towncrier==25.8.0
|
||||
python3 -m pytest tests/github
|
||||
|
||||
check-changelog:
|
||||
if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false
|
||||
runs-on: ubuntu-latest
|
||||
@@ -62,53 +116,160 @@ jobs:
|
||||
uv.lock
|
||||
pyproject.toml
|
||||
|
||||
- name: Check for folder changes and changelog presence
|
||||
- name: Check for folder changes and changelog fragment presence
|
||||
id: check-folders
|
||||
run: |
|
||||
missing_changelogs=""
|
||||
fragment_name_re='^[A-Za-z0-9][A-Za-z0-9._-]*\.(added|changed|deprecated|removed|fixed|security)(\.[0-9]+)?\.md$'
|
||||
manual_pr_link_re='(\[\(#[0-9]+\)\]|\[#[0-9]+\]\(|\(#[0-9]+\)|github\.com/[^[:space:]/]+/[^[:space:]/]+/(pull|issues)/[0-9]+)'
|
||||
folder_alt=$(echo "$MONITORED_FOLDERS" | tr ' ' '|')
|
||||
|
||||
missing_fragments=""
|
||||
invalid_fragments=""
|
||||
linked_fragments=""
|
||||
handwritten_changelogs=""
|
||||
|
||||
all_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n')
|
||||
added=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" | tr ' ' '\n')
|
||||
added_or_renamed=$(printf '%s\n%s' "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES}" | tr ' ' '\n')
|
||||
added_modified_or_renamed=$(printf '%s\n%s\n%s' "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_MODIFIED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES}" | tr ' ' '\n')
|
||||
|
||||
# Returns success if the folder has a valid fragment added, modified, or renamed.
|
||||
has_changelog_update() {
|
||||
local folder="$1"
|
||||
if echo "$added_modified_or_renamed" | grep "^${folder}/changelog.d/" | sed "s|^${folder}/changelog.d/||" | grep -qE "$fragment_name_re"; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ "${STEPS_CHANGED_FILES_OUTPUTS_ANY_CHANGED}" == "true" ]]; then
|
||||
# Check monitored folders
|
||||
for folder in $MONITORED_FOLDERS; do
|
||||
# Get files changed in this folder
|
||||
changed_in_folder=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep "^${folder}/" || true)
|
||||
if echo "$all_changed" | grep -q "^${folder}/CHANGELOG.md$"; then
|
||||
echo "Direct CHANGELOG.md edits are not allowed for ${folder}/"
|
||||
handwritten_changelogs="${handwritten_changelogs}- \`${folder}/CHANGELOG.md\`"$'\n'
|
||||
fi
|
||||
|
||||
changed_in_folder=$(echo "$all_changed" | grep "^${folder}/" | grep -v "^${folder}/CHANGELOG.md$" || true)
|
||||
|
||||
if [ -n "$changed_in_folder" ]; then
|
||||
echo "Detected changes in ${folder}/"
|
||||
|
||||
# Check if CHANGELOG.md was updated
|
||||
if ! echo "$changed_in_folder" | grep -q "^${folder}/CHANGELOG.md$"; then
|
||||
echo "No changelog update found for ${folder}/"
|
||||
missing_changelogs="${missing_changelogs}- \`${folder}\`"$'\n'
|
||||
if ! has_changelog_update "$folder"; then
|
||||
echo "No changelog fragment found for ${folder}/"
|
||||
missing_fragments="${missing_fragments}- \`${folder}\`"$'\n'
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Check root-level dependency files (uv.lock, pyproject.toml)
|
||||
# These are associated with the prowler folder changelog
|
||||
root_deps_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
|
||||
root_deps_changed=$(echo "$all_changed" | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
|
||||
if [ -n "$root_deps_changed" ]; then
|
||||
echo "Detected changes in root dependency files: $root_deps_changed"
|
||||
# Check if prowler/CHANGELOG.md was already updated (might have been caught above)
|
||||
prowler_changelog_updated=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep "^prowler/CHANGELOG.md$" || true)
|
||||
if [ -z "$prowler_changelog_updated" ]; then
|
||||
if ! has_changelog_update "prowler"; then
|
||||
# Only add if prowler wasn't already flagged
|
||||
if ! echo "$missing_changelogs" | grep -q "prowler"; then
|
||||
echo "No changelog update found for root dependency changes"
|
||||
missing_changelogs="${missing_changelogs}- \`prowler\` (root dependency files changed)"$'\n'
|
||||
if ! echo "$missing_fragments" | grep -q "prowler"; then
|
||||
echo "No changelog fragment found for root dependency changes"
|
||||
missing_fragments="${missing_fragments}- \`prowler\` (root dependency files changed)"$'\n'
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Validate the filename of every fragment added by this PR
|
||||
added_fragments=$(echo "$added_or_renamed" | grep -E "^(${folder_alt})/changelog\.d/" || true)
|
||||
for fragment in $added_fragments; do
|
||||
name=$(basename "$fragment")
|
||||
if [ "$name" = ".gitkeep" ] || [ "$name" = "README.md" ]; then
|
||||
continue
|
||||
fi
|
||||
if ! echo "$name" | grep -qE "$fragment_name_re"; then
|
||||
echo "Invalid fragment filename: $fragment"
|
||||
invalid_fragments="${invalid_fragments}- \`${fragment}\`"$'\n'
|
||||
fi
|
||||
done
|
||||
|
||||
# Lint fragment content: the PR link is attached automatically at
|
||||
# compile time, so a hand-written PR or issue link would be wrong
|
||||
touched_fragments=$(echo "$added_modified_or_renamed" | grep -E "^(${folder_alt})/changelog\.d/" || true)
|
||||
for fragment in $touched_fragments; do
|
||||
name=$(basename "$fragment")
|
||||
if [ "$name" = ".gitkeep" ] || [ "$name" = "README.md" ] || [ ! -f "$fragment" ]; then
|
||||
continue
|
||||
fi
|
||||
if grep -qE "$manual_pr_link_re" "$fragment"; then
|
||||
echo "Fragment contains a hand-written PR or issue link: $fragment"
|
||||
linked_fragments="${linked_fragments}- \`${fragment}\`"$'\n'
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
{
|
||||
echo "missing_changelogs<<EOF"
|
||||
echo -e "${missing_changelogs}"
|
||||
echo "EOF"
|
||||
} >> $GITHUB_OUTPUT
|
||||
# Suggest a slug derived from the branch name for the bot comment
|
||||
suggested_slug=$(echo "$HEAD_REF" | tr '[:upper:]' '[:lower:]' | sed 's|.*/||; s/[^a-z0-9._-]/-/g; s/^[^a-z0-9]*//')
|
||||
if [ -z "$suggested_slug" ]; then
|
||||
suggested_slug="my-change"
|
||||
fi
|
||||
|
||||
fragment_help="A changelog fragment is a small Markdown file named \`<slug>.<type>.md\` under \`<component>/changelog.d/\`, where \`<type>\` is one of \`added\`, \`changed\`, \`deprecated\`, \`removed\`, \`fixed\` or \`security\`. Its content is the changelog entry text, without the PR link (added automatically at release time) and without a trailing period. For example:
|
||||
|
||||
\`\`\`
|
||||
echo 'Entry text describing the change' > <component>/changelog.d/${suggested_slug}.fixed.md
|
||||
\`\`\`
|
||||
|
||||
If this PR does not need a changelog entry, add the \`no-changelog\` label instead."
|
||||
|
||||
if [ -n "$missing_fragments" ] || [ -n "$invalid_fragments" ] || [ -n "$linked_fragments" ] || [ -n "$handwritten_changelogs" ]; then
|
||||
comment_body=""
|
||||
if [ -n "$missing_fragments" ]; then
|
||||
comment_body="⚠️ **Changes detected in the following folders without a changelog fragment:**"$'\n\n'"${missing_fragments}"$'\n'
|
||||
fi
|
||||
if [ -n "$invalid_fragments" ]; then
|
||||
comment_body="${comment_body}⚠️ **Changelog fragment filenames that do not follow the naming convention:**"$'\n\n'"${invalid_fragments}"$'\n'
|
||||
fi
|
||||
if [ -n "$linked_fragments" ]; then
|
||||
comment_body="${comment_body}⚠️ **Changelog fragments containing a hand-written PR or issue link (remove it; the PR link is attached automatically at release time):**"$'\n\n'"${linked_fragments}"$'\n'
|
||||
fi
|
||||
if [ -n "$handwritten_changelogs" ]; then
|
||||
comment_body="${comment_body}⚠️ **Direct \`CHANGELOG.md\` edits are not allowed in regular PRs:**"$'\n\n'"${handwritten_changelogs}"$'\n'
|
||||
fi
|
||||
comment_body="${comment_body}${fragment_help}"
|
||||
else
|
||||
comment_body="✅ All required changelog fragments are present."
|
||||
fi
|
||||
|
||||
write_multiline_output() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
local delimiter
|
||||
|
||||
while true; do
|
||||
delimiter="EOF_$(openssl rand -hex 16)"
|
||||
if ! grep -qxF "$delimiter" <<< "$value"; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo "${name}<<${delimiter}"
|
||||
if [ -n "$value" ]; then
|
||||
printf '%s\n' "$value"
|
||||
fi
|
||||
echo "${delimiter}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
}
|
||||
|
||||
write_multiline_output "missing_fragments" "$missing_fragments"
|
||||
write_multiline_output "invalid_fragments" "$invalid_fragments"
|
||||
write_multiline_output "linked_fragments" "$linked_fragments"
|
||||
write_multiline_output "handwritten_changelogs" "$handwritten_changelogs"
|
||||
write_multiline_output "comment_body" "$comment_body"
|
||||
env:
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ANY_CHANGED: ${{ steps.changed-files.outputs.any_changed }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_MODIFIED_FILES: ${{ steps.changed-files.outputs.modified_files }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES: ${{ steps.changed-files.outputs.renamed_files }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
|
||||
- name: Find existing changelog comment
|
||||
if: github.event.pull_request.head.repo.full_name == github.repository
|
||||
@@ -128,14 +289,10 @@ jobs:
|
||||
edit-mode: replace
|
||||
body: |
|
||||
<!-- changelog-check -->
|
||||
${{ steps.check-folders.outputs.missing_changelogs != '' && format('⚠️ **Changes detected in the following folders without a corresponding update to the `CHANGELOG.md`:**
|
||||
${{ steps.check-folders.outputs.comment_body }}
|
||||
|
||||
{0}
|
||||
|
||||
Please add an entry to the corresponding `CHANGELOG.md` file to maintain a clear history of changes.', steps.check-folders.outputs.missing_changelogs) || '✅ All necessary `CHANGELOG.md` files have been updated.' }}
|
||||
|
||||
- name: Fail if changelog is missing
|
||||
if: steps.check-folders.outputs.missing_changelogs != ''
|
||||
- name: Fail if changelog fragment is missing or invalid
|
||||
if: steps.check-folders.outputs.missing_fragments != '' || steps.check-folders.outputs.invalid_fragments != '' || steps.check-folders.outputs.linked_fragments != '' || steps.check-folders.outputs.handwritten_changelogs != ''
|
||||
run: |
|
||||
echo "::error::Missing changelog updates in some folders"
|
||||
echo "::error::Missing, invalid, or disallowed changelog updates"
|
||||
exit 1
|
||||
|
||||
@@ -55,6 +55,7 @@ jobs:
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
|
||||
@@ -12,6 +12,7 @@ on:
|
||||
- '.github/workflows/sdk-codeql.yml'
|
||||
- '.github/codeql/sdk-codeql-config.yml'
|
||||
- '!prowler/CHANGELOG.md'
|
||||
- '!prowler/changelog.d/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
@@ -23,6 +24,7 @@ on:
|
||||
- '.github/workflows/sdk-codeql.yml'
|
||||
- '.github/codeql/sdk-codeql-config.yml'
|
||||
- '!prowler/CHANGELOG.md'
|
||||
- '!prowler/changelog.d/**'
|
||||
schedule:
|
||||
- cron: '00 12 * * *'
|
||||
|
||||
|
||||
@@ -113,6 +113,7 @@ jobs:
|
||||
.github/workflows/sdk-container-checks.yml
|
||||
files_ignore: |
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
|
||||
@@ -73,6 +73,7 @@ jobs:
|
||||
.github/scripts/osv-scan.sh
|
||||
files_ignore: |
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -77,6 +77,7 @@ jobs:
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
|
||||
@@ -10,6 +10,7 @@ on:
|
||||
- '.github/workflows/ui-codeql.yml'
|
||||
- '.github/codeql/ui-codeql-config.yml'
|
||||
- '!ui/CHANGELOG.md'
|
||||
- '!ui/changelog.d/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
@@ -19,6 +20,7 @@ on:
|
||||
- '.github/workflows/ui-codeql.yml'
|
||||
- '.github/codeql/ui-codeql-config.yml'
|
||||
- '!ui/CHANGELOG.md'
|
||||
- '!ui/changelog.d/**'
|
||||
schedule:
|
||||
- cron: '00 12 * * *'
|
||||
|
||||
|
||||
@@ -102,6 +102,7 @@ jobs:
|
||||
files: ui/**
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/changelog.d/**
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ on:
|
||||
- '.github/test-impact.yml'
|
||||
- 'ui/**'
|
||||
- 'api/**' # API changes can affect UI E2E
|
||||
- '!ui/CHANGELOG.md'
|
||||
- '!api/CHANGELOG.md'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
|
||||
@@ -64,6 +64,7 @@ jobs:
|
||||
.github/workflows/ui-tests.yml
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/changelog.d/**
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
|
||||
@@ -72,13 +72,13 @@ repos:
|
||||
exclude: contrib
|
||||
priority: 30
|
||||
|
||||
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/)
|
||||
## PYTHON — SDK (prowler/, tests/, dashboard/, util/, scripts/, docs/scripts/)
|
||||
- repo: https://github.com/myint/autoflake
|
||||
rev: v2.3.3
|
||||
hooks:
|
||||
- id: autoflake
|
||||
name: "SDK - autoflake"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variable"]
|
||||
priority: 20
|
||||
|
||||
@@ -87,7 +87,7 @@ repos:
|
||||
hooks:
|
||||
- id: isort
|
||||
name: "SDK - isort"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
args: ["--profile", "black"]
|
||||
stages: ["pre-commit"]
|
||||
priority: 20
|
||||
@@ -97,7 +97,7 @@ repos:
|
||||
hooks:
|
||||
- id: black
|
||||
name: "SDK - black"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
@@ -105,7 +105,7 @@ repos:
|
||||
hooks:
|
||||
- id: flake8
|
||||
name: "SDK - flake8"
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
args: ["--ignore=E266,W503,E203,E501,W605"]
|
||||
priority: 30
|
||||
|
||||
@@ -142,6 +142,14 @@ repos:
|
||||
files: { glob: ["mcp_server/**/*.py"] }
|
||||
priority: 20
|
||||
|
||||
- id: generate-provider-cards
|
||||
name: "Docs - regenerate provider cards snippet"
|
||||
entry: python docs/scripts/generate_provider_cards.py
|
||||
language: system
|
||||
files: { glob: ["docs/user-guide/providers/**/getting-started-*.mdx", "docs/scripts/generate_provider_cards.py", "docs/snippets/provider-cards.mdx", "api/src/backend/api/models.py"] }
|
||||
pass_filenames: false
|
||||
priority: 20
|
||||
|
||||
## PYTHON — uv (API + SDK)
|
||||
- repo: https://github.com/astral-sh/uv-pre-commit
|
||||
rev: 0.11.14
|
||||
@@ -183,7 +191,7 @@ repos:
|
||||
entry: pylint --disable=W,C,R,E -j 0 -rn -sn
|
||||
language: system
|
||||
types: [python]
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts}/**/*.py"] }
|
||||
files: { glob: ["{prowler,tests,dashboard,util,scripts,docs/scripts}/**/*.py"] }
|
||||
priority: 30
|
||||
|
||||
- id: trufflehog
|
||||
|
||||
@@ -2,6 +2,59 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [1.34.2] (Prowler v5.33.2)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths graph mutations now retry transient Neptune concurrency and deadline failures, while Neo4j mutations use managed transaction retries [(#11968)](https://github.com/prowler-cloud/prowler/pull/11968)
|
||||
- Attack Paths scans now use bounded child node identifiers for normalized list values in Neo4j and Neptune, preventing Neo4j RANGE index key size failures [(#11969)](https://github.com/prowler-cloud/prowler/pull/11969)
|
||||
- `scan-summary` aggregation now upserts summaries in deterministic conflict-key order, preventing PostgreSQL deadlocks during concurrent reaggregation [(#11971)](https://github.com/prowler-cloud/prowler/pull/11971)
|
||||
|
||||
---
|
||||
|
||||
## [1.34.1] (Prowler v5.33.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Session tokens are rejected after account password updates [(#11914)](https://github.com/prowler-cloud/prowler/pull/11914)
|
||||
- Jira dispatch task results now surface user-facing Jira failure messages [(#11925)](https://github.com/prowler-cloud/prowler/pull/11925)
|
||||
- AWS Attack Paths privilege escalation queries no longer fail on Neo4j with `Aggregation column contains implicit grouping expressions` [(#11939)](https://github.com/prowler-cloud/prowler/pull/11939)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- OpenAI-compatible Lighthouse provider base URLs are restricted before connection checks [(#11940)](https://github.com/prowler-cloud/prowler/pull/11940)
|
||||
- `LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS` environment variable to allow internal hosts as OpenAI-compatible Lighthouse AI base URLs [(#11942)](https://github.com/prowler-cloud/prowler/pull/11942)
|
||||
|
||||
---
|
||||
|
||||
## [1.34.0] (Prowler v5.33.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Compliance PDF reports no longer require provider credentials: findings are enriched from the provider metadata stored in the database, so reports generate even after the provider secret is deleted or its credentials become invalid [(#11845)](https://github.com/prowler-cloud/prowler/pull/11845)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Provider scans now queue behind active provider scans instead of dispatching concurrently, and resource failed-finding counters retry database conflicts with stable row locking [(#11848)](https://github.com/prowler-cloud/prowler/pull/11848)
|
||||
|
||||
---
|
||||
|
||||
## [1.33.1] (Prowler v5.32.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths: Scan rows now have database defaults for `is_migrated` and `sink_backend` so `scan-perform-scheduled` inserts survive deploy skew [(#11826)](https://github.com/prowler-cloud/prowler/pull/11826)
|
||||
- Invited users now keep their invitation context when completing authentication with Google, GitHub, or SAML, so the invitation is accepted during login [(#11752)](https://github.com/prowler-cloud/prowler/pull/11752)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- User profile updates now allow users to update their own account while requiring user-management permissions to update other users in the same tenant [(#11792)](https://github.com/prowler-cloud/prowler/pull/11792)
|
||||
- Kubernetes provider credentials now reject kubeconfigs using `exec` authentication in Prowler Cloud, preventing user-supplied commands from running on Cloud workers [(#11753)](https://github.com/prowler-cloud/prowler/pull/11753)
|
||||
|
||||
---
|
||||
|
||||
## [1.33.0] (Prowler v5.32.0)
|
||||
|
||||
### 🚀 Added
|
||||
@@ -19,7 +72,6 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Attack Paths: Provider graph cleanup now deletes Neo4j and Neptune relationships in directed batches before deleting nodes [(#11755)](https://github.com/prowler-cloud/prowler/pull/11755)
|
||||
- `scan-perform` no longer reports an error when a provider is deleted during a running scan [(#11696)](https://github.com/prowler-cloud/prowler/pull/11696)
|
||||
|
||||
|
||||
---
|
||||
|
||||
## [1.32.1] (Prowler v5.31.1)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Changelog fragments
|
||||
|
||||
Each PR adds one small file here instead of editing `CHANGELOG.md` directly, so concurrent PRs never conflict.
|
||||
|
||||
- Filename: `<slug>.<type>.md`, e.g. `my-new-check.added.md` (slug is free-form: letters, digits, `.`, `_`, `-`)
|
||||
- `<type>` is one of: `added`, `changed`, `deprecated`, `removed`, `fixed`, `security`
|
||||
- Content: one line with the changelog entry text, without the PR link and without a trailing period (the PR link is attached automatically at release time)
|
||||
- A PR adds as many fragment files as entries it needs, freely mixing types (one file per entry); same-type entries just use different slugs
|
||||
|
||||
Fragments are compiled into `CHANGELOG.md` when a release is prepared. Full conventions: `skills/prowler-changelog/SKILL.md`.
|
||||
@@ -45,7 +45,7 @@ dependencies = [
|
||||
"gunicorn==26.0.0",
|
||||
"uvloop==0.22.1",
|
||||
"lxml==6.1.0",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@master",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@v5.33",
|
||||
"psycopg2-binary==2.9.9",
|
||||
"pytest-celery[redis] (==1.3.0)",
|
||||
"sentry-sdk[django] (==2.56.0)",
|
||||
@@ -71,7 +71,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.33.0"
|
||||
version = "1.34.2"
|
||||
|
||||
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
|
||||
# target-version tracks this project's lowest supported Python.
|
||||
|
||||
@@ -9,6 +9,7 @@ from api.models import (
|
||||
User,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.utils import accept_invitation_for_user
|
||||
from django.db import transaction
|
||||
|
||||
|
||||
@@ -20,6 +21,22 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
except User.DoesNotExist:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_invitation_token(request):
|
||||
for source_name in ("data", "POST"):
|
||||
data = getattr(request, source_name, None) or {}
|
||||
if not hasattr(data, "get"):
|
||||
continue
|
||||
invitation_token = data.get("invitation_token")
|
||||
if invitation_token:
|
||||
return invitation_token
|
||||
|
||||
wrapped_request = getattr(request, "_request", None)
|
||||
if wrapped_request and wrapped_request is not request:
|
||||
return ProwlerSocialAccountAdapter._get_invitation_token(wrapped_request)
|
||||
|
||||
return None
|
||||
|
||||
def pre_social_login(self, request, sociallogin):
|
||||
# Link existing accounts with the same email address
|
||||
email = sociallogin.account.extra_data.get("email")
|
||||
@@ -83,29 +100,38 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
|
||||
user.name = social_account_name
|
||||
user.save(using=MainRouter.admin_db)
|
||||
|
||||
tenant = Tenant.objects.using(MainRouter.admin_db).create(
|
||||
name=f"{user.email.split('@')[0]} default tenant"
|
||||
)
|
||||
with rls_transaction(str(tenant.id)):
|
||||
Membership.objects.using(MainRouter.admin_db).create(
|
||||
user=user, tenant=tenant, role=Membership.RoleChoices.OWNER
|
||||
)
|
||||
role = Role.objects.using(MainRouter.admin_db).create(
|
||||
name="admin",
|
||||
tenant_id=tenant.id,
|
||||
manage_users=True,
|
||||
manage_account=True,
|
||||
manage_billing=True,
|
||||
manage_providers=True,
|
||||
manage_integrations=True,
|
||||
manage_scans=True,
|
||||
unlimited_visibility=True,
|
||||
)
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
invitation_token = self._get_invitation_token(request)
|
||||
if invitation_token:
|
||||
invitation, _ = accept_invitation_for_user(
|
||||
user=user,
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
invitation_token=invitation_token,
|
||||
)
|
||||
request.prowler_invitation_token = invitation_token
|
||||
request.prowler_invitation_tenant_id = str(invitation.tenant_id)
|
||||
else:
|
||||
tenant = Tenant.objects.using(MainRouter.admin_db).create(
|
||||
name=f"{user.email.split('@')[0]} default tenant"
|
||||
)
|
||||
with rls_transaction(str(tenant.id)):
|
||||
Membership.objects.using(MainRouter.admin_db).create(
|
||||
user=user, tenant=tenant, role=Membership.RoleChoices.OWNER
|
||||
)
|
||||
role = Role.objects.using(MainRouter.admin_db).create(
|
||||
name="admin",
|
||||
tenant_id=tenant.id,
|
||||
manage_users=True,
|
||||
manage_account=True,
|
||||
manage_billing=True,
|
||||
manage_providers=True,
|
||||
manage_integrations=True,
|
||||
manage_scans=True,
|
||||
unlimited_visibility=True,
|
||||
)
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
user=user,
|
||||
role=role,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
else:
|
||||
request.session["saml_user_created"] = str(user.id)
|
||||
|
||||
|
||||
@@ -418,7 +418,8 @@ AWS_APPRUNNER_PRIVESC_UPDATE_SERVICE = AttackPathsQueryDefinition(
|
||||
// Find existing App Runner services with roles attached (potential targets)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'tasks.apprunner.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -523,7 +524,8 @@ AWS_BEDROCK_PRIVESC_INVOKE_CODE_INTERPRETER = AttackPathsQueryDefinition(
|
||||
// Find roles that trust the Bedrock AgentCore service (already attached to existing code interpreters)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock-agentcore.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -607,7 +609,8 @@ AWS_CLOUDFORMATION_PRIVESC_UPDATE_STACK = AttackPathsQueryDefinition(
|
||||
// Find roles that trust CloudFormation service (already attached to existing stacks)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -753,7 +756,8 @@ AWS_CLOUDFORMATION_PRIVESC_CHANGESET = AttackPathsQueryDefinition(
|
||||
// Find roles that trust CloudFormation service (already attached to existing stacks)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -844,7 +848,8 @@ AWS_CODEBUILD_PRIVESC_START_BUILD = AttackPathsQueryDefinition(
|
||||
// Find roles that trust CodeBuild service (already attached to existing projects)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -880,7 +885,8 @@ AWS_CODEBUILD_PRIVESC_START_BUILD_BATCH = AttackPathsQueryDefinition(
|
||||
// Find roles that trust CodeBuild service (already attached to existing projects)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1096,7 +1102,8 @@ AWS_EC2_PRIVESC_MODIFY_INSTANCE_ATTRIBUTE = AttackPathsQueryDefinition(
|
||||
// Find EC2 instances with instance profiles (potential targets)
|
||||
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1187,7 +1194,8 @@ AWS_EC2_PRIVESC_LAUNCH_TEMPLATE = AttackPathsQueryDefinition(
|
||||
// Find launch templates in the account (potential targets)
|
||||
MATCH path_target = (aws)--(template:LaunchTemplate)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1223,7 +1231,8 @@ AWS_EC2INSTANCECONNECT_PRIVESC_SEND_SSH_PUBLIC_KEY = AttackPathsQueryDefinition(
|
||||
// Find EC2 instances with attached roles (targets for credential theft via IMDS)
|
||||
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1539,7 +1548,8 @@ AWS_ECS_PRIVESC_EXECUTE_COMMAND = AttackPathsQueryDefinition(
|
||||
// Target: roles already attached to running tasks (trust ECS tasks service)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ecs-tasks.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1622,7 +1632,8 @@ AWS_GLUE_PRIVESC_UPDATE_DEV_ENDPOINT = AttackPathsQueryDefinition(
|
||||
// Find roles that trust Glue service (already attached to existing dev endpoints)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'glue.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -3337,7 +3348,8 @@ AWS_SSM_PRIVESC_START_SESSION = AttackPathsQueryDefinition(
|
||||
// Find EC2 instances with attached roles (targets for credential theft via IMDS)
|
||||
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -3373,7 +3385,8 @@ AWS_SSM_PRIVESC_SEND_COMMAND = AttackPathsQueryDefinition(
|
||||
// Find EC2 instances with attached roles (targets for credential theft via IMDS)
|
||||
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
@@ -9,17 +11,19 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RetryableSession:
|
||||
"""
|
||||
Wrapper around `neo4j.Session` that retries `neo4j.exceptions.ServiceUnavailable` errors.
|
||||
"""
|
||||
"""Wrapper around ``neo4j.Session`` with a refreshable retry policy."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: Callable[[], neo4j.Session],
|
||||
max_retries: int,
|
||||
retry_if: Callable[[Exception], bool] | None = None,
|
||||
initial_retry_delay_seconds: float = 0,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._max_retries = max(0, max_retries)
|
||||
self._retry_if = retry_if
|
||||
self._initial_retry_delay_seconds = max(0.0, initial_retry_delay_seconds)
|
||||
self._session = self._session_factory()
|
||||
|
||||
def close(self) -> None:
|
||||
@@ -56,24 +60,47 @@ class RetryableSession:
|
||||
method = getattr(self._session, method_name)
|
||||
return method(*args, **kwargs)
|
||||
|
||||
except (
|
||||
BrokenPipeError,
|
||||
ConnectionResetError,
|
||||
neo4j.exceptions.ServiceUnavailable,
|
||||
) as exc: # pragma: no cover - depends on infra
|
||||
except Exception as exc:
|
||||
if not self._should_retry(exc):
|
||||
raise
|
||||
|
||||
last_exc = exc
|
||||
attempt += 1
|
||||
|
||||
if attempt > self._max_retries:
|
||||
raise
|
||||
|
||||
delay = self._retry_delay(attempt)
|
||||
logger.warning(
|
||||
f"Neo4j session {method_name} failed with {type(exc).__name__} ({attempt}/{self._max_retries} attempts). Retrying..."
|
||||
"Graph session %s failed with %s; retry %s/%s in %.3fs",
|
||||
method_name,
|
||||
type(exc).__name__,
|
||||
attempt,
|
||||
self._max_retries,
|
||||
delay,
|
||||
)
|
||||
self._refresh_session()
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
|
||||
raise last_exc if last_exc else RuntimeError("Unexpected retry loop exit")
|
||||
|
||||
def _should_retry(self, exc: Exception) -> bool:
|
||||
if isinstance(
|
||||
exc,
|
||||
(
|
||||
BrokenPipeError,
|
||||
ConnectionResetError,
|
||||
neo4j.exceptions.ServiceUnavailable,
|
||||
),
|
||||
):
|
||||
return True
|
||||
return self._retry_if(exc) if self._retry_if else False
|
||||
|
||||
def _retry_delay(self, attempt: int) -> float:
|
||||
max_delay = self._initial_retry_delay_seconds * (2**attempt)
|
||||
return random.uniform(max_delay / 2, max_delay) if max_delay else 0
|
||||
|
||||
def _refresh_session(self) -> None:
|
||||
if self._session is not None:
|
||||
try:
|
||||
|
||||
@@ -42,6 +42,10 @@ def delete_batches(
|
||||
batch_size: int,
|
||||
drop_t0: float,
|
||||
) -> tuple[int, int]:
|
||||
def delete_batch(tx: Any) -> int:
|
||||
record = tx.run(query, {"batch_size": batch_size}).single()
|
||||
return (record[count_key] if record else 0) or 0
|
||||
|
||||
deleted_total = initial_total
|
||||
batches = 0
|
||||
while True:
|
||||
@@ -56,8 +60,7 @@ def delete_batches(
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
record = session.run(query, {"batch_size": batch_size}).single()
|
||||
deleted = (record[count_key] if record else 0) or 0
|
||||
deleted = session.execute_write(delete_batch)
|
||||
if deleted == 0:
|
||||
return deleted_total, batches
|
||||
|
||||
|
||||
@@ -355,7 +355,7 @@ class Neo4jSink(SinkDatabase):
|
||||
f"ON (n.`{PROVIDER_ELEMENT_ID_PROPERTY}`)"
|
||||
)
|
||||
with self.get_session(database) as session:
|
||||
session.run(query).consume()
|
||||
session.execute_write(lambda tx: tx.run(query).consume())
|
||||
|
||||
def write_nodes(
|
||||
self,
|
||||
@@ -377,7 +377,7 @@ class Neo4jSink(SinkDatabase):
|
||||
SET n += row.props
|
||||
"""
|
||||
with self.get_session(database) as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
@@ -403,7 +403,7 @@ class Neo4jSink(SinkDatabase):
|
||||
SET r += row.props
|
||||
"""
|
||||
with self.get_session(database) as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
# For compatibility with test harnesses that patch the concrete driver
|
||||
def get_driver(self) -> neo4j.Driver:
|
||||
|
||||
@@ -59,17 +59,28 @@ CONNECTION_TIMEOUT = env.int("NEPTUNE_CONNECTION_TIMEOUT", default=10)
|
||||
# Roll connections hourly so SigV4 rotations and cert refreshes don't strand long-lived pool entries
|
||||
MAX_CONNECTION_LIFETIME = env.int("NEPTUNE_MAX_CONNECTION_LIFETIME", default=3600)
|
||||
MAX_CONNECTION_POOL_SIZE = env.int("NEPTUNE_MAX_CONNECTION_POOL_SIZE", default=50)
|
||||
NEPTUNE_WRITE_RETRY_DELAY_SECONDS = 2
|
||||
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
RETRYABLE_WRITE_ERROR_PREFIXES = (
|
||||
"Operation failed due to conflicting concurrent operations",
|
||||
"Operation terminated (deadline exceeded)",
|
||||
)
|
||||
|
||||
# Refresh 60s before the 5-minute SigV4 window closes
|
||||
SIGV4_TOKEN_LIFETIME_MINUTES = 4
|
||||
|
||||
|
||||
def _is_retryable_write_error(exc: Exception) -> bool:
|
||||
if not isinstance(exc, neo4j.exceptions.Neo4jError):
|
||||
return False
|
||||
return bool(exc.message and exc.message.startswith(RETRYABLE_WRITE_ERROR_PREFIXES))
|
||||
|
||||
|
||||
class NeptuneSink(SinkDatabase):
|
||||
"""Neptune-backed sink. Single database; isolation is label-based."""
|
||||
|
||||
@@ -205,11 +216,16 @@ class NeptuneSink(SinkDatabase):
|
||||
|
||||
session_wrapper: RetryableSession | None = None
|
||||
try:
|
||||
is_write_session = default_access_mode != neo4j.READ_ACCESS
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: driver.session(
|
||||
default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
retry_if=_is_retryable_write_error if is_write_session else None,
|
||||
initial_retry_delay_seconds=(
|
||||
NEPTUNE_WRITE_RETRY_DELAY_SECONDS if is_write_session else 0
|
||||
),
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
@@ -405,7 +421,7 @@ class NeptuneSink(SinkDatabase):
|
||||
SET n.`{PROVIDER_ELEMENT_ID_PROPERTY}` = row.provider_element_id
|
||||
"""
|
||||
with self.get_session() as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
@@ -429,7 +445,7 @@ class NeptuneSink(SinkDatabase):
|
||||
SET r += row.props
|
||||
"""
|
||||
with self.get_session() as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
# Test helpers
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0096_attack_paths_scan_is_migrated"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="attackpathsscan",
|
||||
name="is_migrated",
|
||||
field=models.BooleanField(db_default=False, default=False),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="attackpathsscan",
|
||||
name="sink_backend",
|
||||
field=models.CharField(
|
||||
choices=[("neo4j", "Neo4j"), ("neptune", "Neptune")],
|
||||
db_default="neo4j",
|
||||
default="neo4j",
|
||||
max_length=16,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -814,9 +814,10 @@ class AttackPathsScan(RowLevelSecurityProtectedModel):
|
||||
# still using the previous graph shape. Query catalog selection uses this
|
||||
# flag; physical read routing uses sink_backend below.
|
||||
# TODO: drop after Neptune cutover
|
||||
is_migrated = models.BooleanField(default=False)
|
||||
is_migrated = models.BooleanField(default=False, db_default=False)
|
||||
sink_backend = models.CharField(
|
||||
choices=SinkBackendChoices.choices,
|
||||
db_default=SinkBackendChoices.NEO4J,
|
||||
default=SinkBackendChoices.NEO4J,
|
||||
max_length=16,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.33.0
|
||||
version: 1.34.2
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
|
||||
class SSEChannelManager(DefaultChannelManager):
|
||||
"""Connect `django-eventstream` to the platform's SSE viewsets."""
|
||||
|
||||
def get_channels_for_request(self, request: Request, view_kwargs: dict) -> set[str]: # noqa: vulture
|
||||
def get_channels_for_request(self, request: Request, view_kwargs: dict) -> set[str]:
|
||||
"""Return the request's channels scoped to the active JWT tenant.
|
||||
|
||||
Args:
|
||||
@@ -30,6 +30,7 @@ class SSEChannelManager(DefaultChannelManager):
|
||||
The subset of `request.sse_channels` whose embedded tenant
|
||||
matches the active request tenant.
|
||||
"""
|
||||
_ = view_kwargs
|
||||
try:
|
||||
request_tenant_id = UUID(str(getattr(request, "tenant_id", None)))
|
||||
except (TypeError, ValueError):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
@@ -8,6 +9,10 @@ from conftest import TEST_PASSWORD, get_api_tokens, get_authorization_header
|
||||
from django.urls import reverse
|
||||
from drf_simple_apikey.crypto import get_crypto
|
||||
from rest_framework.test import APIClient
|
||||
from rest_framework_simplejwt.token_blacklist.models import (
|
||||
BlacklistedToken,
|
||||
OutstandingToken,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -103,6 +108,118 @@ def test_refresh_token(create_test_user, tenants_fixture):
|
||||
assert new_refresh_response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_password_change_invalidates_existing_tokens(create_test_user, tenants_fixture):
|
||||
client = APIClient()
|
||||
new_password = "ChangedSecret123@"
|
||||
|
||||
access_token, refresh_token = get_api_tokens(
|
||||
client, create_test_user.email, TEST_PASSWORD
|
||||
)
|
||||
auth_headers = get_authorization_header(access_token)
|
||||
outstanding_token_ids = list(
|
||||
OutstandingToken.objects.filter(user=create_test_user).values_list(
|
||||
"id", flat=True
|
||||
)
|
||||
)
|
||||
assert outstanding_token_ids
|
||||
assert not BlacklistedToken.objects.filter(
|
||||
token_id__in=outstanding_token_ids
|
||||
).exists()
|
||||
|
||||
password_change_payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(create_test_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
}
|
||||
}
|
||||
password_change_response = client.patch(
|
||||
reverse("user-detail", kwargs={"pk": create_test_user.id}),
|
||||
data=json.dumps(password_change_payload),
|
||||
headers=auth_headers,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
assert password_change_response.status_code == 200, password_change_response.json()
|
||||
assert BlacklistedToken.objects.filter(
|
||||
token_id__in=outstanding_token_ids
|
||||
).count() == len(outstanding_token_ids)
|
||||
|
||||
old_access_response = client.get(reverse("user-me"), headers=auth_headers)
|
||||
assert old_access_response.status_code == 401
|
||||
|
||||
old_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "tokens-refresh",
|
||||
"attributes": {"refresh": refresh_token},
|
||||
}
|
||||
},
|
||||
format="vnd.api+json",
|
||||
)
|
||||
assert old_refresh_response.status_code == 400
|
||||
|
||||
new_access_token, _ = get_api_tokens(client, create_test_user.email, new_password)
|
||||
new_access_response = client.get(
|
||||
reverse("user-me"), headers=get_authorization_header(new_access_token)
|
||||
)
|
||||
assert new_access_response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_password_change_invalidates_rotated_refresh_token(
|
||||
create_test_user, tenants_fixture
|
||||
):
|
||||
client = APIClient()
|
||||
new_password = "ChangedSecret123@"
|
||||
|
||||
access_token, refresh_token = get_api_tokens(
|
||||
client, create_test_user.email, TEST_PASSWORD
|
||||
)
|
||||
rotated_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "tokens-refresh",
|
||||
"attributes": {"refresh": refresh_token},
|
||||
}
|
||||
},
|
||||
format="vnd.api+json",
|
||||
)
|
||||
assert rotated_refresh_response.status_code == 200
|
||||
rotated_refresh_token = rotated_refresh_response.json()["data"]["attributes"][
|
||||
"refresh"
|
||||
]
|
||||
|
||||
password_change_payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(create_test_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
}
|
||||
}
|
||||
password_change_response = client.patch(
|
||||
reverse("user-detail", kwargs={"pk": create_test_user.id}),
|
||||
data=json.dumps(password_change_payload),
|
||||
headers=get_authorization_header(access_token),
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
assert password_change_response.status_code == 200, password_change_response.json()
|
||||
|
||||
old_rotated_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "tokens-refresh",
|
||||
"attributes": {"refresh": rotated_refresh_token},
|
||||
}
|
||||
},
|
||||
format="vnd.api+json",
|
||||
)
|
||||
assert old_rotated_refresh_response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_user_me_when_inviting_users(create_test_user, tenants_fixture, roles_fixture):
|
||||
client = APIClient()
|
||||
@@ -189,6 +306,8 @@ def test_user_me_when_inviting_users(create_test_user, tenants_fixture, roles_fi
|
||||
class TestTokenSwitchTenant:
|
||||
def test_switch_tenant_with_valid_token(self, tenants_fixture, providers_fixture):
|
||||
client = APIClient()
|
||||
aws_provider = providers_fixture[0]
|
||||
assert aws_provider
|
||||
|
||||
test_user = "test_email@prowler.com"
|
||||
test_password = "Test_password1@"
|
||||
@@ -1403,6 +1522,8 @@ class TestAPIKeyMultiTenantWorkflows:
|
||||
Verifies RLS enforcement after authentication ensures tenant isolation.
|
||||
"""
|
||||
client = APIClient()
|
||||
aws_provider = providers_fixture[0]
|
||||
assert aws_provider
|
||||
|
||||
user1 = User.objects.create_user(
|
||||
name="tenant1_user",
|
||||
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
from allauth.socialaccount.models import SocialLogin
|
||||
from api.adapters import ProwlerSocialAccountAdapter
|
||||
from api.db_router import MainRouter
|
||||
from api.models import SAMLConfiguration
|
||||
from api.models import Invitation, Membership, SAMLConfiguration, Tenant
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
User = get_user_model()
|
||||
@@ -188,6 +188,44 @@ class TestProwlerSocialAccountAdapter:
|
||||
_, called_user = call_args[0]
|
||||
assert called_user.email == create_test_user.email
|
||||
|
||||
def test_save_user_social_with_invitation_joins_invited_tenant(
|
||||
self, rf, create_test_user, tenants_fixture
|
||||
):
|
||||
adapter = ProwlerSocialAccountAdapter()
|
||||
invited_tenant = tenants_fixture[2]
|
||||
invited_email = "frank-invited@example.com"
|
||||
invitation = Invitation.objects.create(
|
||||
tenant=invited_tenant,
|
||||
email=invited_email,
|
||||
inviter=create_test_user,
|
||||
)
|
||||
request = rf.post("/", data={"invitation_token": invitation.token})
|
||||
request.session = {}
|
||||
|
||||
sociallogin = MagicMock(spec=SocialLogin)
|
||||
sociallogin.provider = MagicMock()
|
||||
sociallogin.provider.id = "google"
|
||||
sociallogin.account = MagicMock()
|
||||
sociallogin.account.extra_data = {"name": "Frank"}
|
||||
|
||||
real_user = User.objects.create_user(
|
||||
name="Frank", email=invited_email, password="Secret123!"
|
||||
)
|
||||
tenants_before = Tenant.objects.count()
|
||||
|
||||
with patch("api.adapters.super") as mock_super:
|
||||
mock_super.return_value.save_user.return_value = real_user
|
||||
adapter.save_user(request, sociallogin)
|
||||
|
||||
invitation.refresh_from_db()
|
||||
assert invitation.state == Invitation.State.ACCEPTED
|
||||
assert Tenant.objects.count() == tenants_before
|
||||
assert Membership.objects.filter(
|
||||
user=real_user,
|
||||
tenant=invited_tenant,
|
||||
role=Membership.RoleChoices.MEMBER,
|
||||
).exists()
|
||||
|
||||
def test_save_user_saml_sets_session_flag(self, rf):
|
||||
adapter = ProwlerSocialAccountAdapter()
|
||||
request = rf.get("/")
|
||||
|
||||
@@ -103,20 +103,84 @@ class TestUserViewSet:
|
||||
assert response.json()["data"]["attributes"]["name"] == "Updated Name"
|
||||
|
||||
def test_partial_update_user_with_no_permissions(
|
||||
self, authenticated_client_no_permissions_rbac, create_test_user
|
||||
self, authenticated_client_no_permissions_rbac, create_test_user_rbac_limited
|
||||
):
|
||||
updated_data = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(create_test_user_rbac_limited.id),
|
||||
"attributes": {"name": "Updated Name"},
|
||||
}
|
||||
}
|
||||
response = authenticated_client_no_permissions_rbac.patch(
|
||||
reverse("user-detail", kwargs={"pk": create_test_user.id}),
|
||||
reverse("user-detail", kwargs={"pk": create_test_user_rbac_limited.id}),
|
||||
data=updated_data,
|
||||
format="vnd.api+json",
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json()["data"]["attributes"]["name"] == "Updated Name"
|
||||
|
||||
def test_partial_update_other_user_with_no_permissions_denied(
|
||||
self, authenticated_client_no_permissions_rbac, tenants_fixture
|
||||
):
|
||||
original_email = "target-rbac-update@example.com"
|
||||
original_password = "OriginalPassword123@"
|
||||
target_user = User.objects.create_user(
|
||||
name="target_rbac_update",
|
||||
email=original_email,
|
||||
password=original_password,
|
||||
)
|
||||
Membership.objects.create(user=target_user, tenant=tenants_fixture[0])
|
||||
updated_data = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(target_user.id),
|
||||
"attributes": {
|
||||
"email": "updated-target-rbac@example.com",
|
||||
"password": "UpdatedPassword123@",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
response = authenticated_client_no_permissions_rbac.patch(
|
||||
reverse("user-detail", kwargs={"pk": target_user.id}),
|
||||
data=updated_data,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
target_user.refresh_from_db()
|
||||
assert target_user.email == original_email
|
||||
assert target_user.check_password(original_password)
|
||||
|
||||
def test_partial_update_other_user_with_manage_users_allowed(
|
||||
self, authenticated_client_rbac_manage_users_only
|
||||
):
|
||||
user = authenticated_client_rbac_manage_users_only.user
|
||||
tenant = Membership.objects.filter(user=user).first().tenant
|
||||
target_user = User.objects.create_user(
|
||||
name="target_manage_users_update",
|
||||
email="target-manage-users-update@example.com",
|
||||
password="Password123@",
|
||||
)
|
||||
Membership.objects.create(user=target_user, tenant=tenant)
|
||||
updated_data = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(target_user.id),
|
||||
"attributes": {"name": "Updated Target Name"},
|
||||
}
|
||||
}
|
||||
|
||||
response = authenticated_client_rbac_manage_users_only.patch(
|
||||
reverse("user-detail", kwargs={"pk": target_user.id}),
|
||||
data=updated_data,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
target_user.refresh_from_db()
|
||||
assert target_user.name == "Updated Target Name"
|
||||
|
||||
def test_delete_user_with_all_permissions(
|
||||
self, authenticated_client_rbac, create_test_user_rbac
|
||||
@@ -540,9 +604,7 @@ class TestLimitedVisibility:
|
||||
TEST_PASSWORD = "Thisisapassword123@"
|
||||
|
||||
@pytest.fixture
|
||||
def limited_admin_user(
|
||||
self, django_db_setup, django_db_blocker, tenants_fixture, providers_fixture
|
||||
):
|
||||
def limited_admin_user(self, django_db_blocker, tenants_fixture, providers_fixture):
|
||||
with django_db_blocker.unblock():
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
@@ -626,10 +688,10 @@ class TestLimitedVisibility:
|
||||
response.json()["data"]["relationships"]["providers"]["meta"]["count"] == 1
|
||||
)
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overviews_providers(
|
||||
self,
|
||||
authenticated_client_rbac_limited,
|
||||
scan_summaries_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
# By default, the associated provider is the one which has the overview data
|
||||
@@ -648,6 +710,7 @@ class TestLimitedVisibility:
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert len(response.json()["data"]) == 0
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_name",
|
||||
[
|
||||
@@ -659,7 +722,6 @@ class TestLimitedVisibility:
|
||||
self,
|
||||
endpoint_name,
|
||||
authenticated_client_rbac_limited,
|
||||
scan_summaries_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
# By default, the associated provider is the one which has the overview data
|
||||
@@ -684,10 +746,10 @@ class TestLimitedVisibility:
|
||||
data = response.json()["data"]["attributes"].values()
|
||||
assert all(value == 0 for value in data)
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overviews_services(
|
||||
self,
|
||||
authenticated_client_rbac_limited,
|
||||
scan_summaries_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
# By default, the associated provider is the one which has the overview data
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from neo4j.exceptions import ServiceUnavailable
|
||||
|
||||
|
||||
class TestRetryableSession:
|
||||
@patch("api.attack_paths.retryable_session.time.sleep")
|
||||
@patch("api.attack_paths.retryable_session.random.uniform", return_value=3.0)
|
||||
def test_custom_retry_uses_backoff_and_a_fresh_session(
|
||||
self, mock_uniform, mock_sleep
|
||||
):
|
||||
retryable_error = RuntimeError("retryable")
|
||||
first_session = MagicMock()
|
||||
first_session.execute_write.side_effect = retryable_error
|
||||
second_session = MagicMock()
|
||||
second_session.execute_write.return_value = "success"
|
||||
session_factory = MagicMock(side_effect=[first_session, second_session])
|
||||
work = MagicMock()
|
||||
|
||||
session = RetryableSession(
|
||||
session_factory=session_factory,
|
||||
max_retries=3,
|
||||
retry_if=lambda exc: exc is retryable_error,
|
||||
initial_retry_delay_seconds=2,
|
||||
)
|
||||
|
||||
assert session.execute_write(work) == "success"
|
||||
assert session_factory.call_count == 2
|
||||
first_session.close.assert_called_once_with()
|
||||
mock_uniform.assert_called_once_with(2.0, 4.0)
|
||||
mock_sleep.assert_called_once_with(3.0)
|
||||
|
||||
def test_connection_errors_remain_retryable(self):
|
||||
first_session = MagicMock()
|
||||
first_session.run.side_effect = ServiceUnavailable("unavailable")
|
||||
second_session = MagicMock()
|
||||
second_session.run.return_value = "success"
|
||||
session_factory = MagicMock(side_effect=[first_session, second_session])
|
||||
|
||||
session = RetryableSession(session_factory=session_factory, max_retries=1)
|
||||
|
||||
assert session.run("RETURN 1") == "success"
|
||||
first_session.close.assert_called_once_with()
|
||||
|
||||
def test_non_retryable_error_is_raised_without_refreshing_session(self):
|
||||
error = RuntimeError("do not retry")
|
||||
driver_session = MagicMock()
|
||||
driver_session.execute_write.side_effect = error
|
||||
session_factory = MagicMock(return_value=driver_session)
|
||||
session = RetryableSession(
|
||||
session_factory=session_factory,
|
||||
max_retries=3,
|
||||
retry_if=lambda _: False,
|
||||
initial_retry_delay_seconds=2,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
session.execute_write(MagicMock())
|
||||
|
||||
assert exc_info.value is error
|
||||
session_factory.assert_called_once_with()
|
||||
driver_session.close.assert_not_called()
|
||||
|
||||
def test_retry_exhaustion_raises_the_last_error(self):
|
||||
error = RuntimeError("still retryable")
|
||||
driver_sessions = [MagicMock() for _ in range(3)]
|
||||
for driver_session in driver_sessions:
|
||||
driver_session.execute_write.side_effect = error
|
||||
session_factory = MagicMock(side_effect=driver_sessions)
|
||||
session = RetryableSession(
|
||||
session_factory=session_factory,
|
||||
max_retries=2,
|
||||
retry_if=lambda _: True,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
session.execute_write(MagicMock())
|
||||
|
||||
assert exc_info.value is error
|
||||
assert session_factory.call_count == 3
|
||||
driver_sessions[0].close.assert_called_once_with()
|
||||
driver_sessions[1].close.assert_called_once_with()
|
||||
driver_sessions[2].close.assert_not_called()
|
||||
@@ -1,6 +1,6 @@
|
||||
import pytest
|
||||
from api.v1.serializer_utils.integrations import S3ConfigSerializer
|
||||
from api.v1.serializers import ImageProviderSecret
|
||||
from api.v1.serializers import ImageProviderSecret, KubernetesProviderSecret
|
||||
from rest_framework.exceptions import ValidationError
|
||||
|
||||
|
||||
@@ -132,3 +132,74 @@ class TestImageProviderSecret:
|
||||
serializer = ImageProviderSecret(data={"registry_password": "pass"})
|
||||
assert not serializer.is_valid()
|
||||
assert "non_field_errors" in serializer.errors
|
||||
|
||||
|
||||
class TestKubernetesProviderSecret:
|
||||
def test_valid_static_kubeconfig_is_accepted(self):
|
||||
kubeconfig_content = """
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: test-cluster
|
||||
cluster:
|
||||
server: https://kubernetes.example.test
|
||||
users:
|
||||
- name: test-user
|
||||
user:
|
||||
token: test-token
|
||||
contexts:
|
||||
- name: test-context
|
||||
context:
|
||||
cluster: test-cluster
|
||||
user: test-user
|
||||
current-context: test-context
|
||||
"""
|
||||
|
||||
serializer = KubernetesProviderSecret(
|
||||
data={"kubeconfig_content": kubeconfig_content}
|
||||
)
|
||||
|
||||
assert serializer.is_valid()
|
||||
|
||||
def test_kubeconfig_with_exec_authentication_is_rejected(self):
|
||||
kubeconfig_content = """
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: test-cluster
|
||||
cluster:
|
||||
server: https://kubernetes.example.test
|
||||
users:
|
||||
- name: test-user
|
||||
user:
|
||||
exec:
|
||||
apiVersion: client.authentication.k8s.io/v1
|
||||
command: kubectl
|
||||
contexts:
|
||||
- name: test-context
|
||||
context:
|
||||
cluster: test-cluster
|
||||
user: test-user
|
||||
current-context: test-context
|
||||
"""
|
||||
|
||||
serializer = KubernetesProviderSecret(
|
||||
data={"kubeconfig_content": kubeconfig_content}
|
||||
)
|
||||
|
||||
assert not serializer.is_valid()
|
||||
assert "kubeconfig_content" in serializer.errors
|
||||
|
||||
def test_malformed_kubeconfig_is_rejected(self):
|
||||
serializer = KubernetesProviderSecret(
|
||||
data={"kubeconfig_content": "apiVersion: ["}
|
||||
)
|
||||
|
||||
assert not serializer.is_valid()
|
||||
assert "kubeconfig_content" in serializer.errors
|
||||
|
||||
def test_non_mapping_kubeconfig_is_rejected(self):
|
||||
serializer = KubernetesProviderSecret(data={"kubeconfig_content": "[]"})
|
||||
|
||||
assert not serializer.is_valid()
|
||||
assert "kubeconfig_content" in serializer.errors
|
||||
|
||||
@@ -6,18 +6,20 @@ builds dual writer/reader Bolt drivers.
|
||||
"""
|
||||
|
||||
import json
|
||||
from importlib import import_module
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import neo4j
|
||||
import pytest
|
||||
|
||||
# Prime patch-target resolution. `api.attack_paths.sink/__init__.py` doesn't
|
||||
# eagerly import these submodules (they're loaded on demand inside the
|
||||
# factory), so `mock.patch("api.attack_paths.sink.<sub>.…")` would fail with
|
||||
# AttributeError on first call. Importing here registers them as attributes
|
||||
# of the package before any decorator runs.
|
||||
import_module("api.attack_paths.sink.neo4j")
|
||||
import_module("api.attack_paths.sink.neptune")
|
||||
from api.attack_paths import sink as sink_module
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.attack_paths.sink import factory
|
||||
from api.attack_paths.sink.neo4j import DATABASE_NOT_FOUND_CODE, Neo4jSink
|
||||
from api.attack_paths.sink.neptune import (
|
||||
NEPTUNE_WRITE_RETRY_DELAY_SECONDS,
|
||||
NeptuneSink,
|
||||
_is_retryable_write_error,
|
||||
_NeptuneAuthToken,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -26,8 +28,6 @@ def reset_sink_state():
|
||||
|
||||
The cache lives in `api.attack_paths.sink.factory`, not on the package.
|
||||
"""
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
original_backend = factory._backend
|
||||
original_secondary = dict(factory._secondary_backends)
|
||||
factory._backend = None
|
||||
@@ -40,29 +40,20 @@ def reset_sink_state():
|
||||
|
||||
class TestSinkFactory:
|
||||
def test_default_resolves_to_neo4j(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neo4j"
|
||||
assert factory._resolve_setting() == "neo4j"
|
||||
|
||||
def test_neptune_resolves_correctly(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neptune"
|
||||
assert factory._resolve_setting() == "neptune"
|
||||
|
||||
def test_invalid_value_raises(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "foo"
|
||||
with pytest.raises(RuntimeError, match="ATTACK_PATHS_SINK_DATABASE"):
|
||||
factory._resolve_setting()
|
||||
|
||||
@patch("api.attack_paths.sink.neo4j.neo4j.GraphDatabase.driver")
|
||||
def test_init_builds_neo4j_backend_by_default(self, mock_driver, settings):
|
||||
from api.attack_paths import sink as sink_module
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neo4j"
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
@@ -85,9 +76,6 @@ class TestSinkFactory:
|
||||
def test_init_builds_neptune_backend(
|
||||
self, mock_driver, mock_auth_provider, settings
|
||||
):
|
||||
from api.attack_paths import sink as sink_module
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neptune"
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
@@ -116,8 +104,6 @@ class TestSinkFactory:
|
||||
def test_neptune_reader_falls_back_to_writer(
|
||||
self, mock_driver, mock_auth_provider, settings
|
||||
):
|
||||
from api.attack_paths import sink as sink_module
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neptune"
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
@@ -144,8 +130,6 @@ class TestGetBackendForScan:
|
||||
def test_legacy_scan_in_neo4j_process_uses_active_backend(
|
||||
self, mock_driver, settings
|
||||
):
|
||||
from api.attack_paths import sink as sink_module
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neo4j"
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
@@ -164,8 +148,6 @@ class TestGetBackendForScan:
|
||||
assert backend is sink_module.get_backend()
|
||||
|
||||
def test_neptune_scan_on_neo4j_process_uses_neptune_secondary(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neo4j"
|
||||
active_neo4j = MagicMock(name="neo4j-active")
|
||||
factory._backend = active_neo4j
|
||||
@@ -190,6 +172,29 @@ def _count_result(key: str, count: int) -> MagicMock:
|
||||
return MagicMock(single=MagicMock(return_value={key: count}))
|
||||
|
||||
|
||||
def _run_managed_write(session: MagicMock) -> MagicMock:
|
||||
transaction = MagicMock()
|
||||
session.execute_write.call_args.args[0](transaction)
|
||||
return transaction
|
||||
|
||||
|
||||
def _managed_write_session(
|
||||
results: list[MagicMock],
|
||||
) -> tuple[MagicMock, list[MagicMock]]:
|
||||
session = MagicMock()
|
||||
transactions: list[MagicMock] = []
|
||||
result_iter = iter(results)
|
||||
|
||||
def execute_write(work):
|
||||
transaction = MagicMock()
|
||||
transaction.run.return_value = next(result_iter)
|
||||
transactions.append(transaction)
|
||||
return work(transaction)
|
||||
|
||||
session.execute_write.side_effect = execute_write
|
||||
return session, transactions
|
||||
|
||||
|
||||
def _directed_drop_results(
|
||||
outgoing_rels: int,
|
||||
incoming_rels: int,
|
||||
@@ -207,31 +212,26 @@ def _directed_drop_results(
|
||||
|
||||
class TestNeo4jSinkSyncWrites:
|
||||
def test_ensure_sync_indexes_runs_create_index_idempotent(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.return_value = MagicMock()
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
sink.ensure_sync_indexes("db-tenant-x")
|
||||
|
||||
query = session.run.call_args.args[0]
|
||||
transaction = _run_managed_write(session)
|
||||
query = transaction.run.call_args.args[0]
|
||||
assert "CREATE INDEX" in query
|
||||
assert "IF NOT EXISTS" in query
|
||||
assert "`_ProviderResource`" in query
|
||||
assert "`_provider_element_id`" in query
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
def test_write_nodes_skips_empty_batch(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
with patch.object(sink, "get_session") as get_session:
|
||||
sink.write_nodes("db-tenant-x", "`AWSUser`", [])
|
||||
get_session.assert_not_called()
|
||||
|
||||
def test_write_nodes_merges_on_provider_resource_label(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
@@ -241,15 +241,15 @@ class TestNeo4jSinkSyncWrites:
|
||||
[{"provider_element_id": "p:e", "props": {"k": "v"}}],
|
||||
)
|
||||
|
||||
query, params = session.run.call_args.args
|
||||
transaction = _run_managed_write(session)
|
||||
query, params = transaction.run.call_args.args
|
||||
assert "MERGE (n:`_ProviderResource`" in query
|
||||
assert "`_provider_element_id`: row.provider_element_id" in query
|
||||
assert "SET n:`AWSUser`:`_ProviderResource`" in query
|
||||
assert params == {"rows": [{"provider_element_id": "p:e", "props": {"k": "v"}}]}
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
def test_write_relationships_scopes_endpoints_by_provider_label(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
provider_id = "00000000-0000-0000-0000-000000000abc"
|
||||
@@ -268,24 +268,22 @@ class TestNeo4jSinkSyncWrites:
|
||||
],
|
||||
)
|
||||
|
||||
query = session.run.call_args.args[0]
|
||||
transaction = _run_managed_write(session)
|
||||
query = transaction.run.call_args.args[0]
|
||||
assert ":`_Provider_00000000000000000000000000000abc`" in query
|
||||
assert ":RESOURCE" in query.replace("`", "")
|
||||
assert "MERGE (s)-[r:`RESOURCE`" in query
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
|
||||
class TestNeptuneSinkSyncWrites:
|
||||
def test_ensure_sync_indexes_is_noop(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
with patch.object(sink, "get_session") as get_session:
|
||||
sink.ensure_sync_indexes("ignored")
|
||||
get_session.assert_not_called()
|
||||
|
||||
def test_write_nodes_merges_on_neptune_id_with_provider_resource_label(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
@@ -295,16 +293,16 @@ class TestNeptuneSinkSyncWrites:
|
||||
[{"provider_element_id": "p:e", "props": {"k": "v"}}],
|
||||
)
|
||||
|
||||
query = session.run.call_args.args[0]
|
||||
transaction = _run_managed_write(session)
|
||||
query = transaction.run.call_args.args[0]
|
||||
# Neptune assigns a default `vertex` label to any unlabeled node,
|
||||
# so the MERGE must pin a real label at creation time.
|
||||
assert "MERGE (n:`_ProviderResource` {`~id`: row.provider_element_id})" in query
|
||||
assert "SET n:`AWSUser`" in query
|
||||
assert "SET n.`_provider_element_id` = row.provider_element_id" in query
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
def test_write_relationships_matches_endpoints_by_id(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
@@ -322,30 +320,86 @@ class TestNeptuneSinkSyncWrites:
|
||||
],
|
||||
)
|
||||
|
||||
query = session.run.call_args.args[0]
|
||||
transaction = _run_managed_write(session)
|
||||
query = transaction.run.call_args.args[0]
|
||||
assert "MATCH (s) WHERE id(s) = row.start_element_id" in query
|
||||
assert "MATCH (e) WHERE id(e) = row.end_element_id" in query
|
||||
assert "MERGE (s)-[r:`RESOURCE`" in query
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
|
||||
class TestNeptuneRetryPolicy:
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
"Operation failed due to conflicting concurrent operations "
|
||||
+ "(please retry), 0 transactions are currently rolling back.",
|
||||
"Operation terminated (deadline exceeded)",
|
||||
],
|
||||
)
|
||||
def test_observed_transient_write_errors_are_retryable(self, message):
|
||||
error = MagicMock(spec=neo4j.exceptions.Neo4jError)
|
||||
error.message = message
|
||||
|
||||
assert _is_retryable_write_error(error) is True
|
||||
|
||||
def test_unrelated_database_error_is_not_retryable(self):
|
||||
error = MagicMock(spec=neo4j.exceptions.Neo4jError)
|
||||
error.message = "Operation terminated (out of memory)"
|
||||
|
||||
assert _is_retryable_write_error(error) is False
|
||||
|
||||
def test_non_neo4j_error_is_not_retryable(self):
|
||||
error = RuntimeError(
|
||||
"Operation failed due to conflicting concurrent operations"
|
||||
)
|
||||
|
||||
assert _is_retryable_write_error(error) is False
|
||||
|
||||
@patch("api.attack_paths.sink.neptune.RetryableSession")
|
||||
def test_writer_session_enables_neptune_retry_policy(self, retryable_session):
|
||||
sink = NeptuneSink()
|
||||
driver = MagicMock()
|
||||
with patch.object(sink, "_get_writer", return_value=driver):
|
||||
with sink.get_session():
|
||||
pass
|
||||
|
||||
kwargs = retryable_session.call_args.kwargs
|
||||
assert kwargs["retry_if"] is _is_retryable_write_error
|
||||
assert (
|
||||
kwargs["initial_retry_delay_seconds"] == NEPTUNE_WRITE_RETRY_DELAY_SECONDS
|
||||
)
|
||||
|
||||
@patch("api.attack_paths.sink.neptune.RetryableSession")
|
||||
def test_reader_session_does_not_enable_write_retry_policy(self, retryable_session):
|
||||
sink = NeptuneSink()
|
||||
driver = MagicMock()
|
||||
with patch.object(sink, "_get_reader", return_value=driver):
|
||||
with sink.get_session(default_access_mode=neo4j.READ_ACCESS):
|
||||
pass
|
||||
|
||||
kwargs = retryable_session.call_args.kwargs
|
||||
assert kwargs["retry_if"] is None
|
||||
assert kwargs["initial_retry_delay_seconds"] == 0
|
||||
|
||||
|
||||
class TestNeptuneSinkDropSubgraph:
|
||||
def test_drop_subgraph_deletes_directed_rels_before_nodes_in_bounded_batches(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
session.run.side_effect = _directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
session, transactions = _managed_write_session(
|
||||
_directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
)
|
||||
)
|
||||
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
deleted = sink.drop_subgraph("ignored", "provider-1")
|
||||
|
||||
assert deleted == 10
|
||||
assert session.run.call_count == 6
|
||||
queries = [call.args[0] for call in session.run.call_args_list]
|
||||
assert session.execute_write.call_count == 6
|
||||
queries = [transaction.run.call_args.args[0] for transaction in transactions]
|
||||
|
||||
assert ")-[r]->()" in queries[0]
|
||||
assert ")<-[r]-()" in queries[2]
|
||||
@@ -362,14 +416,13 @@ class TestNeo4jSinkDropSubgraph:
|
||||
"""Neo4j drop deletes relationships then nodes in batches (no ``DETACH DELETE``)."""
|
||||
|
||||
def test_drop_subgraph_deletes_directed_rels_before_nodes_in_bounded_batches(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.side_effect = _directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
session, transactions = _managed_write_session(
|
||||
_directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
)
|
||||
)
|
||||
|
||||
provider_id = "00000000-0000-0000-0000-000000000abc"
|
||||
@@ -378,9 +431,9 @@ class TestNeo4jSinkDropSubgraph:
|
||||
|
||||
# Only phase-2 node counts contribute to the return value.
|
||||
assert deleted == 10
|
||||
assert session.run.call_count == 6
|
||||
assert session.execute_write.call_count == 6
|
||||
|
||||
queries = [call.args[0] for call in session.run.call_args_list]
|
||||
queries = [transaction.run.call_args.args[0] for transaction in transactions]
|
||||
# Regression guard: the memory blow-up was caused by DETACH DELETE.
|
||||
assert all("DETACH DELETE" not in query for query in queries)
|
||||
assert all("DISTINCT r" not in query for query in queries)
|
||||
@@ -399,12 +452,9 @@ class TestNeo4jSinkDropSubgraph:
|
||||
assert last_rel < first_node
|
||||
|
||||
def test_drop_subgraph_returns_zero_when_database_does_not_exist(self):
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.attack_paths.sink.neo4j import DATABASE_NOT_FOUND_CODE, Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.side_effect = GraphDatabaseQueryException(
|
||||
session.execute_write.side_effect = GraphDatabaseQueryException(
|
||||
message="db missing", code=DATABASE_NOT_FOUND_CODE
|
||||
)
|
||||
|
||||
@@ -418,8 +468,6 @@ class TestSinkHasProviderData:
|
||||
"""``has_provider_data`` is the read-path probe used by API views."""
|
||||
|
||||
def test_neo4j_returns_true_when_provider_node_exists(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.return_value.single.return_value = MagicMock()
|
||||
@@ -433,9 +481,6 @@ class TestSinkHasProviderData:
|
||||
assert ":`_Provider_00000000000000000000000000000abc`" in query
|
||||
|
||||
def test_neo4j_returns_false_when_database_does_not_exist(self):
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.attack_paths.sink.neo4j import DATABASE_NOT_FOUND_CODE, Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.side_effect = GraphDatabaseQueryException(
|
||||
@@ -448,8 +493,6 @@ class TestSinkHasProviderData:
|
||||
assert present is False
|
||||
|
||||
def test_neptune_returns_true_when_provider_node_exists(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
session.run.return_value.single.return_value = MagicMock()
|
||||
@@ -463,8 +506,6 @@ class TestGetBackendForScanCutover:
|
||||
"""``get_backend_for_scan`` keeps old-sink scans queryable after cutover."""
|
||||
|
||||
def test_legacy_scan_on_neptune_process_uses_neo4j_secondary(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neptune"
|
||||
active_neptune = MagicMock(name="neptune-active")
|
||||
factory._backend = active_neptune
|
||||
@@ -487,8 +528,6 @@ class TestSinkVerifyConnectivity:
|
||||
|
||||
@patch("api.attack_paths.sink.neo4j.neo4j.GraphDatabase.driver")
|
||||
def test_neo4j_verifies_its_driver(self, mock_driver, settings):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
"neo4j": {
|
||||
@@ -513,8 +552,6 @@ class TestSinkVerifyConnectivity:
|
||||
def test_neptune_verifies_reader_not_writer(
|
||||
self, mock_driver, mock_auth_provider, settings
|
||||
):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
"neptune": {
|
||||
@@ -548,8 +585,6 @@ class TestSinkInitToleratesUnreachableSink:
|
||||
|
||||
@patch("api.attack_paths.sink.neo4j.neo4j.GraphDatabase.driver")
|
||||
def test_neo4j_init_continues_when_verify_fails(self, mock_driver, settings):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
"neo4j": {
|
||||
@@ -573,8 +608,6 @@ class TestSinkInitToleratesUnreachableSink:
|
||||
def test_neptune_init_continues_when_verify_fails(
|
||||
self, mock_driver, mock_auth_provider, settings
|
||||
):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
"neptune": {
|
||||
@@ -601,8 +634,6 @@ class TestNeptuneAdminNoOps:
|
||||
|
||||
@pytest.mark.parametrize("method", ["create_database", "drop_database"])
|
||||
def test_admin_ops_return_none_without_touching_a_session(self, method):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
with patch.object(sink, "get_session") as get_session:
|
||||
assert getattr(sink, method)("ignored") is None
|
||||
@@ -617,8 +648,6 @@ class TestNeptuneAuthToken:
|
||||
def test_host_header_includes_non_default_port(self, mock_boto, mock_sigv4):
|
||||
# Neptune runs on 8182; the SigV4 canonical Host must keep the port or
|
||||
# the signature is rejected.
|
||||
from api.attack_paths.sink.neptune import _NeptuneAuthToken
|
||||
|
||||
credentials = MagicMock()
|
||||
credentials.get_frozen_credentials.return_value = MagicMock()
|
||||
mock_boto.return_value.get_credentials.return_value = credentials
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
from api.validators import (
|
||||
resolve_lighthouse_openai_compatible_host,
|
||||
validate_lighthouse_openai_compatible_base_url,
|
||||
)
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.test import override_settings
|
||||
|
||||
|
||||
def test_lighthouse_base_url_rejects_http_scheme():
|
||||
with pytest.raises(ValidationError, match="HTTPS"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"http://openrouter.ai/api/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://openrouter.ai:0/api/v1",
|
||||
"https://openrouter.ai:-1/api/v1",
|
||||
"https://openrouter.ai:65536/api/v1",
|
||||
"https://openrouter.ai:invalid/api/v1",
|
||||
],
|
||||
)
|
||||
def test_lighthouse_base_url_rejects_invalid_port(base_url):
|
||||
with pytest.raises(ValidationError, match="port is invalid"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
base_url,
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("port", [1, 65535])
|
||||
def test_lighthouse_base_url_accepts_valid_port_boundaries(port):
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
f"https://openrouter.ai:{port}/api/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_rejects_localhost():
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://localhost/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ip_address", ["10.0.0.1", "172.16.0.1", "192.168.1.1"])
|
||||
def test_lighthouse_base_url_rejects_private_ip_literal(ip_address):
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
f"https://{ip_address}/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_rejects_metadata_ip_literal():
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://169.254.169.254/latest/meta-data",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://[::ffff:169.254.169.254]/v1",
|
||||
"https://[64:ff9b::a9fe:a9fe]/v1",
|
||||
"https://[2002:a9fe:a9fe::]/v1",
|
||||
],
|
||||
)
|
||||
def test_lighthouse_base_url_rejects_embedded_non_global_ip(base_url):
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
base_url,
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://[::ffff:93.184.216.34]/v1",
|
||||
"https://[64:ff9b::5db8:d822]/v1",
|
||||
"https://[2002:5db8:d822::]/v1",
|
||||
],
|
||||
)
|
||||
def test_lighthouse_base_url_accepts_embedded_public_ip(base_url):
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
base_url,
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_accepts_hostname_without_dns_resolution():
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://openrouter.ai/api/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_rejects_post_dns_internal_address(monkeypatch):
|
||||
def resolve_to_metadata(*_args, **_kwargs):
|
||||
return [
|
||||
(
|
||||
socket.AF_INET,
|
||||
socket.SOCK_STREAM,
|
||||
6,
|
||||
"",
|
||||
("169.254.169.254", 443),
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr("api.validators.socket.getaddrinfo", resolve_to_metadata)
|
||||
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://metadata.example.test/v1"
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_accepts_public_resolved_address(monkeypatch):
|
||||
def resolve_to_public(*_args, **_kwargs):
|
||||
return [
|
||||
(
|
||||
socket.AF_INET,
|
||||
socket.SOCK_STREAM,
|
||||
6,
|
||||
"",
|
||||
("93.184.216.34", 443),
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr("api.validators.socket.getaddrinfo", resolve_to_public)
|
||||
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url("https://openrouter.ai/api/v1")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"]
|
||||
)
|
||||
def test_lighthouse_base_url_accepts_allowlisted_host_without_resolution(monkeypatch):
|
||||
def fail_resolution(*_args, **_kwargs):
|
||||
raise AssertionError("allowlisted hosts must not be resolved")
|
||||
|
||||
monkeypatch.setattr("api.validators.socket.getaddrinfo", fail_resolution)
|
||||
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://custom-openai.internal/v1"
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"]
|
||||
)
|
||||
def test_lighthouse_resolve_returns_allowlisted_hostname_unpinned():
|
||||
assert resolve_lighthouse_openai_compatible_host(
|
||||
"Custom-OpenAI.internal.", 443
|
||||
) == ("custom-openai.internal",)
|
||||
|
||||
|
||||
@override_settings(LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["localhost"])
|
||||
def test_lighthouse_base_url_accepts_allowlisted_blocked_host():
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://localhost/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["10.0.0.1"])
|
||||
def test_lighthouse_base_url_accepts_allowlisted_private_ip_literal():
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://10.0.0.1/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=[" Custom-OpenAI.Internal. "]
|
||||
)
|
||||
def test_lighthouse_allowlist_entries_are_normalized():
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://custom-openai.internal/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"]
|
||||
)
|
||||
def test_lighthouse_base_url_rejects_host_not_in_allowlist():
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://localhost/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@override_settings(LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=[""])
|
||||
def test_lighthouse_allowlist_ignores_empty_entries():
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://localhost/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"]
|
||||
)
|
||||
def test_lighthouse_base_url_allowlisted_host_still_requires_https():
|
||||
with pytest.raises(ValidationError, match="HTTPS"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"http://custom-openai.internal/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
@@ -59,8 +59,13 @@ from api.models import (
|
||||
from api.rls import Tenant
|
||||
from api.uuid_utils import datetime_to_uuid7
|
||||
from api.v1.serializers import TokenSerializer
|
||||
from api.v1.views import ComplianceOverviewViewSet, TenantFinishACSView
|
||||
from api.v1.views import (
|
||||
ComplianceOverviewViewSet,
|
||||
CustomSAMLLoginView,
|
||||
TenantFinishACSView,
|
||||
)
|
||||
from botocore.exceptions import ClientError, NoCredentialsError
|
||||
from celery import states
|
||||
from conftest import (
|
||||
API_JSON_CONTENT_TYPE,
|
||||
TEST_PASSWORD,
|
||||
@@ -244,6 +249,63 @@ class TestUserViewSet:
|
||||
create_test_user.refresh_from_db()
|
||||
assert create_test_user.company_name == new_company_name
|
||||
|
||||
def test_users_partial_update_same_tenant_other_user_password_denied(
|
||||
self, authenticated_client_no_permissions_rbac, tenants_fixture
|
||||
):
|
||||
original_password = "OriginalPassword123@"
|
||||
new_password = "UpdatedPassword123@"
|
||||
target_user = User.objects.create_user(
|
||||
password=original_password,
|
||||
email="target-password-update@example.com",
|
||||
)
|
||||
Membership.objects.create(user=target_user, tenant=tenants_fixture[0])
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(target_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
},
|
||||
}
|
||||
|
||||
response = authenticated_client_no_permissions_rbac.patch(
|
||||
reverse("user-detail", kwargs={"pk": target_user.id}),
|
||||
data=payload,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
target_user.refresh_from_db()
|
||||
assert target_user.check_password(original_password)
|
||||
assert not target_user.check_password(new_password)
|
||||
|
||||
def test_users_partial_update_same_tenant_other_user_email_denied(
|
||||
self, authenticated_client_no_permissions_rbac, tenants_fixture
|
||||
):
|
||||
original_email = "target-email-update@example.com"
|
||||
new_email = "updated-target-email@example.com"
|
||||
target_user = User.objects.create_user(
|
||||
password="OriginalPassword123@",
|
||||
email=original_email,
|
||||
)
|
||||
Membership.objects.create(user=target_user, tenant=tenants_fixture[0])
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(target_user.id),
|
||||
"attributes": {"email": new_email},
|
||||
},
|
||||
}
|
||||
|
||||
response = authenticated_client_no_permissions_rbac.patch(
|
||||
reverse("user-detail", kwargs={"pk": target_user.id}),
|
||||
data=payload,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
target_user.refresh_from_db()
|
||||
assert target_user.email == original_email
|
||||
|
||||
def test_users_partial_update_invalid_content_type(
|
||||
self, authenticated_client, create_test_user
|
||||
):
|
||||
@@ -1491,13 +1553,13 @@ class TestProviderViewSet:
|
||||
("provider_groups", ["provider-groups"]),
|
||||
],
|
||||
)
|
||||
@pytest.mark.usefixtures("create_provider_group_relationship")
|
||||
def test_providers_list_include(
|
||||
self,
|
||||
include_values,
|
||||
expected_resources,
|
||||
authenticated_client,
|
||||
providers_fixture,
|
||||
create_provider_group_relationship,
|
||||
):
|
||||
response = authenticated_client.get(
|
||||
reverse("provider-list"), {"include": include_values}
|
||||
@@ -2898,7 +2960,24 @@ class TestProviderSecretViewSet:
|
||||
Provider.ProviderChoices.KUBERNETES.value,
|
||||
ProviderSecret.TypeChoices.STATIC,
|
||||
{
|
||||
"kubeconfig_content": "kubeconfig-content",
|
||||
"kubeconfig_content": """
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: test-cluster
|
||||
cluster:
|
||||
server: https://kubernetes.example.test
|
||||
users:
|
||||
- name: test-user
|
||||
user:
|
||||
token: test-token
|
||||
contexts:
|
||||
- name: test-context
|
||||
context:
|
||||
cluster: test-cluster
|
||||
user: test-user
|
||||
current-context: test-context
|
||||
""",
|
||||
},
|
||||
),
|
||||
# M365 client secret credentials
|
||||
@@ -3542,7 +3621,7 @@ class TestScanViewSet:
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scan_json_payload, expected_scanner_args",
|
||||
"scan_json_payload, _expected_scanner_args",
|
||||
[
|
||||
# Case 1: No scanner_args in payload (should use provider's scanner_args)
|
||||
(
|
||||
@@ -3583,21 +3662,15 @@ class TestScanViewSet:
|
||||
),
|
||||
],
|
||||
)
|
||||
@patch("api.v1.views.Task.objects.get")
|
||||
@patch("api.v1.views.perform_scan_task.apply_async")
|
||||
@patch("api.v1.views.enqueue_scan_execution_on_commit")
|
||||
def test_scans_create_valid(
|
||||
self,
|
||||
mock_perform_scan_task,
|
||||
mock_task_get,
|
||||
mock_enqueue_scan_execution,
|
||||
authenticated_client,
|
||||
scan_json_payload,
|
||||
expected_scanner_args,
|
||||
_expected_scanner_args,
|
||||
providers_fixture,
|
||||
tasks_fixture,
|
||||
):
|
||||
prowler_task = tasks_fixture[0]
|
||||
mock_perform_scan_task.return_value.id = prowler_task.id
|
||||
mock_task_get.return_value = prowler_task
|
||||
*_, provider5 = providers_fixture
|
||||
# Provider5 has these scanner_args
|
||||
# scanner_args={"key1": "value1", "key2": {"key21": "value21"}}
|
||||
@@ -3622,8 +3695,121 @@ class TestScanViewSet:
|
||||
assert scan.name == scan_json_payload["data"]["attributes"]["name"]
|
||||
assert scan.provider == provider5
|
||||
assert scan.trigger == Scan.TriggerChoices.MANUAL
|
||||
mock_enqueue_scan_execution.assert_called_once()
|
||||
# assert scan.scanner_args == expected_scanner_args
|
||||
|
||||
@patch("tasks.tasks.perform_scan_task.apply_async")
|
||||
def test_scans_create_queues_scan_when_provider_has_active_scan(
|
||||
self,
|
||||
mock_perform_scan_task,
|
||||
authenticated_client,
|
||||
providers_fixture,
|
||||
tenants_fixture,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider, *_ = providers_fixture
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-perform",
|
||||
status=states.PENDING,
|
||||
)
|
||||
prowler_task = Task.objects.create(
|
||||
id=task_result.task_id,
|
||||
tenant_id=tenant.id,
|
||||
task_runner_task=task_result,
|
||||
)
|
||||
Scan.objects.create(
|
||||
name="Active scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
tenant_id=tenant.id,
|
||||
task=prowler_task,
|
||||
)
|
||||
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
response = authenticated_client.post(
|
||||
reverse("scan-list"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "scans",
|
||||
"attributes": {"name": "Duplicate Scan"},
|
||||
"relationships": {
|
||||
"provider": {
|
||||
"data": {"type": "providers", "id": str(provider.id)}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
assert response.json()["data"]["id"] != str(prowler_task.id)
|
||||
assert Scan.objects.count() == 2
|
||||
queued_scan = Scan.objects.exclude(task=prowler_task).get()
|
||||
assert queued_scan.trigger == Scan.TriggerChoices.MANUAL
|
||||
assert queued_scan.state == StateChoices.AVAILABLE
|
||||
assert queued_scan.task.task_runner_task.status == "QUEUED"
|
||||
mock_perform_scan_task.assert_not_called()
|
||||
|
||||
@patch("tasks.tasks.perform_scan_task.apply_async")
|
||||
def test_scans_create_queues_scan_when_scheduled_scan_is_claimed(
|
||||
self,
|
||||
mock_perform_scan_task,
|
||||
authenticated_client,
|
||||
providers_fixture,
|
||||
tenants_fixture,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
tenant, *_ = tenants_fixture
|
||||
provider, *_ = providers_fixture
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid4()),
|
||||
task_name="scan-perform-scheduled",
|
||||
status=states.STARTED,
|
||||
)
|
||||
prowler_task = Task.objects.create(
|
||||
id=task_result.task_id,
|
||||
tenant_id=tenant.id,
|
||||
task_runner_task=task_result,
|
||||
)
|
||||
Scan.objects.create(
|
||||
name="Claimed scheduled scan",
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
tenant_id=tenant.id,
|
||||
task=prowler_task,
|
||||
)
|
||||
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
response = authenticated_client.post(
|
||||
reverse("scan-list"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "scans",
|
||||
"attributes": {"name": "Manual Scan"},
|
||||
"relationships": {
|
||||
"provider": {
|
||||
"data": {"type": "providers", "id": str(provider.id)}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_202_ACCEPTED
|
||||
assert response.json()["data"]["id"] != str(prowler_task.id)
|
||||
assert Scan.objects.count() == 2
|
||||
queued_scan = Scan.objects.exclude(task=prowler_task).get()
|
||||
assert queued_scan.trigger == Scan.TriggerChoices.MANUAL
|
||||
assert queued_scan.state == StateChoices.AVAILABLE
|
||||
assert queued_scan.task.task_runner_task.status == "QUEUED"
|
||||
mock_perform_scan_task.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scan_json_payload, error_code",
|
||||
[
|
||||
@@ -4070,7 +4256,7 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
type("env", (), {"str": lambda self, *_args, **_kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
presigned_url = (
|
||||
@@ -4176,7 +4362,7 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.TaskSerializer",
|
||||
lambda *args, **kwargs: type("S", (), {"data": dummy}),
|
||||
lambda *_args, **_kwargs: type("S", (), {"data": dummy}),
|
||||
)
|
||||
|
||||
framework = get_compliance_frameworks(scan.provider.provider)[0]
|
||||
@@ -4234,7 +4420,7 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
type("env", (), {"str": lambda self, *_args, **_kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
match_key = "path/compliance/mitre_attack_aws.csv"
|
||||
@@ -4245,6 +4431,7 @@ class TestScanViewSet:
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
del Prefix
|
||||
return {"Contents": [{"Key": match_key}]}
|
||||
|
||||
def generate_presigned_url(self, ClientMethod, Params, ExpiresIn):
|
||||
@@ -4276,7 +4463,7 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
type("env", (), {"str": lambda self, *_args, **_kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
old_key = "path/compliance/prowler-output-aws-20240101000000_cis_1.4_aws.csv"
|
||||
@@ -4284,6 +4471,7 @@ class TestScanViewSet:
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
del Prefix
|
||||
return {
|
||||
"Contents": [
|
||||
{
|
||||
@@ -4357,11 +4545,12 @@ class TestScanViewSet:
|
||||
|
||||
monkeypatch.setattr(
|
||||
"api.v1.views.env",
|
||||
type("env", (), {"str": lambda self, *args, **kwargs: "test-bucket"})(),
|
||||
type("env", (), {"str": lambda self, *_args, **_kwargs: "test-bucket"})(),
|
||||
)
|
||||
|
||||
class FakeS3Client:
|
||||
def list_objects_v2(self, Bucket, Prefix):
|
||||
del Prefix
|
||||
return {"Contents": []}
|
||||
|
||||
def get_object(self, Bucket, Key):
|
||||
@@ -4547,7 +4736,7 @@ class TestScanViewSet:
|
||||
inserted_at=base + timedelta(hours=1)
|
||||
)
|
||||
|
||||
mock_task_serializer.side_effect = lambda instance, *a, **k: SimpleNamespace(
|
||||
mock_task_serializer.side_effect = lambda instance, *_a, **_k: SimpleNamespace(
|
||||
data={"id": str(instance.id), "state": StateChoices.EXECUTING}
|
||||
)
|
||||
|
||||
@@ -6279,9 +6468,8 @@ class TestResourceViewSet:
|
||||
)
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
def test_resources_metadata_retrieve(
|
||||
self, authenticated_client, resources_fixture, backfill_scan_metadata_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("backfill_scan_metadata_fixture")
|
||||
def test_resources_metadata_retrieve(self, authenticated_client, resources_fixture):
|
||||
resource_1, *_ = resources_fixture
|
||||
response = authenticated_client.get(
|
||||
reverse("resource-metadata"),
|
||||
@@ -6301,8 +6489,9 @@ class TestResourceViewSet:
|
||||
assert set(data["data"]["attributes"]["types"]) == expected_resource_types
|
||||
assert set(data["data"]["attributes"]["groups"]) == expected_groups
|
||||
|
||||
@pytest.mark.usefixtures("backfill_scan_metadata_fixture")
|
||||
def test_resources_metadata_resource_filter_retrieve(
|
||||
self, authenticated_client, resources_fixture, backfill_scan_metadata_fixture
|
||||
self, authenticated_client, resources_fixture
|
||||
):
|
||||
resource_1, *_ = resources_fixture
|
||||
response = authenticated_client.get(
|
||||
@@ -7796,9 +7985,8 @@ class TestFindingViewSet:
|
||||
)
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
def test_findings_metadata_retrieve(
|
||||
self, authenticated_client, findings_fixture, backfill_scan_metadata_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("backfill_scan_metadata_fixture")
|
||||
def test_findings_metadata_retrieve(self, authenticated_client, findings_fixture):
|
||||
finding_1, *_ = findings_fixture
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-metadata"),
|
||||
@@ -7821,8 +8009,9 @@ class TestFindingViewSet:
|
||||
)
|
||||
# assert data["data"]["attributes"]["tags"] == expected_tags
|
||||
|
||||
@pytest.mark.usefixtures("backfill_scan_metadata_fixture")
|
||||
def test_findings_metadata_resource_filter_retrieve(
|
||||
self, authenticated_client, findings_fixture, backfill_scan_metadata_fixture
|
||||
self, authenticated_client, findings_fixture
|
||||
):
|
||||
finding_1, *_ = findings_fixture
|
||||
response = authenticated_client.get(
|
||||
@@ -8008,9 +8197,8 @@ class TestFindingViewSet:
|
||||
attributes = response.json()["data"]["attributes"]
|
||||
assert set(attributes["categories"]) == {"gen-ai", "security"}
|
||||
|
||||
def test_findings_metadata_latest_categories(
|
||||
self, authenticated_client, latest_scan_finding_with_categories
|
||||
):
|
||||
@pytest.mark.usefixtures("latest_scan_finding_with_categories")
|
||||
def test_findings_metadata_latest_categories(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-metadata_latest"),
|
||||
)
|
||||
@@ -8018,9 +8206,8 @@ class TestFindingViewSet:
|
||||
attributes = response.json()["data"]["attributes"]
|
||||
assert set(attributes["categories"]) == {"gen-ai", "iam"}
|
||||
|
||||
def test_findings_metadata_latest_groups(
|
||||
self, authenticated_client, latest_scan_finding_with_categories
|
||||
):
|
||||
@pytest.mark.usefixtures("latest_scan_finding_with_categories")
|
||||
def test_findings_metadata_latest_groups(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("finding-metadata_latest"),
|
||||
)
|
||||
@@ -8540,16 +8727,14 @@ class TestInvitationViewSet:
|
||||
expires_at=self.TOMORROW,
|
||||
)
|
||||
|
||||
data = {
|
||||
"invitation_token": invitation.token,
|
||||
}
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
assert not Membership.objects.filter(
|
||||
user__email__iexact=user.email, tenant=tenant
|
||||
).exists()
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
invitation.refresh_from_db()
|
||||
@@ -8558,13 +8743,46 @@ class TestInvitationViewSet:
|
||||
).exists()
|
||||
assert invitation.state == Invitation.State.ACCEPTED.value
|
||||
|
||||
def test_invitations_accept_invitation_invalid_token(self, authenticated_client):
|
||||
data = {
|
||||
"invitation_token": "invalid_token",
|
||||
}
|
||||
def test_invitations_accept_invitation_existing_membership(
|
||||
self,
|
||||
authenticated_client,
|
||||
create_test_user,
|
||||
tenants_fixture,
|
||||
):
|
||||
*_, tenant = tenants_fixture
|
||||
user = create_test_user
|
||||
|
||||
invitation = Invitation.objects.create(
|
||||
tenant=tenant,
|
||||
email=TEST_USER,
|
||||
inviter=user,
|
||||
expires_at=self.TOMORROW,
|
||||
)
|
||||
Membership.objects.create(user=user, tenant=tenant)
|
||||
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"),
|
||||
data=data,
|
||||
format="vnd.api+json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
invitation.refresh_from_db()
|
||||
assert invitation.state == Invitation.State.ACCEPTED.value
|
||||
assert (
|
||||
Membership.objects.filter(
|
||||
user__email__iexact=user.email, tenant=tenant
|
||||
).count()
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_invitations_accept_invitation_invalid_token(self, authenticated_client):
|
||||
data = {"invitation_token": "invalid_token"}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
@@ -8578,12 +8796,10 @@ class TestInvitationViewSet:
|
||||
invitation.email = TEST_USER
|
||||
invitation.save()
|
||||
|
||||
data = {
|
||||
"invitation_token": invitation.token,
|
||||
}
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_410_GONE
|
||||
@@ -8619,12 +8835,10 @@ class TestInvitationViewSet:
|
||||
invitation.email = TEST_USER
|
||||
invitation.save()
|
||||
|
||||
data = {
|
||||
"invitation_token": invitation.token,
|
||||
}
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
@@ -8642,12 +8856,10 @@ class TestInvitationViewSet:
|
||||
invitation.email = TEST_USER
|
||||
invitation.save()
|
||||
|
||||
data = {
|
||||
"invitation_token": invitation.token,
|
||||
}
|
||||
data = {"invitation_token": invitation.token}
|
||||
|
||||
response = authenticated_client.post(
|
||||
reverse("invitation-accept"), data=data, format="json"
|
||||
reverse("invitation-accept"), data=data, format="vnd.api+json"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
@@ -10723,9 +10935,8 @@ class TestOverviewViewSet:
|
||||
response = authenticated_client.put(reverse("overview-list"))
|
||||
assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED
|
||||
|
||||
def test_overview_providers_list(
|
||||
self, authenticated_client, scan_summaries_fixture, resources_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_providers_list(self, authenticated_client, resources_fixture):
|
||||
response = authenticated_client.get(reverse("overview-providers"))
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert len(response.json()["data"]) == 1
|
||||
@@ -10736,10 +10947,10 @@ class TestOverviewViewSet:
|
||||
# Aggregated resources include all AWS providers present in the tenant
|
||||
assert response.json()["data"][0]["attributes"]["resources"]["total"] == 3
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_providers_aggregates_same_provider_type(
|
||||
self,
|
||||
authenticated_client,
|
||||
scan_summaries_fixture,
|
||||
resources_fixture,
|
||||
providers_fixture,
|
||||
tenants_fixture,
|
||||
@@ -10790,10 +11001,10 @@ class TestOverviewViewSet:
|
||||
assert attributes["findings"]["muted"] == 7
|
||||
assert attributes["resources"]["total"] == 4
|
||||
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_providers_count(
|
||||
self,
|
||||
authenticated_client,
|
||||
scan_summaries_fixture,
|
||||
resources_fixture,
|
||||
providers_fixture,
|
||||
tenants_fixture,
|
||||
@@ -11259,15 +11470,15 @@ class TestOverviewViewSet:
|
||||
assert data[0]["id"] == str(snapshot1.id)
|
||||
assert data[0]["attributes"]["overall_score"] == "55.55"
|
||||
|
||||
def test_overview_services_list_no_required_filters(
|
||||
self, authenticated_client, scan_summaries_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_services_list_no_required_filters(self, authenticated_client):
|
||||
response = authenticated_client.get(reverse("overview-services"))
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
# Should return services from latest scans
|
||||
assert len(response.json()["data"]) == 2
|
||||
|
||||
def test_overview_regions_list(self, authenticated_client, scan_summaries_fixture):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_regions_list(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("overview-regions"), {"filter[inserted_at]": TODAY}
|
||||
)
|
||||
@@ -11293,7 +11504,8 @@ class TestOverviewViewSet:
|
||||
assert regions["aws:region2"]["fail"] == 1
|
||||
assert regions["aws:region2"]["muted"] == 3
|
||||
|
||||
def test_overview_services_list(self, authenticated_client, scan_summaries_fixture):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_services_list(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("overview-services"), {"filter[inserted_at]": TODAY}
|
||||
)
|
||||
@@ -11941,9 +12153,8 @@ class TestOverviewViewSet:
|
||||
assert results_by_type["internet-exposed"]["total_findings"] == 10
|
||||
assert results_by_type["internet-exposed"]["failed_findings"] == 5
|
||||
|
||||
def test_overview_services_region_filter(
|
||||
self, authenticated_client, scan_summaries_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("scan_summaries_fixture")
|
||||
def test_overview_services_region_filter(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
reverse("overview-services"),
|
||||
{"filter[region]": "region1"},
|
||||
@@ -12011,7 +12222,7 @@ class TestOverviewViewSet:
|
||||
assert "gcp-service" not in service_ids
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status_filter,field_to_check",
|
||||
"status_filter,_field_to_check",
|
||||
[
|
||||
("FAIL", "fail"),
|
||||
("PASS", "_pass"),
|
||||
@@ -12023,7 +12234,7 @@ class TestOverviewViewSet:
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
status_filter,
|
||||
field_to_check,
|
||||
_field_to_check,
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
@@ -12663,8 +12874,9 @@ class TestOverviewViewSet:
|
||||
assert data[0]["attributes"]["new_failed_findings"] == 5
|
||||
assert data[0]["attributes"]["resources_count"] == 10
|
||||
|
||||
@pytest.mark.usefixtures("tenant_compliance_summary_fixture")
|
||||
def test_compliance_watchlist_no_filters_uses_tenant_summary(
|
||||
self, authenticated_client, tenant_compliance_summary_fixture
|
||||
self, authenticated_client
|
||||
):
|
||||
response = authenticated_client.get(reverse("overview-compliance-watchlist"))
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
@@ -12684,10 +12896,10 @@ class TestOverviewViewSet:
|
||||
assert by_id["gdpr_aws"]["requirements_failed"] == 0
|
||||
assert by_id["gdpr_aws"]["total_requirements"] == 7
|
||||
|
||||
@pytest.mark.usefixtures("provider_compliance_scores_fixture")
|
||||
def test_compliance_watchlist_with_provider_filter_uses_provider_scores(
|
||||
self,
|
||||
authenticated_client,
|
||||
provider_compliance_scores_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
provider1 = providers_fixture[0]
|
||||
@@ -12704,9 +12916,8 @@ class TestOverviewViewSet:
|
||||
assert by_id["aws_cis_2.0"]["requirements_manual"] == 1
|
||||
assert by_id["aws_cis_2.0"]["total_requirements"] == 3
|
||||
|
||||
def test_compliance_watchlist_fail_dominant_logic(
|
||||
self, authenticated_client, provider_compliance_scores_fixture
|
||||
):
|
||||
@pytest.mark.usefixtures("provider_compliance_scores_fixture")
|
||||
def test_compliance_watchlist_fail_dominant_logic(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
f"{reverse('overview-compliance-watchlist')}?filter[provider_type]=aws"
|
||||
)
|
||||
@@ -12721,10 +12932,10 @@ class TestOverviewViewSet:
|
||||
assert aws_cis["requirements_manual"] == 1
|
||||
assert aws_cis["total_requirements"] == 3
|
||||
|
||||
@pytest.mark.usefixtures("provider_compliance_scores_fixture")
|
||||
def test_compliance_watchlist_provider_id_in_filter(
|
||||
self,
|
||||
authenticated_client,
|
||||
provider_compliance_scores_fixture,
|
||||
providers_fixture,
|
||||
):
|
||||
provider1, provider2, *_ = providers_fixture
|
||||
@@ -12737,10 +12948,10 @@ class TestOverviewViewSet:
|
||||
data = response.json()["data"]
|
||||
assert len(data) >= 1
|
||||
|
||||
@pytest.mark.usefixtures("provider_compliance_scores_fixture")
|
||||
def test_compliance_watchlist_provider_groups_filter(
|
||||
self,
|
||||
authenticated_client,
|
||||
provider_compliance_scores_fixture,
|
||||
providers_fixture,
|
||||
provider_groups_fixture,
|
||||
tenants_fixture,
|
||||
@@ -13640,6 +13851,26 @@ class TestSAMLTokenValidation:
|
||||
assert response2.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestCustomSAMLLoginView:
|
||||
def test_dispatch_clears_stale_callback_url_when_request_has_none(self):
|
||||
request = RequestFactory().get("/api/v1/saml/login/testtenant/")
|
||||
request.session = {
|
||||
"saml_callback_url": "/invitation/accept?invitation_token=old-token"
|
||||
}
|
||||
|
||||
with patch(
|
||||
"allauth.socialaccount.providers.saml.views.LoginView.dispatch",
|
||||
return_value=JsonResponse({}),
|
||||
):
|
||||
response = CustomSAMLLoginView.as_view()(
|
||||
request, organization_slug="testtenant"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert "saml_callback_url" not in request.session
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestSAMLInitiateAPIView:
|
||||
def test_valid_email_domain_and_certificates(
|
||||
@@ -13651,7 +13882,7 @@ class TestSAMLInitiateAPIView:
|
||||
url = reverse("api_saml_initiate")
|
||||
payload = {"email_domain": saml_setup["email"]}
|
||||
|
||||
response = authenticated_client.post(url, data=payload, format="json")
|
||||
response = authenticated_client.post(url, data=payload, format="vnd.api+json")
|
||||
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
assert (
|
||||
@@ -13660,11 +13891,42 @@ class TestSAMLInitiateAPIView:
|
||||
)
|
||||
assert "SAMLRequest" not in response.url
|
||||
|
||||
def test_valid_email_domain_preserves_safe_callback_url(
|
||||
self, authenticated_client, saml_setup
|
||||
):
|
||||
url = reverse("api_saml_initiate")
|
||||
callback_url = "/invitation/accept?invitation_token=test-token"
|
||||
payload = {
|
||||
"email_domain": saml_setup["email"],
|
||||
"callback_url": callback_url,
|
||||
}
|
||||
|
||||
response = authenticated_client.post(url, data=payload, format="vnd.api+json")
|
||||
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
query_params = parse_qs(urlparse(response.url).query)
|
||||
assert query_params["callback_url"] == [callback_url]
|
||||
|
||||
def test_valid_email_domain_rejects_external_callback_url(
|
||||
self, authenticated_client, saml_setup
|
||||
):
|
||||
url = reverse("api_saml_initiate")
|
||||
payload = {
|
||||
"email_domain": saml_setup["email"],
|
||||
"callback_url": "https://attacker.example/invitation",
|
||||
}
|
||||
|
||||
response = authenticated_client.post(url, data=payload, format="vnd.api+json")
|
||||
|
||||
assert response.status_code == status.HTTP_302_FOUND
|
||||
query_params = parse_qs(urlparse(response.url).query)
|
||||
assert "callback_url" not in query_params
|
||||
|
||||
def test_invalid_email_domain(self, authenticated_client):
|
||||
url = reverse("api_saml_initiate")
|
||||
payload = {"email_domain": "user@unauthorized.com"}
|
||||
|
||||
response = authenticated_client.post(url, data=payload, format="json")
|
||||
response = authenticated_client.post(url, data=payload, format="vnd.api+json")
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
assert response.json()["errors"]["detail"] == "Unauthorized domain."
|
||||
@@ -13847,7 +14109,8 @@ class TestTenantFinishACSView:
|
||||
)
|
||||
)
|
||||
request.user = user
|
||||
request.session = {}
|
||||
callback_url = "/invitation/accept?invitation_token=test-token"
|
||||
request.session = {"saml_callback_url": callback_url}
|
||||
|
||||
with (
|
||||
patch(
|
||||
@@ -13889,6 +14152,7 @@ class TestTenantFinishACSView:
|
||||
assert parsed_url.netloc == expected_callback_host
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
assert "id" in query_params
|
||||
assert query_params["callbackUrl"] == [callback_url]
|
||||
|
||||
token_id = query_params["id"][0]
|
||||
token_obj = SAMLToken.objects.get(id=token_id)
|
||||
@@ -16866,6 +17130,76 @@ class TestLighthouseProviderConfigViewSet:
|
||||
error_detail = str(resp.json()).lower()
|
||||
assert "base_url" in error_detail
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://127.0.0.1/v1",
|
||||
"https://169.254.169.254/latest/meta-data",
|
||||
],
|
||||
)
|
||||
def test_openai_compatible_rejects_internal_base_url_on_create(
|
||||
self, authenticated_client, base_url
|
||||
):
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai_compatible",
|
||||
"base_url": base_url,
|
||||
"credentials": {"api_key": "compat-key"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert resp.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "base_url" in str(resp.json()).lower()
|
||||
|
||||
def test_openai_compatible_rejects_internal_base_url_on_update(
|
||||
self, authenticated_client
|
||||
):
|
||||
create_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai_compatible",
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
"credentials": {"api_key": "compat-key-123"},
|
||||
},
|
||||
}
|
||||
}
|
||||
create_resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=create_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert create_resp.status_code == status.HTTP_201_CREATED
|
||||
provider_id = create_resp.json()["data"]["id"]
|
||||
|
||||
patch_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"id": provider_id,
|
||||
"attributes": {
|
||||
"base_url": "https://169.254.169.254/latest/meta-data",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
patch_resp = authenticated_client.patch(
|
||||
reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}),
|
||||
data=patch_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert patch_resp.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "base_url" in str(patch_resp.json()).lower()
|
||||
|
||||
def test_openai_compatible_invalid_credentials(self, authenticated_client):
|
||||
payload = {
|
||||
"data": {
|
||||
@@ -18292,10 +18626,10 @@ class TestFindingGroupViewSet:
|
||||
],
|
||||
ids=["summary_path", "finding_level_path"],
|
||||
)
|
||||
@pytest.mark.usefixtures("finding_groups_title_variants_fixture")
|
||||
def test_check_title_icontains_includes_all_title_variants(
|
||||
self,
|
||||
authenticated_client,
|
||||
finding_groups_title_variants_fixture,
|
||||
extra_filters,
|
||||
):
|
||||
"""
|
||||
|
||||
@@ -7,9 +7,19 @@ from allauth.socialaccount.providers.oauth2.client import OAuth2Client
|
||||
from api.db_router import MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.exceptions import InvitationTokenExpiredException
|
||||
from api.models import Integration, Invitation, Processor, Provider, Resource
|
||||
from api.models import (
|
||||
Integration,
|
||||
Invitation,
|
||||
Membership,
|
||||
Processor,
|
||||
Provider,
|
||||
Resource,
|
||||
Role,
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.v1.serializers import FindingMetadataSerializer
|
||||
from django.contrib.postgres.aggregates import ArrayAgg
|
||||
from django.db import transaction
|
||||
from django.db.models import Subquery
|
||||
from prowler.lib.outputs.jira.jira import Jira, JiraBasicAuthError
|
||||
from prowler.providers.aws.lib.s3.s3 import S3
|
||||
@@ -538,6 +548,35 @@ def validate_invitation(
|
||||
return invitation
|
||||
|
||||
|
||||
def accept_invitation_for_user(
|
||||
*, user, invitation_token: str, raise_not_found: bool = False
|
||||
):
|
||||
with transaction.atomic(using=MainRouter.admin_db):
|
||||
invitation = validate_invitation(
|
||||
invitation_token, user.email, raise_not_found=raise_not_found
|
||||
)
|
||||
with rls_transaction(str(invitation.tenant_id), using=MainRouter.admin_db):
|
||||
membership, _ = Membership.objects.using(MainRouter.admin_db).get_or_create(
|
||||
user=user,
|
||||
tenant=invitation.tenant,
|
||||
defaults={"role": Membership.RoleChoices.MEMBER},
|
||||
)
|
||||
invitation_roles = Role.objects.using(MainRouter.admin_db).filter(
|
||||
invitations=invitation
|
||||
)
|
||||
for role in invitation_roles:
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).get_or_create(
|
||||
user=user,
|
||||
role=role,
|
||||
defaults={"tenant": invitation.tenant},
|
||||
)
|
||||
|
||||
invitation.state = Invitation.State.ACCEPTED
|
||||
invitation.save(using=MainRouter.admin_db)
|
||||
|
||||
return invitation, membership
|
||||
|
||||
|
||||
# ToRemove after removing the fallback mechanism in /findings/metadata
|
||||
def get_findings_metadata_no_aggregations(tenant_id: str, filtered_queryset):
|
||||
filtered_ids = filtered_queryset.order_by().values("id")
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
from api.db_router import MainRouter
|
||||
from rest_framework_simplejwt.token_blacklist.models import (
|
||||
BlacklistedToken,
|
||||
OutstandingToken,
|
||||
)
|
||||
|
||||
|
||||
def blacklist_user_refresh_tokens(user_id):
|
||||
outstanding_token_ids = list(
|
||||
OutstandingToken.objects.using(MainRouter.admin_db)
|
||||
.filter(user_id=user_id)
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
if outstanding_token_ids:
|
||||
BlacklistedToken.objects.using(MainRouter.admin_db).bulk_create(
|
||||
[BlacklistedToken(token_id=token_id) for token_id in outstanding_token_ids],
|
||||
ignore_conflicts=True,
|
||||
)
|
||||
@@ -213,7 +213,8 @@ from rest_framework_json_api import serializers
|
||||
"properties": {
|
||||
"kubeconfig_content": {
|
||||
"type": "string",
|
||||
"description": "The content of the Kubernetes kubeconfig file, encoded as a string.",
|
||||
"description": "The content of the Kubernetes kubeconfig file, encoded as a string. "
|
||||
"Kubeconfig exec authentication is not supported in Prowler Cloud for security reasons.",
|
||||
}
|
||||
},
|
||||
"required": ["kubeconfig_content"],
|
||||
|
||||
@@ -2,6 +2,7 @@ import base64
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import yaml
|
||||
from api.db_router import MainRouter
|
||||
from api.exceptions import ConflictException
|
||||
from api.models import (
|
||||
@@ -37,6 +38,7 @@ from api.models import (
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.rls import Tenant
|
||||
from api.v1.serializer_utils.authentication import blacklist_user_refresh_tokens
|
||||
from api.v1.serializer_utils.integrations import (
|
||||
AWSCredentialSerializer,
|
||||
IntegrationConfigField,
|
||||
@@ -55,12 +57,13 @@ from api.v1.serializer_utils.lighthouse import (
|
||||
)
|
||||
from api.v1.serializer_utils.processors import ProcessorConfigField
|
||||
from api.v1.serializer_utils.providers import ProviderSecretField
|
||||
from api.validators import validate_lighthouse_openai_compatible_base_url
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import authenticate
|
||||
from django.contrib.auth.models import update_last_login
|
||||
from django.contrib.auth.password_validation import validate_password
|
||||
from django.core.exceptions import ValidationError as DjangoValidationError
|
||||
from django.db import IntegrityError
|
||||
from django.db import IntegrityError, transaction
|
||||
from drf_spectacular.utils import extend_schema_field
|
||||
from jwt.exceptions import InvalidKeyError
|
||||
from prowler.lib.mutelist.mutelist import Mutelist
|
||||
@@ -71,11 +74,28 @@ from rest_framework_json_api.relations import SerializerMethodResourceRelatedFie
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
from rest_framework_simplejwt.exceptions import TokenError
|
||||
from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
|
||||
from rest_framework_simplejwt.settings import api_settings
|
||||
from rest_framework_simplejwt.tokens import RefreshToken
|
||||
from rest_framework_simplejwt.utils import get_md5_hash_password
|
||||
|
||||
# Base
|
||||
|
||||
|
||||
def _validate_lighthouse_base_url_without_dns(base_url: str) -> None:
|
||||
try:
|
||||
validate_lighthouse_openai_compatible_base_url(base_url, resolve_dns=False)
|
||||
except DjangoValidationError as error:
|
||||
raise ValidationError({"base_url": error.messages[0]}) from error
|
||||
|
||||
|
||||
def _reraise_lighthouse_credentials_errors(error: ValidationError) -> None:
|
||||
details = error.detail.copy()
|
||||
for key, value in details.items():
|
||||
error.detail[f"credentials/{key}"] = value
|
||||
del error.detail[key]
|
||||
raise error
|
||||
|
||||
|
||||
class BaseModelSerializerV1(serializers.ModelSerializer):
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
@@ -231,6 +251,18 @@ class TokenRefreshSerializer(BaseSerializerV1):
|
||||
try:
|
||||
# Validate the refresh token
|
||||
refresh = RefreshToken(refresh_token)
|
||||
if api_settings.CHECK_REVOKE_TOKEN:
|
||||
user_id = refresh.payload.get(api_settings.USER_ID_CLAIM)
|
||||
try:
|
||||
user = User.objects.using(MainRouter.admin_db).get(
|
||||
**{api_settings.USER_ID_FIELD: user_id}
|
||||
)
|
||||
except User.DoesNotExist:
|
||||
raise TokenError("User not found.") from None
|
||||
if refresh.get(api_settings.REVOKE_TOKEN_CLAIM) != (
|
||||
get_md5_hash_password(user.password)
|
||||
):
|
||||
raise TokenError("The user's password has been changed.")
|
||||
# Generate new access token
|
||||
access_token = refresh.access_token
|
||||
|
||||
@@ -404,7 +436,13 @@ class UserUpdateSerializer(BaseWriteSerializer):
|
||||
password = validated_data.pop("password", None)
|
||||
if password:
|
||||
validate_password(password, user=instance)
|
||||
instance.set_password(password)
|
||||
with transaction.atomic(using=MainRouter.admin_db):
|
||||
instance.set_password(password)
|
||||
for attr, value in validated_data.items():
|
||||
setattr(instance, attr, value)
|
||||
blacklist_user_refresh_tokens(instance.id)
|
||||
instance.save(using=MainRouter.admin_db)
|
||||
return instance
|
||||
return super().update(instance, validated_data)
|
||||
|
||||
|
||||
@@ -1530,6 +1568,32 @@ class FindingMetadataSerializer(BaseSerializerV1):
|
||||
|
||||
|
||||
# Provider secrets
|
||||
KUBERNETES_KUBECONFIG_EXEC_ERROR = (
|
||||
"Kubernetes kubeconfig exec authentication is not supported in Prowler Cloud "
|
||||
"for security reasons."
|
||||
)
|
||||
KUBERNETES_KUBECONFIG_INVALID_ERROR = "Invalid Kubernetes kubeconfig content."
|
||||
|
||||
|
||||
def kubeconfig_contains_exec_auth(kubeconfig: dict) -> bool:
|
||||
users = kubeconfig.get("users", [])
|
||||
if not isinstance(users, list):
|
||||
raise ValidationError(KUBERNETES_KUBECONFIG_INVALID_ERROR)
|
||||
|
||||
for user_entry in users:
|
||||
if not isinstance(user_entry, dict):
|
||||
raise ValidationError(KUBERNETES_KUBECONFIG_INVALID_ERROR)
|
||||
|
||||
user = user_entry.get("user", {})
|
||||
if not isinstance(user, dict):
|
||||
raise ValidationError(KUBERNETES_KUBECONFIG_INVALID_ERROR)
|
||||
|
||||
if "exec" in user:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class BaseWriteProviderSecretSerializer(BaseWriteSerializer):
|
||||
@staticmethod
|
||||
def validate_secret_based_on_provider(
|
||||
@@ -1711,6 +1775,22 @@ class MongoDBAtlasProviderSecret(serializers.Serializer):
|
||||
class KubernetesProviderSecret(serializers.Serializer):
|
||||
kubeconfig_content = serializers.CharField()
|
||||
|
||||
def validate_kubeconfig_content(self, kubeconfig_content):
|
||||
try:
|
||||
kubeconfig = yaml.safe_load(kubeconfig_content)
|
||||
except yaml.YAMLError as exc:
|
||||
raise serializers.ValidationError(
|
||||
KUBERNETES_KUBECONFIG_INVALID_ERROR
|
||||
) from exc
|
||||
|
||||
if not isinstance(kubeconfig, dict):
|
||||
raise serializers.ValidationError(KUBERNETES_KUBECONFIG_INVALID_ERROR)
|
||||
|
||||
if kubeconfig_contains_exec_auth(kubeconfig):
|
||||
raise serializers.ValidationError(KUBERNETES_KUBECONFIG_EXEC_ERROR)
|
||||
|
||||
return kubeconfig_content
|
||||
|
||||
class Meta:
|
||||
resource_name = "provider-secrets"
|
||||
|
||||
@@ -3147,6 +3227,9 @@ class ProcessorUpdateSerializer(BaseWriteSerializer):
|
||||
|
||||
class SamlInitiateSerializer(BaseSerializerV1):
|
||||
email_domain = serializers.CharField()
|
||||
callback_url = serializers.CharField(
|
||||
required=False, allow_blank=True, max_length=2048
|
||||
)
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "saml-initiate"
|
||||
@@ -3578,11 +3661,7 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
elif (
|
||||
provider_type == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
|
||||
):
|
||||
@@ -3591,27 +3670,20 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
elif (
|
||||
provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE
|
||||
):
|
||||
if not base_url:
|
||||
raise ValidationError({"base_url": "Base URL is required."})
|
||||
_validate_lighthouse_base_url_without_dns(base_url)
|
||||
try:
|
||||
OpenAICompatibleCredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
|
||||
return super().validate(attrs)
|
||||
|
||||
@@ -3674,11 +3746,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
elif (
|
||||
credentials is not None
|
||||
and provider_type
|
||||
@@ -3702,11 +3770,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
|
||||
# Then enforce invariants about not changing the auth method
|
||||
# If the existing config uses an API key, forbid introducing access keys.
|
||||
@@ -3733,24 +3797,23 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
|
||||
}
|
||||
)
|
||||
elif (
|
||||
credentials is not None
|
||||
and provider_type
|
||||
provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE
|
||||
):
|
||||
if base_url is None:
|
||||
pass
|
||||
elif not base_url:
|
||||
effective_base_url = (
|
||||
base_url if "base_url" in attrs else getattr(self.instance, "base_url")
|
||||
)
|
||||
if not effective_base_url:
|
||||
raise ValidationError({"base_url": "Base URL cannot be empty."})
|
||||
try:
|
||||
OpenAICompatibleCredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
if "base_url" in attrs:
|
||||
_validate_lighthouse_base_url_without_dns(effective_base_url)
|
||||
if credentials is not None:
|
||||
try:
|
||||
OpenAICompatibleCredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
|
||||
return super().validate(attrs)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from collections import defaultdict
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
|
||||
from urllib.parse import urljoin
|
||||
from urllib.parse import urlencode, urljoin
|
||||
|
||||
import sentry_sdk
|
||||
from allauth.socialaccount.models import SocialAccount, SocialApp
|
||||
@@ -129,6 +129,7 @@ from api.renderers import APIJSONRenderer, PlainTextRenderer
|
||||
from api.rls import Tenant
|
||||
from api.utils import (
|
||||
CustomOAuth2Client,
|
||||
accept_invitation_for_user,
|
||||
get_findings_metadata_no_aggregations,
|
||||
initialize_prowler_integration,
|
||||
initialize_prowler_provider,
|
||||
@@ -236,7 +237,7 @@ from api.v1.serializers import (
|
||||
UserUpdateSerializer,
|
||||
)
|
||||
from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError
|
||||
from celery import chain, states
|
||||
from celery import chain
|
||||
from celery.result import AsyncResult
|
||||
from config.custom_logging import BackendLogger
|
||||
from config.env import env
|
||||
@@ -282,7 +283,6 @@ from django.utils.dateparse import parse_date
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.views.decorators.cache import cache_control
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from django_celery_results.models import TaskResult
|
||||
from drf_spectacular.settings import spectacular_settings
|
||||
from drf_spectacular.types import OpenApiTypes
|
||||
from drf_spectacular.utils import (
|
||||
@@ -321,17 +321,20 @@ from tasks.beat import schedule_provider_scan
|
||||
from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils
|
||||
from tasks.jobs.export import get_s3_client
|
||||
from tasks.tasks import (
|
||||
QUEUED_SCAN_TASK_STATE,
|
||||
backfill_compliance_summaries_task,
|
||||
backfill_scan_resource_summaries_task,
|
||||
check_integration_connection_task,
|
||||
check_lighthouse_connection_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
check_provider_connection_task,
|
||||
create_scan_task_record,
|
||||
delete_provider_task,
|
||||
delete_tenant_task,
|
||||
enqueue_scan_execution_on_commit,
|
||||
get_active_provider_scan,
|
||||
jira_integration_task,
|
||||
mute_historical_findings_task,
|
||||
perform_scan_task,
|
||||
reaggregate_all_finding_group_summaries_task,
|
||||
refresh_lighthouse_provider_models_task,
|
||||
)
|
||||
@@ -542,6 +545,46 @@ class SchemaView(SpectacularAPIView):
|
||||
return super().get(request, *args, **kwargs)
|
||||
|
||||
|
||||
SAML_CALLBACK_SESSION_KEY = "saml_callback_url"
|
||||
|
||||
|
||||
def _safe_callback_path(value):
|
||||
if not value or not isinstance(value, str):
|
||||
return None
|
||||
if not value.startswith("/") or value.startswith("//"):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _get_request_invitation_token(request):
|
||||
for source_name in ("data", "POST"):
|
||||
data = getattr(request, source_name, None) or {}
|
||||
if not hasattr(data, "get"):
|
||||
continue
|
||||
invitation_token = data.get("invitation_token")
|
||||
if invitation_token:
|
||||
return invitation_token
|
||||
|
||||
wrapped_request = getattr(request, "_request", None)
|
||||
if wrapped_request and wrapped_request is not request:
|
||||
return _get_request_invitation_token(wrapped_request)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _accept_social_invitation(request, user):
|
||||
invitation_token = _get_request_invitation_token(request)
|
||||
tenant_id = getattr(request, "prowler_invitation_tenant_id", None)
|
||||
if invitation_token and not tenant_id:
|
||||
invitation, _ = accept_invitation_for_user(
|
||||
user=user,
|
||||
invitation_token=invitation_token,
|
||||
raise_not_found=True,
|
||||
)
|
||||
tenant_id = str(invitation.tenant_id)
|
||||
return tenant_id
|
||||
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
class GoogleSocialLoginView(SocialLoginView):
|
||||
adapter_class = GoogleOAuth2Adapter
|
||||
@@ -552,7 +595,11 @@ class GoogleSocialLoginView(SocialLoginView):
|
||||
original_response = super().get_response()
|
||||
|
||||
if self.user and self.user.is_authenticated:
|
||||
serializer = TokenSocialLoginSerializer(data={"email": self.user.email})
|
||||
tenant_id = _accept_social_invitation(self.request, self.user)
|
||||
serializer_data = {"email": self.user.email}
|
||||
if tenant_id:
|
||||
serializer_data["tenant_id"] = tenant_id
|
||||
serializer = TokenSocialLoginSerializer(data=serializer_data)
|
||||
try:
|
||||
serializer.is_valid(raise_exception=True)
|
||||
except TokenError as e:
|
||||
@@ -577,7 +624,11 @@ class GithubSocialLoginView(SocialLoginView):
|
||||
original_response = super().get_response()
|
||||
|
||||
if self.user and self.user.is_authenticated:
|
||||
serializer = TokenSocialLoginSerializer(data={"email": self.user.email})
|
||||
tenant_id = _accept_social_invitation(self.request, self.user)
|
||||
serializer_data = {"email": self.user.email}
|
||||
if tenant_id:
|
||||
serializer_data["tenant_id"] = tenant_id
|
||||
serializer = TokenSocialLoginSerializer(data=serializer_data)
|
||||
|
||||
try:
|
||||
serializer.is_valid(raise_exception=True)
|
||||
@@ -637,6 +688,10 @@ class CustomSAMLLoginView(LoginView):
|
||||
|
||||
This approach maintains security while providing better UX.
|
||||
"""
|
||||
callback_url = _safe_callback_path(request.GET.get("callback_url"))
|
||||
request.session.pop(SAML_CALLBACK_SESSION_KEY, None)
|
||||
if callback_url:
|
||||
request.session[SAML_CALLBACK_SESSION_KEY] = callback_url
|
||||
if request.method == "GET":
|
||||
# Convert GET to POST while preserving parameters
|
||||
request.method = "POST"
|
||||
@@ -681,6 +736,11 @@ class SAMLInitiateAPIView(GenericAPIView):
|
||||
"saml_login", kwargs={"organization_slug": config.email_domain}
|
||||
)
|
||||
login_url = urljoin(api_host, login_path)
|
||||
callback_url = _safe_callback_path(
|
||||
serializer.validated_data.get("callback_url")
|
||||
)
|
||||
if callback_url:
|
||||
login_url = f"{login_url}?{urlencode({'callback_url': callback_url})}"
|
||||
|
||||
return redirect(login_url)
|
||||
|
||||
@@ -896,7 +956,13 @@ class TenantFinishACSView(FinishACSView):
|
||||
token=token_data, user=user
|
||||
)
|
||||
callback_url = env.str("SAML_SSO_CALLBACK_URL")
|
||||
redirect_url = f"{callback_url}?id={saml_token.id}"
|
||||
redirect_params = {"id": str(saml_token.id)}
|
||||
saml_callback_url = _safe_callback_path(
|
||||
request.session.pop(SAML_CALLBACK_SESSION_KEY, None)
|
||||
)
|
||||
if saml_callback_url:
|
||||
redirect_params["callbackUrl"] = saml_callback_url
|
||||
redirect_url = f"{callback_url}?{urlencode(redirect_params)}"
|
||||
request.session.pop("saml_user_created", None)
|
||||
|
||||
return redirect(redirect_url)
|
||||
@@ -948,8 +1014,8 @@ class UserViewSet(BaseUserViewset):
|
||||
"""
|
||||
Returns the required permissions based on the request method.
|
||||
"""
|
||||
if self.action == "me":
|
||||
# No permissions required for me request
|
||||
if self.action in ["me", "partial_update"]:
|
||||
# No permissions required for me and partial_update requests
|
||||
self.required_permissions = []
|
||||
else:
|
||||
# Require permission for the rest of the requests
|
||||
@@ -1003,6 +1069,24 @@ class UserViewSet(BaseUserViewset):
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
def partial_update(self, request, *args, **kwargs):
|
||||
user = self.get_object()
|
||||
if user.id != self.request.user.id:
|
||||
role = get_role(self.request.user, self.request.tenant_id)
|
||||
if not getattr(role, Permissions.MANAGE_USERS.value, False):
|
||||
raise ValidationError(
|
||||
"Only users with manage users permission can update other users."
|
||||
)
|
||||
|
||||
serializer = self.get_serializer(user, data=request.data, partial=True)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
self.perform_update(serializer)
|
||||
|
||||
if getattr(user, "_prefetched_objects_cache", None):
|
||||
user._prefetched_objects_cache = {}
|
||||
|
||||
return Response(serializer.data)
|
||||
|
||||
def destroy(self, request, *args, **kwargs):
|
||||
if kwargs["pk"] != str(self.request.user.id):
|
||||
raise ValidationError("Only the current user can be deleted.")
|
||||
@@ -2635,12 +2719,23 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
def create(self, request, *args, **kwargs):
|
||||
input_serializer = self.get_serializer(data=request.data)
|
||||
input_serializer.is_valid(raise_exception=True)
|
||||
provider = input_serializer.validated_data.get("provider")
|
||||
active_scan = None
|
||||
|
||||
# Broker publish is deferred to on_commit so the worker cannot read
|
||||
# Scan before BaseRLSViewSet's dispatch-wide atomic commits.
|
||||
pre_task_id = str(uuid.uuid4())
|
||||
|
||||
with transaction.atomic():
|
||||
if provider:
|
||||
provider = Provider.objects.select_for_update().get(
|
||||
id=provider.id,
|
||||
tenant_id=self.request.tenant_id,
|
||||
)
|
||||
active_scan = get_active_provider_scan(
|
||||
self.request.tenant_id, provider.id
|
||||
)
|
||||
|
||||
scan = input_serializer.save()
|
||||
scan.task_id = pre_task_id
|
||||
scan.save(update_fields=["task_id"])
|
||||
@@ -2651,29 +2746,18 @@ class ScanViewSet(BaseRLSViewSet):
|
||||
provider_id=str(scan.provider_id),
|
||||
)
|
||||
|
||||
task_result, _ = TaskResult.objects.get_or_create(
|
||||
task_id=pre_task_id,
|
||||
defaults={"status": states.PENDING, "task_name": "scan-perform"},
|
||||
)
|
||||
prowler_task, _ = Task.objects.update_or_create(
|
||||
id=pre_task_id,
|
||||
prowler_task = create_scan_task_record(
|
||||
tenant_id=self.request.tenant_id,
|
||||
defaults={"task_runner_task": task_result},
|
||||
task_id=pre_task_id,
|
||||
task_status=(QUEUED_SCAN_TASK_STATE if active_scan else None),
|
||||
)
|
||||
|
||||
scan_kwargs = {
|
||||
"tenant_id": self.request.tenant_id,
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
# Disabled for now
|
||||
# checks_to_execute=scan.scanner_args.get("checks_to_execute")
|
||||
}
|
||||
|
||||
transaction.on_commit(
|
||||
lambda: perform_scan_task.apply_async(
|
||||
kwargs=scan_kwargs, task_id=pre_task_id
|
||||
if not active_scan:
|
||||
enqueue_scan_execution_on_commit(
|
||||
tenant_id=self.request.tenant_id,
|
||||
scan=scan,
|
||||
task_id=pre_task_id,
|
||||
)
|
||||
)
|
||||
|
||||
self.response_serializer_class = TaskSerializer
|
||||
output_serializer = self.get_serializer(prowler_task)
|
||||
@@ -4371,25 +4455,12 @@ class InvitationAcceptViewSet(BaseRLSViewSet):
|
||||
invitation_token = serializer.validated_data["invitation_token"]
|
||||
user_email = request.user.email
|
||||
|
||||
invitation = validate_invitation(
|
||||
invitation_token, user_email, raise_not_found=True
|
||||
)
|
||||
|
||||
# Proceed with accepting the invitation
|
||||
user = User.objects.using(MainRouter.admin_db).get(email=user_email)
|
||||
membership = Membership.objects.using(MainRouter.admin_db).create(
|
||||
invitation, membership = accept_invitation_for_user(
|
||||
user=user,
|
||||
tenant=invitation.tenant,
|
||||
invitation_token=invitation_token,
|
||||
raise_not_found=True,
|
||||
)
|
||||
user_role = []
|
||||
for role in invitation.roles.all():
|
||||
user_role.append(
|
||||
UserRoleRelationship.objects.using(MainRouter.admin_db).create(
|
||||
user=user, role=role, tenant=invitation.tenant
|
||||
)
|
||||
)
|
||||
invitation.state = Invitation.State.ACCEPTED
|
||||
invitation.save(using=MainRouter.admin_db)
|
||||
|
||||
self.response_serializer_class = MembershipSerializer
|
||||
membership_serializer = self.get_serializer(membership)
|
||||
|
||||
@@ -1,14 +1,155 @@
|
||||
import ipaddress
|
||||
import socket
|
||||
import string
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.utils.translation import gettext as _
|
||||
|
||||
LIGHTHOUSE_OPENAI_COMPATIBLE_ALLOWED_SCHEMES = frozenset({"https"})
|
||||
LIGHTHOUSE_NAT64_WELL_KNOWN_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
|
||||
LIGHTHOUSE_BLOCKED_METADATA_HOSTS = frozenset(
|
||||
{
|
||||
"169.254.169.254",
|
||||
"169.254.170.2",
|
||||
"fd00:ec2::254",
|
||||
"localhost",
|
||||
"metadata.google.internal",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_hostname(hostname: str) -> str:
|
||||
return hostname.rstrip(".").lower()
|
||||
|
||||
|
||||
def _lighthouse_openai_compatible_allowed_hosts() -> frozenset[str]:
|
||||
return frozenset(
|
||||
_normalize_hostname(allowed_host.strip())
|
||||
for allowed_host in settings.LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS
|
||||
if allowed_host and allowed_host.strip()
|
||||
)
|
||||
|
||||
|
||||
def _validate_lighthouse_public_ip(address: str) -> None:
|
||||
ip_address = ipaddress.ip_address(address)
|
||||
if isinstance(ip_address, ipaddress.IPv6Address):
|
||||
# Classify transition addresses by their effective IPv4 destination.
|
||||
embedded_ip_address = ip_address.ipv4_mapped or ip_address.sixtofour
|
||||
if (
|
||||
embedded_ip_address is None
|
||||
and ip_address in LIGHTHOUSE_NAT64_WELL_KNOWN_PREFIX
|
||||
):
|
||||
embedded_ip_address = ipaddress.IPv4Address(int(ip_address) & 0xFFFFFFFF)
|
||||
if embedded_ip_address is not None:
|
||||
ip_address = embedded_ip_address
|
||||
if not ip_address.is_global:
|
||||
raise ValidationError(
|
||||
_("Base URL must use an external public endpoint."),
|
||||
code="lighthouse_base_url_not_public",
|
||||
)
|
||||
|
||||
|
||||
def resolve_lighthouse_openai_compatible_host(
|
||||
hostname: str,
|
||||
port: int,
|
||||
*,
|
||||
resolve_dns: bool = True,
|
||||
) -> tuple[str, ...]:
|
||||
"""Return public IP addresses that are safe for Lighthouse outbound use."""
|
||||
hostname = _normalize_hostname(hostname)
|
||||
if hostname in _lighthouse_openai_compatible_allowed_hosts():
|
||||
# Operator-allowlisted hosts skip the public-endpoint checks; returning
|
||||
# the hostname makes the network backend connect through regular DNS
|
||||
# resolution instead of pinned addresses.
|
||||
return (hostname,)
|
||||
|
||||
if hostname in LIGHTHOUSE_BLOCKED_METADATA_HOSTS or hostname.endswith(".localhost"):
|
||||
raise ValidationError(
|
||||
_("Base URL must use an external public endpoint."),
|
||||
code="lighthouse_base_url_blocked_host",
|
||||
)
|
||||
|
||||
try:
|
||||
_validate_lighthouse_public_ip(hostname)
|
||||
except ValueError:
|
||||
if not resolve_dns:
|
||||
return ()
|
||||
else:
|
||||
return (hostname,)
|
||||
|
||||
try:
|
||||
resolved_addresses = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM)
|
||||
except socket.gaierror as error:
|
||||
raise ValidationError(
|
||||
_("Base URL host could not be resolved."),
|
||||
code="lighthouse_base_url_resolution_failed",
|
||||
) from error
|
||||
|
||||
if not resolved_addresses:
|
||||
raise ValidationError(
|
||||
_("Base URL host could not be resolved."),
|
||||
code="lighthouse_base_url_resolution_failed",
|
||||
)
|
||||
|
||||
public_addresses: list[str] = []
|
||||
for resolved_address in resolved_addresses:
|
||||
socket_address = resolved_address[4]
|
||||
resolved_ip_address = socket_address[0]
|
||||
_validate_lighthouse_public_ip(resolved_ip_address)
|
||||
if resolved_ip_address not in public_addresses:
|
||||
public_addresses.append(resolved_ip_address)
|
||||
|
||||
return tuple(public_addresses)
|
||||
|
||||
|
||||
def validate_lighthouse_openai_compatible_base_url(
|
||||
base_url: str,
|
||||
*,
|
||||
resolve_dns: bool = True,
|
||||
) -> None:
|
||||
"""Validate an OpenAI-compatible Lighthouse base URL before outbound use."""
|
||||
parsed = urlparse(str(base_url))
|
||||
if parsed.scheme.lower() not in LIGHTHOUSE_OPENAI_COMPATIBLE_ALLOWED_SCHEMES:
|
||||
raise ValidationError(
|
||||
_("Base URL must use HTTPS."),
|
||||
code="lighthouse_base_url_invalid_scheme",
|
||||
)
|
||||
|
||||
if not parsed.hostname:
|
||||
raise ValidationError(
|
||||
_("Base URL must include a host."),
|
||||
code="lighthouse_base_url_missing_host",
|
||||
)
|
||||
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError as error:
|
||||
raise ValidationError(
|
||||
_("Base URL port is invalid."),
|
||||
code="lighthouse_base_url_invalid_port",
|
||||
) from error
|
||||
|
||||
if port is not None and not 1 <= port <= 65535:
|
||||
raise ValidationError(
|
||||
_("Base URL port is invalid."),
|
||||
code="lighthouse_base_url_invalid_port",
|
||||
)
|
||||
|
||||
resolve_lighthouse_openai_compatible_host(
|
||||
parsed.hostname,
|
||||
port or 443,
|
||||
resolve_dns=resolve_dns,
|
||||
)
|
||||
|
||||
|
||||
class MaximumLengthValidator:
|
||||
def __init__(self, max_length=72):
|
||||
self.max_length = max_length
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if len(password) > self.max_length:
|
||||
raise ValidationError(
|
||||
_(
|
||||
@@ -31,6 +172,7 @@ class SpecialCharactersValidator:
|
||||
self.min_special_characters = min_special_characters
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if (
|
||||
sum(1 for char in password if char in self.special_characters)
|
||||
< self.min_special_characters
|
||||
@@ -55,6 +197,7 @@ class UppercaseValidator:
|
||||
self.min_uppercase = min_uppercase
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if sum(1 for char in password if char.isupper()) < self.min_uppercase:
|
||||
raise ValidationError(
|
||||
_(
|
||||
@@ -75,6 +218,7 @@ class LowercaseValidator:
|
||||
self.min_lowercase = min_lowercase
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if sum(1 for char in password if char.islower()) < self.min_lowercase:
|
||||
raise ValidationError(
|
||||
_(
|
||||
@@ -95,6 +239,7 @@ class NumericValidator:
|
||||
self.min_numeric = min_numeric
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if sum(1 for char in password if char.isdigit()) < self.min_numeric:
|
||||
raise ValidationError(
|
||||
_(
|
||||
|
||||
@@ -230,6 +230,7 @@ SIMPLE_JWT = {
|
||||
"JTI_CLAIM": "jti",
|
||||
"USER_ID_FIELD": "id",
|
||||
"USER_ID_CLAIM": "sub",
|
||||
"CHECK_REVOKE_TOKEN": True,
|
||||
# Issuer and Audience claims, for the moment we will keep these values as default values, they may change in the
|
||||
# future.
|
||||
"AUDIENCE": env.str("DJANGO_JWT_AUDIENCE", "https://api.prowler.com"),
|
||||
@@ -316,6 +317,15 @@ ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
|
||||
# Valid values: "neo4j" (default, OSS and local dev), "neptune" (hosted).
|
||||
ATTACK_PATHS_SINK_DATABASE = env.str("ATTACK_PATHS_SINK_DATABASE", default="neo4j")
|
||||
|
||||
# Lighthouse AI
|
||||
# Comma-separated hostnames (or IP literals) that bypass the SSRF validation
|
||||
# applied to OpenAI-compatible provider base URLs, so self-hosted deployments
|
||||
# can point Lighthouse AI at internal endpoints. Empty by default: every base
|
||||
# URL must resolve to a public endpoint.
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS = env.list(
|
||||
"LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS", default=[]
|
||||
)
|
||||
|
||||
# Orphan task recovery feature flags. The master switch is OFF by default, so task
|
||||
# recovery is opt-in; enable it with DJANGO_TASK_RECOVERY_ENABLED=true. The per-group
|
||||
# toggles default to enabled, so once the master is on every group recovers unless a
|
||||
|
||||
@@ -19,6 +19,7 @@ import json
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
import neo4j
|
||||
@@ -392,11 +393,11 @@ def _build_child_props(
|
||||
def _build_child_id(provider_id: str, child_label: str, value_key: str) -> str:
|
||||
"""Deterministic `_provider_element_id` for a list-item child node.
|
||||
|
||||
Dedupes within (tenant, provider): multiple parents referencing the same
|
||||
value share one child node via the existing MERGE-on-_provider_element_id
|
||||
index in both sinks.
|
||||
Hashing the value keeps the ID bounded while preserving deduplication within
|
||||
each provider and child label.
|
||||
"""
|
||||
return f"{provider_id}::{child_label}::{value_key}"
|
||||
value_digest = sha256(value_key.encode("utf-8")).hexdigest()
|
||||
return f"{provider_id}::{child_label}::{value_digest}"
|
||||
|
||||
|
||||
def _build_catalog_index(
|
||||
|
||||
@@ -14,6 +14,7 @@ from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
|
||||
from prowler.lib.outputs.csv.csv import CSV
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
from prowler.lib.outputs.html.html import HTML
|
||||
from prowler.lib.outputs.jira.exceptions.exceptions import JiraBaseException
|
||||
from prowler.lib.outputs.ocsf.ocsf import OCSF
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.aws.lib.s3.s3 import S3
|
||||
@@ -26,6 +27,8 @@ from tasks.utils import batched
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
JIRA_GENERIC_SEND_ERROR = "Failed to create Jira issue."
|
||||
|
||||
|
||||
def get_s3_client_from_integration(
|
||||
integration: Integration,
|
||||
@@ -483,6 +486,7 @@ def send_findings_to_jira(
|
||||
jira_integration = initialize_prowler_integration(integration)
|
||||
|
||||
num_tickets_created = 0
|
||||
error_messages = []
|
||||
for finding_id in finding_ids:
|
||||
with rls_transaction(tenant_id):
|
||||
finding_instance = (
|
||||
@@ -512,35 +516,54 @@ def send_findings_to_jira(
|
||||
recommendation = remediation.get("recommendation", {})
|
||||
remediation_code = remediation.get("code", {})
|
||||
|
||||
# Send the individual finding to Jira
|
||||
result = jira_integration.send_finding(
|
||||
check_id=finding_instance.check_id,
|
||||
check_title=check_metadata.get("checktitle", ""),
|
||||
severity=finding_instance.severity,
|
||||
status=finding_instance.status,
|
||||
status_extended=finding_instance.status_extended or "",
|
||||
provider=finding_instance.scan.provider.provider,
|
||||
region=region,
|
||||
resource_uid=resource_uid,
|
||||
resource_name=resource_name,
|
||||
risk=check_metadata.get("risk", ""),
|
||||
recommendation_text=recommendation.get("text", ""),
|
||||
recommendation_url=recommendation.get("url", ""),
|
||||
remediation_code_native_iac=remediation_code.get("nativeiac", ""),
|
||||
remediation_code_terraform=remediation_code.get("terraform", ""),
|
||||
remediation_code_cli=remediation_code.get("cli", ""),
|
||||
remediation_code_other=remediation_code.get("other", ""),
|
||||
resource_tags=resource_tags,
|
||||
compliance=finding_instance.compliance or {},
|
||||
project_key=project_key,
|
||||
issue_type=issue_type,
|
||||
)
|
||||
try:
|
||||
# Send the individual finding to Jira
|
||||
result = jira_integration.send_finding(
|
||||
check_id=finding_instance.check_id,
|
||||
check_title=check_metadata.get("checktitle", ""),
|
||||
severity=finding_instance.severity,
|
||||
status=finding_instance.status,
|
||||
status_extended=finding_instance.status_extended or "",
|
||||
provider=finding_instance.scan.provider.provider,
|
||||
region=region,
|
||||
resource_uid=resource_uid,
|
||||
resource_name=resource_name,
|
||||
risk=check_metadata.get("risk", ""),
|
||||
recommendation_text=recommendation.get("text", ""),
|
||||
recommendation_url=recommendation.get("url", ""),
|
||||
remediation_code_native_iac=remediation_code.get("nativeiac", ""),
|
||||
remediation_code_terraform=remediation_code.get("terraform", ""),
|
||||
remediation_code_cli=remediation_code.get("cli", ""),
|
||||
remediation_code_other=remediation_code.get("other", ""),
|
||||
resource_tags=resource_tags,
|
||||
compliance=finding_instance.compliance or {},
|
||||
project_key=project_key,
|
||||
issue_type=issue_type,
|
||||
)
|
||||
except JiraBaseException as error:
|
||||
error_message = error.message or JIRA_GENERIC_SEND_ERROR
|
||||
logger.exception(
|
||||
"Failed to send finding %s to Jira: %s", finding_id, error_message
|
||||
)
|
||||
error_messages.append(error_message)
|
||||
continue
|
||||
except Exception:
|
||||
logger.exception("Failed to send finding %s to Jira", finding_id)
|
||||
error_messages.append(JIRA_GENERIC_SEND_ERROR)
|
||||
continue
|
||||
|
||||
if result:
|
||||
num_tickets_created += 1
|
||||
else:
|
||||
logger.error(f"Failed to send finding {finding_id} to Jira")
|
||||
error_message = JIRA_GENERIC_SEND_ERROR
|
||||
logger.error(error_message)
|
||||
error_messages.append(error_message)
|
||||
|
||||
return {
|
||||
result = {
|
||||
"created_count": num_tickets_created,
|
||||
"failed_count": len(finding_ids) - num_tickets_created,
|
||||
}
|
||||
if error_messages:
|
||||
result["error"] = "; ".join(dict.fromkeys(error_messages))
|
||||
|
||||
return result
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import ssl
|
||||
from collections.abc import Iterable
|
||||
|
||||
import boto3
|
||||
import httpcore
|
||||
import httpx
|
||||
import openai
|
||||
from api.models import LighthouseProviderConfiguration, LighthouseProviderModels
|
||||
from api.validators import (
|
||||
resolve_lighthouse_openai_compatible_host,
|
||||
validate_lighthouse_openai_compatible_base_url,
|
||||
)
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
@@ -43,6 +52,90 @@ EXCLUDED_OPENAI_MODEL_SUBSTRINGS = (
|
||||
"-instruct", # Legacy instruct models (gpt-3.5-turbo-instruct, etc.)
|
||||
)
|
||||
|
||||
OPENAI_COMPATIBLE_AUTHENTICATION_ERROR = "API key is invalid or missing"
|
||||
OPENAI_COMPATIBLE_CONNECTION_ERROR = "Provider connection failed"
|
||||
|
||||
|
||||
class _OpenAICompatibleProviderError(Exception):
|
||||
"""Sanitized OpenAI-compatible provider error safe for task results."""
|
||||
|
||||
|
||||
def _sanitize_openai_compatible_error(error: Exception) -> str:
|
||||
status_code = getattr(error, "status_code", None)
|
||||
if status_code is None:
|
||||
response = getattr(error, "response", None)
|
||||
status_code = getattr(response, "status_code", None)
|
||||
|
||||
if status_code == 401:
|
||||
return OPENAI_COMPATIBLE_AUTHENTICATION_ERROR
|
||||
return OPENAI_COMPATIBLE_CONNECTION_ERROR
|
||||
|
||||
|
||||
class _LighthouseOpenAICompatibleNetworkBackend(httpcore.SyncBackend):
|
||||
"""Validate and pin DNS results immediately before TCP connections."""
|
||||
|
||||
def connect_tcp(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
timeout: float | None = None,
|
||||
local_address: str | None = None,
|
||||
socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None,
|
||||
) -> httpcore.NetworkStream:
|
||||
resolved_addresses = resolve_lighthouse_openai_compatible_host(host, port)
|
||||
last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None
|
||||
|
||||
for address in resolved_addresses:
|
||||
try:
|
||||
return super().connect_tcp(
|
||||
address,
|
||||
port,
|
||||
timeout=timeout,
|
||||
local_address=local_address,
|
||||
socket_options=socket_options,
|
||||
)
|
||||
except (httpcore.ConnectError, httpcore.ConnectTimeout) as error:
|
||||
last_error = error
|
||||
|
||||
if last_error:
|
||||
raise last_error
|
||||
raise httpcore.ConnectError("No resolved addresses are available")
|
||||
|
||||
|
||||
class _LighthouseOpenAICompatibleHTTPTransport(httpx.HTTPTransport):
|
||||
"""HTTP transport that connects only to validated public IP addresses."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._pool = httpcore.ConnectionPool(
|
||||
ssl_context=ssl.create_default_context(),
|
||||
network_backend=_LighthouseOpenAICompatibleNetworkBackend(),
|
||||
)
|
||||
|
||||
|
||||
def _create_openai_compatible_http_client() -> httpx.Client:
|
||||
"""Create the restricted HTTP client used for OpenAI-compatible providers."""
|
||||
return httpx.Client(
|
||||
follow_redirects=False,
|
||||
trust_env=False,
|
||||
transport=_LighthouseOpenAICompatibleHTTPTransport(),
|
||||
)
|
||||
|
||||
|
||||
def _list_openai_compatible_models(base_url: str, api_key: str):
|
||||
validate_lighthouse_openai_compatible_base_url(base_url)
|
||||
try:
|
||||
with _create_openai_compatible_http_client() as http_client:
|
||||
client = openai.OpenAI(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
http_client=http_client,
|
||||
)
|
||||
return client.models.list()
|
||||
except Exception as error:
|
||||
raise _OpenAICompatibleProviderError(
|
||||
_sanitize_openai_compatible_error(error)
|
||||
) from error
|
||||
|
||||
|
||||
def _extract_error_message(e: Exception) -> str:
|
||||
"""
|
||||
@@ -114,6 +207,7 @@ def _extract_openai_compatible_params(
|
||||
return None
|
||||
if not isinstance(base_url, str) or not base_url:
|
||||
return None
|
||||
validate_lighthouse_openai_compatible_base_url(base_url, resolve_dns=False)
|
||||
return {"base_url": base_url, "api_key": api_key}
|
||||
|
||||
|
||||
@@ -285,13 +379,7 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> dict:
|
||||
"error": "Base URL or API key is invalid or missing",
|
||||
}
|
||||
|
||||
# Test connection using OpenAI SDK with custom base_url
|
||||
# Note: base_url should include version (e.g., https://openrouter.ai/api/v1)
|
||||
client = openai.OpenAI(
|
||||
api_key=params["api_key"],
|
||||
base_url=params["base_url"],
|
||||
)
|
||||
_ = client.models.list()
|
||||
_ = _list_openai_compatible_models(params["base_url"], params["api_key"])
|
||||
|
||||
else:
|
||||
return {"connected": False, "error": "Unsupported provider type"}
|
||||
@@ -361,8 +449,7 @@ def _fetch_openai_compatible_models(base_url: str, api_key: str) -> dict[str, st
|
||||
|
||||
Note: base_url should include version (e.g., https://openrouter.ai/api/v1)
|
||||
"""
|
||||
client = openai.OpenAI(api_key=api_key, base_url=base_url)
|
||||
models = client.models.list()
|
||||
models = _list_openai_compatible_models(base_url, api_key)
|
||||
|
||||
available_models: dict[str, str] = {}
|
||||
for model in models.data:
|
||||
|
||||
@@ -11,7 +11,6 @@ from uuid import UUID
|
||||
from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, Scan, ScanSummary, StateChoices, ThreatScoreSnapshot
|
||||
from api.utils import initialize_prowler_provider
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.django.base import DJANGO_TMP_OUTPUT_DIRECTORY
|
||||
from prowler.lib.check.compliance_models import (
|
||||
@@ -27,6 +26,7 @@ from tasks.jobs.reports import (
|
||||
ENSReportGenerator,
|
||||
NIS2ReportGenerator,
|
||||
ThreatScoreReportGenerator,
|
||||
build_provider_metadata,
|
||||
)
|
||||
from tasks.jobs.threatscore import compute_threatscore_metrics
|
||||
from tasks.jobs.threatscore_utils import (
|
||||
@@ -841,24 +841,12 @@ def generate_compliance_reports(
|
||||
tenant_id, scan_id
|
||||
)
|
||||
|
||||
# Initialize the Prowler provider once for the whole report batch. Each
|
||||
# generator used to re-init this in _load_compliance_data, paying the
|
||||
# boto3/Azure-SDK construction cost 5 times per scan. The instance is
|
||||
# only used by FindingOutput.transform_api_finding to enrich findings,
|
||||
# so a single shared instance is correct.
|
||||
logger.info("Initializing prowler_provider once for all reports (scan %s)", scan_id)
|
||||
try:
|
||||
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
except Exception as init_error:
|
||||
# If init fails the generators will fall back to lazy init in
|
||||
# _load_compliance_data; we just log and continue.
|
||||
logger.warning(
|
||||
"Could not pre-initialize prowler_provider for scan %s: %s",
|
||||
scan_id,
|
||||
init_error,
|
||||
)
|
||||
prowler_provider = None
|
||||
# Build a credential-free provider metadata stub once for the whole
|
||||
# report batch. FindingOutput.transform_api_finding only reads static
|
||||
# attributes (type plus a few identity fields), so reports never decrypt
|
||||
# the ProviderSecret nor construct a cloud SDK session — generation keeps
|
||||
# working after credentials are deleted or invalidated (PROWLER-2145).
|
||||
prowler_provider = build_provider_metadata(provider_obj)
|
||||
|
||||
# Create shared findings cache up front so the eviction closure below
|
||||
# can reference it. Defined BEFORE the closure to avoid the UnboundLocalError
|
||||
|
||||
@@ -98,6 +98,7 @@ from .config import (
|
||||
from .csa import CSAReportGenerator
|
||||
from .ens import ENSReportGenerator
|
||||
from .nis2 import NIS2ReportGenerator
|
||||
from .provider_metadata import build_provider_metadata
|
||||
from .threatscore import ThreatScoreReportGenerator
|
||||
|
||||
__all__ = [
|
||||
@@ -105,6 +106,7 @@ __all__ = [
|
||||
"BaseComplianceReportGenerator",
|
||||
"ComplianceData",
|
||||
"RequirementData",
|
||||
"build_provider_metadata",
|
||||
"create_pdf_styles",
|
||||
"get_requirement_metadata",
|
||||
# Framework-specific generators
|
||||
|
||||
@@ -11,7 +11,6 @@ from typing import Any
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import Provider, StatusChoices
|
||||
from api.utils import initialize_prowler_provider
|
||||
from celery.utils.log import get_task_logger
|
||||
from prowler.lib.check.compliance_models import (
|
||||
Compliance,
|
||||
@@ -52,6 +51,7 @@ from .config import (
|
||||
PADDING_SMALL,
|
||||
FrameworkConfig,
|
||||
)
|
||||
from .provider_metadata import build_provider_metadata
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@@ -178,7 +178,8 @@ class ComplianceData:
|
||||
attributes_by_requirement_id: Mapping of requirement IDs to their attributes
|
||||
findings_by_check_id: Mapping of check IDs to their findings
|
||||
provider_obj: Provider model object
|
||||
prowler_provider: Initialized Prowler provider
|
||||
prowler_provider: Credential-free provider metadata stub (see
|
||||
``build_provider_metadata``)
|
||||
"""
|
||||
|
||||
tenant_id: str
|
||||
@@ -439,10 +440,10 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj: Optional pre-fetched Provider object
|
||||
requirement_statistics: Optional pre-aggregated statistics
|
||||
findings_cache: Optional pre-loaded findings cache
|
||||
prowler_provider: Optional pre-initialized Prowler provider. When
|
||||
generating multiple reports for the same scan the master
|
||||
function initializes this once and passes it in to avoid
|
||||
re-running boto3/Azure-SDK setup per framework.
|
||||
prowler_provider: Optional provider metadata stub (see
|
||||
``build_provider_metadata``). When generating multiple
|
||||
reports for the same scan the master function builds it
|
||||
once and passes it in.
|
||||
**kwargs: Additional framework-specific arguments
|
||||
"""
|
||||
framework = self.config.display_name
|
||||
@@ -896,9 +897,9 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj: Optional pre-fetched Provider
|
||||
requirement_statistics: Optional pre-aggregated statistics
|
||||
findings_cache: Optional pre-loaded findings
|
||||
prowler_provider: Optional pre-initialized Prowler provider. When
|
||||
the master function initializes it once and passes it in,
|
||||
we skip the per-report ``initialize_prowler_provider`` call.
|
||||
prowler_provider: Optional provider metadata stub. When the
|
||||
master function builds it once and passes it in, we skip
|
||||
the per-report ``build_provider_metadata`` call.
|
||||
|
||||
Returns:
|
||||
Aggregated ComplianceData object
|
||||
@@ -909,7 +910,7 @@ class BaseComplianceReportGenerator(ABC):
|
||||
provider_obj = Provider.objects.get(id=provider_id)
|
||||
|
||||
if prowler_provider is None:
|
||||
prowler_provider = initialize_prowler_provider(provider_obj)
|
||||
prowler_provider = build_provider_metadata(provider_obj)
|
||||
provider_type = provider_obj.provider
|
||||
|
||||
# Load compliance framework — fall back to the universal loader
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from prowler.providers.github.models import GithubIdentityInfo
|
||||
|
||||
|
||||
def build_provider_metadata(provider) -> SimpleNamespace:
|
||||
"""Build a credential-free stand-in for the Prowler SDK provider.
|
||||
|
||||
``FindingOutput.transform_api_finding`` only reads static attributes
|
||||
from the provider (``type`` plus a few identity/metadata fields used to
|
||||
label accounts), so compliance reports never need the decrypted
|
||||
``ProviderSecret`` nor a live cloud SDK session. This builds an object
|
||||
exposing exactly those attributes from the ``Provider`` DB row, which
|
||||
keeps report generation working when the provider secret has been
|
||||
deleted or its credentials are no longer valid (PROWLER-2145).
|
||||
|
||||
Args:
|
||||
provider: The API ``Provider`` model instance (only ``provider``,
|
||||
``uid`` and ``alias`` are read).
|
||||
|
||||
Returns:
|
||||
A ``SimpleNamespace`` mimicking the SDK provider attributes consumed
|
||||
by ``FindingOutput.transform_api_finding`` / ``generate_output``.
|
||||
"""
|
||||
provider_type = provider.provider
|
||||
uid = provider.uid
|
||||
display_name = provider.alias or uid
|
||||
|
||||
# Defaults cover every attribute read unconditionally in
|
||||
# FindingOutput.generate_output (``provider.auth_method`` is accessed
|
||||
# directly for several provider types); identity lookups go through
|
||||
# get_nested_attribute/getattr, which tolerate missing attributes.
|
||||
stub = SimpleNamespace(
|
||||
type=provider_type,
|
||||
auth_method="",
|
||||
identity=SimpleNamespace(),
|
||||
)
|
||||
|
||||
if provider_type == "aws":
|
||||
stub.identity = SimpleNamespace(account=uid)
|
||||
elif provider_type == "azure":
|
||||
stub.identity = SimpleNamespace(
|
||||
identity_type="",
|
||||
identity_id="",
|
||||
tenant_ids=[""],
|
||||
tenant_domain="",
|
||||
subscriptions={uid: display_name},
|
||||
)
|
||||
elif provider_type == "gcp":
|
||||
stub.identity = SimpleNamespace(profile="")
|
||||
stub.projects = {
|
||||
uid: SimpleNamespace(
|
||||
id=uid,
|
||||
name=display_name,
|
||||
labels={},
|
||||
organization=None,
|
||||
)
|
||||
}
|
||||
elif provider_type == "kubernetes":
|
||||
stub.identity = SimpleNamespace(context=uid, cluster=uid)
|
||||
elif provider_type == "m365":
|
||||
stub.identity = SimpleNamespace(
|
||||
identity_type="",
|
||||
identity_id="",
|
||||
tenant_domain=uid,
|
||||
tenant_id="",
|
||||
)
|
||||
elif provider_type == "github":
|
||||
# generate_output assigns account fields only inside
|
||||
# isinstance(identity, Github*IdentityInfo) branches, so the stub
|
||||
# must carry a real GithubIdentityInfo instance.
|
||||
stub.identity = GithubIdentityInfo(
|
||||
account_id=uid,
|
||||
account_name=display_name,
|
||||
account_url="",
|
||||
)
|
||||
elif provider_type == "mongodbatlas":
|
||||
stub.identity = SimpleNamespace(
|
||||
organization_id=uid,
|
||||
organization_name=display_name,
|
||||
)
|
||||
elif provider_type == "iac":
|
||||
stub.provider_uid = uid
|
||||
elif provider_type == "oraclecloud":
|
||||
stub.identity = SimpleNamespace(
|
||||
tenancy_id=uid,
|
||||
tenancy_name=display_name,
|
||||
)
|
||||
elif provider_type == "alibabacloud":
|
||||
stub.identity = SimpleNamespace(
|
||||
identity_arn="",
|
||||
account_id=uid,
|
||||
account_name=display_name,
|
||||
)
|
||||
elif provider_type == "cloudflare":
|
||||
stub.identity = SimpleNamespace(
|
||||
audited_accounts=[uid],
|
||||
accounts=[],
|
||||
)
|
||||
elif provider_type == "openstack":
|
||||
stub.identity = SimpleNamespace(
|
||||
username="",
|
||||
project_id=uid,
|
||||
project_name=display_name,
|
||||
)
|
||||
elif provider_type == "googleworkspace":
|
||||
stub.identity = SimpleNamespace(
|
||||
delegated_user="",
|
||||
customer_id=uid,
|
||||
domain=display_name,
|
||||
)
|
||||
elif provider_type == "vercel":
|
||||
stub.identity = SimpleNamespace(
|
||||
team=None,
|
||||
user_id=uid,
|
||||
username=display_name,
|
||||
)
|
||||
elif provider_type == "okta":
|
||||
stub.identity = SimpleNamespace(
|
||||
org_domain=uid,
|
||||
client_id="",
|
||||
)
|
||||
|
||||
return stub
|
||||
@@ -1,6 +1,7 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
@@ -306,6 +307,55 @@ def _store_resources(
|
||||
return resource_instance, (resource_instance.uid, resource_instance.region)
|
||||
|
||||
|
||||
def _bulk_update_resource_failed_findings_counts(
|
||||
tenant_id: str,
|
||||
scan_id: str,
|
||||
resources_to_update: list[Resource],
|
||||
) -> None:
|
||||
"""Persist failed finding counters with stable row locking and retry."""
|
||||
if not resources_to_update:
|
||||
return
|
||||
|
||||
sorted_resources = sorted(
|
||||
resources_to_update, key=lambda resource: str(resource.id)
|
||||
)
|
||||
for start in range(0, len(sorted_resources), SCAN_DB_BATCH_SIZE):
|
||||
chunk = sorted_resources[start : start + SCAN_DB_BATCH_SIZE]
|
||||
chunk_ids = [resource.id for resource in chunk]
|
||||
|
||||
for attempt in range(CELERY_DEADLOCK_ATTEMPTS):
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
list(
|
||||
Resource.objects.select_for_update()
|
||||
.filter(id__in=chunk_ids)
|
||||
.order_by("id")
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
Resource.objects.bulk_update(
|
||||
chunk,
|
||||
["failed_findings_count"],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
)
|
||||
break
|
||||
except OperationalError:
|
||||
if attempt < CELERY_DEADLOCK_ATTEMPTS - 1:
|
||||
logger.warning(
|
||||
"Resource failed findings count update hit a database "
|
||||
"conflict on scan %s. Retrying chunk %s/%s "
|
||||
"(attempt %s/%s).",
|
||||
scan_id,
|
||||
start // SCAN_DB_BATCH_SIZE + 1,
|
||||
(len(sorted_resources) + SCAN_DB_BATCH_SIZE - 1)
|
||||
// SCAN_DB_BATCH_SIZE,
|
||||
attempt + 1,
|
||||
CELERY_DEADLOCK_ATTEMPTS,
|
||||
)
|
||||
time.sleep((0.1 * (2**attempt)) + random.uniform(0, 0.1))
|
||||
continue
|
||||
raise
|
||||
|
||||
|
||||
def _copy_compliance_requirement_rows(
|
||||
tenant_id: str, rows: list[dict[str, Any]]
|
||||
) -> None:
|
||||
@@ -1182,16 +1232,11 @@ def perform_prowler_scan(
|
||||
resources_to_update.append(resource_instance)
|
||||
|
||||
if resources_to_update:
|
||||
# Single rls_transaction wrapping the bulk_update (previously
|
||||
# `update_objects_in_batches` opened one rls_transaction per
|
||||
# chunk; for tenants with many resources this collapsed N
|
||||
# BEGINs/COMMITs into 1).
|
||||
with rls_transaction(tenant_id):
|
||||
Resource.objects.bulk_update(
|
||||
resources_to_update,
|
||||
["failed_findings_count"],
|
||||
batch_size=SCAN_DB_BATCH_SIZE,
|
||||
)
|
||||
_bulk_update_resource_failed_findings_counts(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
resources_to_update=resources_to_update,
|
||||
)
|
||||
|
||||
except ProviderDeletedException as e:
|
||||
logger.warning(str(e))
|
||||
@@ -1419,7 +1464,7 @@ def aggregate_findings(tenant_id: str, scan_id: str):
|
||||
)
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
scan_aggregations = {
|
||||
scan_aggregations = [
|
||||
ScanSummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
@@ -1444,9 +1489,18 @@ def aggregate_findings(tenant_id: str, scan_id: str):
|
||||
for agg in aggregation
|
||||
if agg["resources__service"] is not None
|
||||
and agg["resources__region"] is not None
|
||||
}
|
||||
# Upsert so re-runs (post-mute reaggregation) don't trip
|
||||
# `unique_scan_summary`; race-safe under concurrent writers.
|
||||
]
|
||||
# Needed sort so concurrent upserts acquire locks consistently
|
||||
scan_aggregations.sort(
|
||||
key=lambda summary: (
|
||||
summary.tenant_id,
|
||||
summary.scan_id,
|
||||
summary.check_id,
|
||||
summary.service,
|
||||
summary.severity,
|
||||
summary.region,
|
||||
)
|
||||
)
|
||||
ScanSummary.objects.bulk_create(
|
||||
scan_aggregations,
|
||||
batch_size=3000,
|
||||
|
||||
@@ -178,7 +178,9 @@ def _load_findings_for_requirement_checks(
|
||||
tenant_id (str): The tenant ID for Row-Level Security context.
|
||||
scan_id (str): The ID of the scan to retrieve findings for.
|
||||
check_ids (list[str]): List of check IDs to load findings for.
|
||||
prowler_provider: The initialized Prowler provider instance.
|
||||
prowler_provider: Credential-free provider metadata stub (see
|
||||
``tasks.jobs.reports.build_provider_metadata``) consumed by
|
||||
``FindingOutput.transform_api_finding``.
|
||||
findings_cache (dict, optional): Cache of already loaded findings.
|
||||
If provided, checks are first looked up in cache before querying database.
|
||||
total_counts_out (dict, optional): If provided, populated with
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from shutil import rmtree
|
||||
from uuid import uuid4
|
||||
|
||||
from api.compliance import (
|
||||
get_compliance_frameworks,
|
||||
@@ -10,14 +11,24 @@ from api.compliance import (
|
||||
from api.db_router import READ_REPLICA_ALIAS
|
||||
from api.db_utils import delete_related_daily_task, rls_transaction
|
||||
from api.decorators import handle_provider_deletion, set_tenant
|
||||
from api.models import Finding, Integration, Provider, Scan, ScanSummary, StateChoices
|
||||
from api.models import (
|
||||
Finding,
|
||||
Integration,
|
||||
Provider,
|
||||
Scan,
|
||||
ScanSummary,
|
||||
StateChoices,
|
||||
Task,
|
||||
)
|
||||
from api.utils import initialize_prowler_provider
|
||||
from api.v1.serializers import ScanTaskSerializer
|
||||
from celery import chain, group, shared_task
|
||||
from celery import chain, group, shared_task, states
|
||||
from celery.utils.log import get_task_logger
|
||||
from config.celery import RLSTask
|
||||
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE, DJANGO_TMP_OUTPUT_DIRECTORY
|
||||
from django.db import transaction
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
from django_celery_results.models import TaskResult
|
||||
from prowler.lib.check.compliance_models import Compliance
|
||||
from prowler.lib.outputs.compliance.compliance import (
|
||||
process_universal_compliance_frameworks,
|
||||
@@ -85,6 +96,220 @@ from tasks.utils import (
|
||||
)
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
QUEUED_SCAN_TASK_STATE = "QUEUED"
|
||||
DISPATCHED_SCAN_TASK_STATES = (states.PENDING, states.STARTED, "PROGRESS")
|
||||
|
||||
|
||||
def _get_dispatched_provider_scan(tenant_id: str, provider_id: str):
|
||||
"""Return a scan that has already been dispatched for a provider."""
|
||||
executing_scan = (
|
||||
Scan.objects.select_for_update()
|
||||
.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
state=StateChoices.EXECUTING,
|
||||
)
|
||||
.order_by("-inserted_at")
|
||||
.first()
|
||||
)
|
||||
if executing_scan:
|
||||
return executing_scan
|
||||
|
||||
return (
|
||||
Scan.objects.select_for_update(of=("self",))
|
||||
.select_related("task__task_runner_task")
|
||||
.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
state__in=(StateChoices.AVAILABLE, StateChoices.SCHEDULED),
|
||||
task__isnull=False,
|
||||
task__task_runner_task__status__in=DISPATCHED_SCAN_TASK_STATES,
|
||||
)
|
||||
.order_by("-inserted_at")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def _get_queued_provider_scan(tenant_id: str, provider_id: str):
|
||||
"""Return the next DB-queued scan for a provider."""
|
||||
return (
|
||||
Scan.objects.select_for_update(of=("self",))
|
||||
.select_related("task__task_runner_task")
|
||||
.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task__isnull=False,
|
||||
task__task_runner_task__status=QUEUED_SCAN_TASK_STATE,
|
||||
)
|
||||
.order_by("inserted_at", "id")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def get_active_provider_scan(tenant_id: str, provider_id: str):
|
||||
"""Return a dispatched or DB-queued scan for a provider."""
|
||||
return _get_dispatched_provider_scan(
|
||||
tenant_id, provider_id
|
||||
) or _get_queued_provider_scan(tenant_id, provider_id)
|
||||
|
||||
|
||||
def create_scan_task_record(
|
||||
tenant_id: str,
|
||||
task_id: str,
|
||||
task_name: str = "scan-perform",
|
||||
task_status: str | None = states.PENDING,
|
||||
) -> Task:
|
||||
if task_status is None:
|
||||
task_status = states.PENDING
|
||||
|
||||
task_result, _ = TaskResult.objects.update_or_create(
|
||||
task_id=str(task_id),
|
||||
defaults={"status": task_status, "task_name": task_name},
|
||||
)
|
||||
prowler_task, _ = Task.objects.update_or_create(
|
||||
id=str(task_id),
|
||||
tenant_id=tenant_id,
|
||||
defaults={"task_runner_task": task_result},
|
||||
)
|
||||
return prowler_task
|
||||
|
||||
|
||||
def enqueue_scan_execution_on_commit(
|
||||
tenant_id: str,
|
||||
scan: Scan,
|
||||
task_id: str,
|
||||
) -> None:
|
||||
transaction.on_commit(
|
||||
lambda: perform_scan_task.apply_async(
|
||||
kwargs={
|
||||
"tenant_id": str(tenant_id),
|
||||
"scan_id": str(scan.id),
|
||||
"provider_id": str(scan.provider_id),
|
||||
},
|
||||
task_id=str(task_id),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _get_queued_scheduled_scan(tenant_id: str, provider_id: str):
|
||||
return (
|
||||
Scan.objects.select_for_update(of=("self",))
|
||||
.select_related("task__task_runner_task")
|
||||
.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task__isnull=False,
|
||||
task__task_runner_task__status=QUEUED_SCAN_TASK_STATE,
|
||||
)
|
||||
.order_by("inserted_at", "id")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def _get_or_create_queued_scheduled_scan(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
periodic_task_instance: PeriodicTask,
|
||||
scheduled_at: datetime,
|
||||
) -> Scan:
|
||||
queued_scan = _get_queued_scheduled_scan(tenant_id, provider_id)
|
||||
if queued_scan:
|
||||
return queued_scan
|
||||
|
||||
task_id = str(uuid4())
|
||||
queued_task = create_scan_task_record(
|
||||
tenant_id=tenant_id,
|
||||
task_id=task_id,
|
||||
task_status=QUEUED_SCAN_TASK_STATE,
|
||||
)
|
||||
return Scan.objects.create(
|
||||
tenant_id=tenant_id,
|
||||
name="Daily scheduled scan",
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduled_at=scheduled_at,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
task=queued_task,
|
||||
)
|
||||
|
||||
|
||||
def _dispatch_next_queued_provider_scan(tenant_id: str, provider_id: str):
|
||||
with rls_transaction(tenant_id):
|
||||
if not Provider.objects.select_for_update().filter(pk=provider_id).exists():
|
||||
return None
|
||||
|
||||
if _get_dispatched_provider_scan(tenant_id, provider_id):
|
||||
return None
|
||||
|
||||
queued_scan = _get_queued_provider_scan(tenant_id, provider_id)
|
||||
if not queued_scan or not queued_scan.task:
|
||||
return None
|
||||
|
||||
task_result = queued_scan.task.task_runner_task
|
||||
task_result.status = states.PENDING
|
||||
task_result.task_name = "scan-perform"
|
||||
task_result.save(update_fields=["status", "task_name"])
|
||||
enqueue_scan_execution_on_commit(
|
||||
tenant_id=tenant_id,
|
||||
scan=queued_scan,
|
||||
task_id=str(queued_scan.task_id),
|
||||
)
|
||||
return queued_scan
|
||||
|
||||
|
||||
def _dispatch_next_queued_provider_scan_best_effort(
|
||||
tenant_id: str, provider_id: str
|
||||
) -> None:
|
||||
try:
|
||||
_dispatch_next_queued_provider_scan(tenant_id, provider_id)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to dispatch next queued scan for provider %s", provider_id
|
||||
)
|
||||
|
||||
|
||||
def _get_or_create_next_scheduled_scan(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
periodic_task_instance: PeriodicTask,
|
||||
next_scan_datetime: datetime,
|
||||
) -> Scan:
|
||||
interval = periodic_task_instance.interval
|
||||
now = datetime.now(UTC)
|
||||
while next_scan_datetime <= now:
|
||||
next_scan_datetime += timedelta(**{interval.period: interval.every})
|
||||
|
||||
return _get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
scheduled_at=next_scan_datetime,
|
||||
update_state=True,
|
||||
)
|
||||
|
||||
|
||||
def _ensure_next_scheduled_scan_best_effort(
|
||||
tenant_id: str,
|
||||
provider_id: str,
|
||||
periodic_task_instance: PeriodicTask,
|
||||
next_scan_datetime: datetime,
|
||||
) -> None:
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
_get_or_create_next_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
periodic_task_instance=periodic_task_instance,
|
||||
next_scan_datetime=next_scan_datetime,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to ensure next scheduled scan for provider %s", provider_id
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_orphan_scheduled_scans(
|
||||
@@ -117,6 +342,7 @@ def _cleanup_orphan_scheduled_scans(
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=scheduler_task_id,
|
||||
task__isnull=True,
|
||||
)
|
||||
|
||||
scheduled_scan_exists = Scan.objects.filter(
|
||||
@@ -292,16 +518,17 @@ def perform_scan_task(
|
||||
)
|
||||
return None
|
||||
|
||||
result = perform_prowler_scan(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
provider_id=provider_id,
|
||||
checks_to_execute=checks_to_execute,
|
||||
)
|
||||
|
||||
_perform_scan_complete_tasks(tenant_id, scan_id, provider_id)
|
||||
|
||||
return result
|
||||
try:
|
||||
result = perform_prowler_scan(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
provider_id=provider_id,
|
||||
checks_to_execute=checks_to_execute,
|
||||
)
|
||||
_perform_scan_complete_tasks(tenant_id, scan_id, provider_id)
|
||||
return result
|
||||
finally:
|
||||
_dispatch_next_queued_provider_scan_best_effort(tenant_id, provider_id)
|
||||
|
||||
|
||||
# acks_late=False: like scan-perform; a dropped run is re-fired by Beat on the next tick.
|
||||
@@ -335,7 +562,7 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
task_id = self.request.id
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
if not Provider.objects.filter(pk=provider_id).exists():
|
||||
if not Provider.objects.select_for_update().filter(pk=provider_id).exists():
|
||||
logger.warning(
|
||||
"scheduled scan-perform skipped: provider %s no longer exists "
|
||||
"(tenant=%s)",
|
||||
@@ -348,22 +575,6 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
periodic_task_instance = PeriodicTask.objects.get(
|
||||
name=f"scan-perform-scheduled-{provider_id}"
|
||||
)
|
||||
executing_scan = (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.EXECUTING,
|
||||
)
|
||||
.order_by("-started_at")
|
||||
.first()
|
||||
)
|
||||
if executing_scan:
|
||||
logger.warning(
|
||||
f"Scheduled scan already executing for provider {provider_id}. Skipping."
|
||||
)
|
||||
return ScanTaskSerializer(instance=executing_scan).data
|
||||
|
||||
executed_scan = Scan.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
@@ -388,6 +599,26 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
)
|
||||
|
||||
active_scan = get_active_provider_scan(tenant_id, provider_id)
|
||||
if active_scan:
|
||||
logger.warning(
|
||||
"Scan already queued or executing for provider %s. Queueing scheduled run.",
|
||||
provider_id,
|
||||
)
|
||||
queued_scheduled_scan = _get_or_create_queued_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
periodic_task_instance=periodic_task_instance,
|
||||
scheduled_at=current_scan_datetime,
|
||||
)
|
||||
_get_or_create_next_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
periodic_task_instance=periodic_task_instance,
|
||||
next_scan_datetime=next_scan_datetime,
|
||||
)
|
||||
return ScanTaskSerializer(instance=queued_scheduled_scan).data
|
||||
|
||||
scan_instance = _get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
@@ -403,24 +634,16 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str):
|
||||
scan_id=str(scan_instance.id),
|
||||
provider_id=provider_id,
|
||||
)
|
||||
_perform_scan_complete_tasks(tenant_id, str(scan_instance.id), provider_id)
|
||||
return result
|
||||
finally:
|
||||
with rls_transaction(tenant_id):
|
||||
now = datetime.now(UTC)
|
||||
if next_scan_datetime <= now:
|
||||
interval_delta = timedelta(**{interval.period: interval.every})
|
||||
while next_scan_datetime <= now:
|
||||
next_scan_datetime += interval_delta
|
||||
_get_or_create_scheduled_scan(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
scheduler_task_id=periodic_task_instance.id,
|
||||
scheduled_at=next_scan_datetime,
|
||||
update_state=True,
|
||||
)
|
||||
|
||||
_perform_scan_complete_tasks(tenant_id, str(scan_instance.id), provider_id)
|
||||
|
||||
return result
|
||||
_ensure_next_scheduled_scan_best_effort(
|
||||
tenant_id=tenant_id,
|
||||
provider_id=provider_id,
|
||||
periodic_task_instance=periodic_task_instance,
|
||||
next_scan_datetime=next_scan_datetime,
|
||||
)
|
||||
_dispatch_next_queued_provider_scan_best_effort(tenant_id, provider_id)
|
||||
|
||||
|
||||
@shared_task(name="scan-summary", queue="overview")
|
||||
|
||||
@@ -2,8 +2,10 @@ from contextlib import nullcontext
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from api.db_utils import rls_transaction
|
||||
from api.models import (
|
||||
AttackPathsScan,
|
||||
Finding,
|
||||
@@ -15,6 +17,7 @@ from api.models import (
|
||||
StatusChoices,
|
||||
Task,
|
||||
)
|
||||
from django.db import DEFAULT_DB_ALIAS
|
||||
from django_celery_results.models import TaskResult
|
||||
from prowler.lib.check.models import Severity
|
||||
from tasks.jobs.attack_paths import findings as findings_module
|
||||
@@ -1899,6 +1902,55 @@ def _make_session_ctx(session, call_order=None, name=None):
|
||||
return ctx
|
||||
|
||||
|
||||
class TestBuildChildId:
|
||||
def test_large_value_is_hashed_and_preserved_as_child_data(self):
|
||||
value = "x" * 22_796
|
||||
spec = sync_module.NormalizedList(
|
||||
"SomeLabel",
|
||||
"values",
|
||||
"SomeLabelValuesItem",
|
||||
"HAS_VALUES",
|
||||
)
|
||||
record = {
|
||||
"element_id": "elem-1",
|
||||
"labels": ["SomeLabel"],
|
||||
"props": {"values": [value]},
|
||||
}
|
||||
|
||||
_, parent, children, relationships = sync_module._node_to_sync_dict(
|
||||
record,
|
||||
"prov-1",
|
||||
sync_module._build_catalog_index([spec]),
|
||||
)
|
||||
|
||||
child = children[0]["row"]
|
||||
child_id = child["provider_element_id"]
|
||||
prefix = "prov-1::SomeLabelValuesItem::"
|
||||
assert parent["provider_element_id"] == "prov-1:elem-1"
|
||||
assert child["props"]["value"] == value
|
||||
assert len(child_id) == len(prefix) + 64
|
||||
assert value not in child_id
|
||||
assert relationships[0]["row"]["end_element_id"] == child_id
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider_id", "child_label", "value_key"),
|
||||
[
|
||||
("prov-2", "ChildLabel", "value"),
|
||||
("prov-1", "OtherChildLabel", "value"),
|
||||
("prov-1", "ChildLabel", "other-value"),
|
||||
],
|
||||
)
|
||||
def test_each_identity_component_changes_id(
|
||||
self, provider_id, child_label, value_key
|
||||
):
|
||||
child_id = sync_module._build_child_id("prov-1", "ChildLabel", "value")
|
||||
|
||||
assert sync_module._build_child_id("prov-1", "ChildLabel", "value") == child_id
|
||||
assert (
|
||||
sync_module._build_child_id(provider_id, child_label, value_key) != child_id
|
||||
)
|
||||
|
||||
|
||||
class TestSyncNodes:
|
||||
def test_iter_sink_batches_rejects_zero_batch_size(self):
|
||||
with pytest.raises(
|
||||
@@ -2244,6 +2296,58 @@ class TestInternetAnalysis:
|
||||
class TestAttackPathsDbUtilsGraphDataReady:
|
||||
"""Tests for db_utils functions related to graph_data_ready lifecycle."""
|
||||
|
||||
def test_database_defaults_allow_legacy_insert_without_cutover_columns(
|
||||
self, tenants_fixture, providers_fixture, scans_fixture
|
||||
):
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
provider.provider = Provider.ProviderChoices.AWS
|
||||
provider.save()
|
||||
scan = scans_fixture[0]
|
||||
scan.provider = provider
|
||||
scan.save()
|
||||
|
||||
attack_paths_scan_id = uuid4()
|
||||
now = datetime.now(tz=UTC)
|
||||
|
||||
with rls_transaction(str(tenant.id), using=DEFAULT_DB_ALIAS) as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO attack_paths_scans (
|
||||
id,
|
||||
inserted_at,
|
||||
updated_at,
|
||||
state,
|
||||
progress,
|
||||
graph_data_ready,
|
||||
started_at,
|
||||
tenant_id,
|
||||
provider_id,
|
||||
scan_id
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
[
|
||||
attack_paths_scan_id,
|
||||
now,
|
||||
now,
|
||||
StateChoices.SCHEDULED,
|
||||
0,
|
||||
False,
|
||||
now,
|
||||
tenant.id,
|
||||
provider.id,
|
||||
scan.id,
|
||||
],
|
||||
)
|
||||
|
||||
attack_paths_scan = AttackPathsScan.objects.get(id=attack_paths_scan_id)
|
||||
|
||||
assert attack_paths_scan.is_migrated is False
|
||||
assert (
|
||||
attack_paths_scan.sink_backend == AttackPathsScan.SinkBackendChoices.NEO4J
|
||||
)
|
||||
|
||||
def test_create_attack_paths_scan_first_scan_defaults_to_false(
|
||||
self, tenants_fixture, providers_fixture, scans_fixture
|
||||
):
|
||||
|
||||
@@ -5,6 +5,10 @@ from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.models import Integration
|
||||
from api.utils import prowler_integration_connection_test
|
||||
from django.db import OperationalError
|
||||
from prowler.lib.outputs.jira.exceptions.exceptions import (
|
||||
JiraRefreshTokenError,
|
||||
JiraRequiredCustomFieldsError,
|
||||
)
|
||||
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
|
||||
from prowler.providers.common.models import Connection
|
||||
from tasks.jobs.integrations import (
|
||||
@@ -1830,10 +1834,213 @@ class TestJiraIntegration:
|
||||
)
|
||||
|
||||
# Assertions
|
||||
assert result == {"created_count": 2, "failed_count": 1}
|
||||
assert result == {
|
||||
"created_count": 2,
|
||||
"failed_count": 1,
|
||||
"error": "Failed to create Jira issue.",
|
||||
}
|
||||
|
||||
# Verify error was logged for the failed finding
|
||||
mock_logger.error.assert_called_with("Failed to send finding finding-2 to Jira")
|
||||
mock_logger.error.assert_called_with("Failed to create Jira issue.")
|
||||
|
||||
@patch("tasks.jobs.integrations.rls_transaction")
|
||||
@patch("tasks.jobs.integrations.Finding")
|
||||
@patch("tasks.jobs.integrations.Integration")
|
||||
@patch("tasks.jobs.integrations.initialize_prowler_integration")
|
||||
@patch("tasks.jobs.integrations.logger")
|
||||
def test_send_findings_to_jira_preserves_exception_message(
|
||||
self,
|
||||
mock_logger,
|
||||
mock_initialize_integration,
|
||||
mock_integration_model,
|
||||
mock_finding_model,
|
||||
mock_rls_transaction,
|
||||
):
|
||||
"""Test Jira send exceptions are returned for UI polling."""
|
||||
tenant_id = "tenant-123"
|
||||
integration_id = "integration-456"
|
||||
project_key = "PROJ"
|
||||
issue_type = "Task"
|
||||
finding_ids = ["finding-1"]
|
||||
error_message = "Jira project requires custom fields: Team is required"
|
||||
|
||||
mock_rls_transaction.return_value.__enter__ = MagicMock()
|
||||
mock_rls_transaction.return_value.__exit__ = MagicMock()
|
||||
|
||||
integration = MagicMock()
|
||||
mock_integration_model.objects.get.return_value = integration
|
||||
|
||||
mock_jira_integration = MagicMock()
|
||||
|
||||
mock_jira_integration.send_finding.side_effect = JiraRequiredCustomFieldsError(
|
||||
message=error_message
|
||||
)
|
||||
mock_initialize_integration.return_value = mock_jira_integration
|
||||
|
||||
finding = MagicMock()
|
||||
finding.id = "finding-1"
|
||||
finding.check_id = "check_001"
|
||||
finding.severity = "high"
|
||||
finding.status = "FAIL"
|
||||
finding.status_extended = "Resource is not compliant"
|
||||
finding.compliance = {}
|
||||
finding.resources.exists.return_value = False
|
||||
finding.resources.first.return_value = None
|
||||
finding.scan.provider.provider = "aws"
|
||||
finding.check_metadata = {
|
||||
"checktitle": "Check Title",
|
||||
"risk": "High risk",
|
||||
"remediation": {"recommendation": {}, "code": {}},
|
||||
}
|
||||
mock_select_related = mock_finding_model.all_objects.select_related.return_value
|
||||
mock_finding_query = mock_select_related.prefetch_related.return_value
|
||||
mock_finding_query.get.return_value = finding
|
||||
|
||||
result = send_findings_to_jira(
|
||||
tenant_id, integration_id, project_key, issue_type, finding_ids
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"created_count": 0,
|
||||
"failed_count": 1,
|
||||
"error": error_message,
|
||||
}
|
||||
mock_logger.exception.assert_called_with(
|
||||
"Failed to send finding %s to Jira: %s",
|
||||
"finding-1",
|
||||
error_message,
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.integrations.rls_transaction")
|
||||
@patch("tasks.jobs.integrations.Finding")
|
||||
@patch("tasks.jobs.integrations.Integration")
|
||||
@patch("tasks.jobs.integrations.initialize_prowler_integration")
|
||||
@patch("tasks.jobs.integrations.logger")
|
||||
def test_send_findings_to_jira_preserves_refresh_token_error_message(
|
||||
self,
|
||||
mock_logger,
|
||||
mock_initialize_integration,
|
||||
mock_integration_model,
|
||||
mock_finding_model,
|
||||
mock_rls_transaction,
|
||||
):
|
||||
"""Test Jira refresh token exceptions return their UI-friendly message."""
|
||||
tenant_id = "tenant-123"
|
||||
integration_id = "integration-456"
|
||||
project_key = "PROJ"
|
||||
issue_type = "Task"
|
||||
finding_ids = ["finding-1"]
|
||||
error_message = "Failed to refresh the access token"
|
||||
|
||||
mock_rls_transaction.return_value.__enter__ = MagicMock()
|
||||
mock_rls_transaction.return_value.__exit__ = MagicMock()
|
||||
|
||||
integration = MagicMock()
|
||||
mock_integration_model.objects.get.return_value = integration
|
||||
|
||||
mock_jira_integration = MagicMock()
|
||||
|
||||
mock_jira_integration.send_finding.side_effect = JiraRefreshTokenError(
|
||||
message=error_message
|
||||
)
|
||||
mock_initialize_integration.return_value = mock_jira_integration
|
||||
|
||||
finding = MagicMock()
|
||||
finding.id = "finding-1"
|
||||
finding.check_id = "check_001"
|
||||
finding.severity = "high"
|
||||
finding.status = "FAIL"
|
||||
finding.status_extended = "Resource is not compliant"
|
||||
finding.compliance = {}
|
||||
finding.resources.exists.return_value = False
|
||||
finding.resources.first.return_value = None
|
||||
finding.scan.provider.provider = "aws"
|
||||
finding.check_metadata = {
|
||||
"checktitle": "Check Title",
|
||||
"risk": "High risk",
|
||||
"remediation": {"recommendation": {}, "code": {}},
|
||||
}
|
||||
mock_select_related = mock_finding_model.all_objects.select_related.return_value
|
||||
mock_finding_query = mock_select_related.prefetch_related.return_value
|
||||
mock_finding_query.get.return_value = finding
|
||||
|
||||
result = send_findings_to_jira(
|
||||
tenant_id, integration_id, project_key, issue_type, finding_ids
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"created_count": 0,
|
||||
"failed_count": 1,
|
||||
"error": error_message,
|
||||
}
|
||||
mock_logger.exception.assert_called_with(
|
||||
"Failed to send finding %s to Jira: %s",
|
||||
"finding-1",
|
||||
error_message,
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.integrations.rls_transaction")
|
||||
@patch("tasks.jobs.integrations.Finding")
|
||||
@patch("tasks.jobs.integrations.Integration")
|
||||
@patch("tasks.jobs.integrations.initialize_prowler_integration")
|
||||
@patch("tasks.jobs.integrations.logger")
|
||||
def test_send_findings_to_jira_sanitizes_unexpected_exception_message(
|
||||
self,
|
||||
mock_logger,
|
||||
mock_initialize_integration,
|
||||
mock_integration_model,
|
||||
mock_finding_model,
|
||||
mock_rls_transaction,
|
||||
):
|
||||
"""Test unexpected Jira send exceptions do not leak raw details to UI."""
|
||||
tenant_id = "tenant-123"
|
||||
integration_id = "integration-456"
|
||||
project_key = "PROJ"
|
||||
issue_type = "Task"
|
||||
finding_ids = ["finding-1"]
|
||||
|
||||
mock_rls_transaction.return_value.__enter__ = MagicMock()
|
||||
mock_rls_transaction.return_value.__exit__ = MagicMock()
|
||||
|
||||
integration = MagicMock()
|
||||
mock_integration_model.objects.get.return_value = integration
|
||||
|
||||
mock_jira_integration = MagicMock()
|
||||
mock_jira_integration.send_finding.side_effect = Exception("token=secret-value")
|
||||
mock_initialize_integration.return_value = mock_jira_integration
|
||||
|
||||
finding = MagicMock()
|
||||
finding.id = "finding-1"
|
||||
finding.check_id = "check_001"
|
||||
finding.severity = "high"
|
||||
finding.status = "FAIL"
|
||||
finding.status_extended = "Resource is not compliant"
|
||||
finding.compliance = {}
|
||||
finding.resources.exists.return_value = False
|
||||
finding.resources.first.return_value = None
|
||||
finding.scan.provider.provider = "aws"
|
||||
finding.check_metadata = {
|
||||
"checktitle": "Check Title",
|
||||
"risk": "High risk",
|
||||
"remediation": {"recommendation": {}, "code": {}},
|
||||
}
|
||||
mock_select_related = mock_finding_model.all_objects.select_related.return_value
|
||||
mock_finding_query = mock_select_related.prefetch_related.return_value
|
||||
mock_finding_query.get.return_value = finding
|
||||
|
||||
result = send_findings_to_jira(
|
||||
tenant_id, integration_id, project_key, issue_type, finding_ids
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"created_count": 0,
|
||||
"failed_count": 1,
|
||||
"error": "Failed to create Jira issue.",
|
||||
}
|
||||
assert "secret-value" not in result["error"]
|
||||
mock_logger.exception.assert_called_with(
|
||||
"Failed to send finding %s to Jira", "finding-1"
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.integrations.rls_transaction")
|
||||
@patch("tasks.jobs.integrations.Finding")
|
||||
|
||||
@@ -1044,10 +1044,10 @@ class TestStaleCleanupProtectionHelpers:
|
||||
class TestGenerateThreatscoreReportFunction:
|
||||
"""Test suite for generate_threatscore_report function."""
|
||||
|
||||
@patch("tasks.jobs.reports.base.initialize_prowler_provider")
|
||||
@patch("tasks.jobs.reports.base.build_provider_metadata")
|
||||
def test_generate_threatscore_report_exception_handling(
|
||||
self,
|
||||
mock_initialize_provider,
|
||||
mock_build_provider_metadata,
|
||||
tenants_fixture,
|
||||
scans_fixture,
|
||||
providers_fixture,
|
||||
@@ -1057,7 +1057,7 @@ class TestGenerateThreatscoreReportFunction:
|
||||
scan = scans_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
|
||||
mock_initialize_provider.side_effect = Exception("Test exception")
|
||||
mock_build_provider_metadata.side_effect = Exception("Test exception")
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
generate_threatscore_report(
|
||||
@@ -1167,7 +1167,6 @@ class TestGenerateComplianceReportsOptimized:
|
||||
assert result["cis"] == {"upload": False, "path": ""}
|
||||
mock_cis.assert_not_called()
|
||||
|
||||
@patch("api.utils.initialize_prowler_provider")
|
||||
@patch("tasks.jobs.report.rmtree")
|
||||
@patch("tasks.jobs.report._upload_to_s3")
|
||||
@patch("tasks.jobs.report.generate_cis_report")
|
||||
@@ -1194,7 +1193,6 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_cis,
|
||||
mock_upload_to_s3,
|
||||
mock_rmtree,
|
||||
mock_init_provider,
|
||||
):
|
||||
"""After each framework finishes, exclusive entries are evicted.
|
||||
|
||||
@@ -1223,7 +1221,6 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_aggregate_stats.return_value = {}
|
||||
mock_generate_output_dir.return_value = "/tmp/tenant/scan/x/prowler-out"
|
||||
mock_upload_to_s3.return_value = "s3://bucket/tenant/scan/x/report.pdf"
|
||||
mock_init_provider.return_value = Mock(name="prowler_provider")
|
||||
|
||||
# Seed the cache as if both frameworks had already loaded their
|
||||
# findings. We mutate it indirectly: each generator wrapper is a
|
||||
@@ -1266,7 +1263,7 @@ class TestGenerateComplianceReportsOptimized:
|
||||
"shared must remain in cache because ENS still needs it"
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.report.initialize_prowler_provider")
|
||||
@patch("tasks.jobs.report.build_provider_metadata")
|
||||
@patch("tasks.jobs.report.rmtree")
|
||||
@patch("tasks.jobs.report._upload_to_s3")
|
||||
@patch("tasks.jobs.report.generate_cis_report")
|
||||
@@ -1279,7 +1276,7 @@ class TestGenerateComplianceReportsOptimized:
|
||||
@patch("tasks.jobs.report.Compliance.get_bulk")
|
||||
@patch("tasks.jobs.report.Provider.objects.get")
|
||||
@patch("tasks.jobs.report.ScanSummary.objects.filter")
|
||||
def test_prowler_provider_initialized_once(
|
||||
def test_provider_metadata_built_once(
|
||||
self,
|
||||
mock_scan_summary_filter,
|
||||
mock_provider_get,
|
||||
@@ -1293,11 +1290,11 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_cis,
|
||||
mock_upload_to_s3,
|
||||
mock_rmtree,
|
||||
mock_init_provider,
|
||||
mock_build_metadata,
|
||||
):
|
||||
"""``initialize_prowler_provider`` must be called exactly once for
|
||||
the whole batch (PROWLER-1733). Previously each generator re-init'd
|
||||
the SDK provider in ``_load_compliance_data`` → 5 inits per scan.
|
||||
"""``build_provider_metadata`` must be called exactly once for the
|
||||
whole batch and its result shared across all 5 reports
|
||||
(PROWLER-1733 / PROWLER-2145).
|
||||
"""
|
||||
mock_scan_summary_filter.return_value.exists.return_value = True
|
||||
mock_provider_get.return_value = Mock(uid="provider-uid", provider="aws")
|
||||
@@ -1306,7 +1303,7 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_aggregate_stats.return_value = {}
|
||||
mock_generate_output_dir.return_value = "/tmp/tenant/scan/x/prowler-out"
|
||||
mock_upload_to_s3.return_value = "s3://bucket/tenant/scan/x/report.pdf"
|
||||
mock_init_provider.return_value = Mock(name="prowler_provider")
|
||||
mock_build_metadata.return_value = Mock(name="prowler_provider")
|
||||
|
||||
generate_compliance_reports(
|
||||
tenant_id=str(uuid.uuid4()),
|
||||
@@ -1325,14 +1322,14 @@ class TestGenerateComplianceReportsOptimized:
|
||||
mock_nis2.assert_called_once()
|
||||
mock_csa.assert_called_once()
|
||||
mock_cis.assert_called_once()
|
||||
# …but the SDK provider was initialized only once.
|
||||
assert mock_init_provider.call_count == 1, (
|
||||
f"expected 1 init, got {mock_init_provider.call_count} "
|
||||
# …but the provider metadata stub was built only once.
|
||||
assert mock_build_metadata.call_count == 1, (
|
||||
f"expected 1 build, got {mock_build_metadata.call_count} "
|
||||
f"(prowler_provider must be shared across reports)"
|
||||
)
|
||||
|
||||
# The shared instance must reach every wrapper as kwargs.
|
||||
shared = mock_init_provider.return_value
|
||||
shared = mock_build_metadata.return_value
|
||||
for mock_wrapper in (
|
||||
mock_threatscore,
|
||||
mock_ens,
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Tests for the credential-free provider metadata stub (PROWLER-2145).
|
||||
|
||||
Every provider object used here is a plain ``SimpleNamespace`` WITHOUT a
|
||||
``secret`` attribute: any code path trying to read ``provider.secret`` (the
|
||||
coupling these tests guard against) would raise ``AttributeError`` and fail
|
||||
the test. No database is required.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from api.models import Provider
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
from prowler.providers.github.models import GithubIdentityInfo
|
||||
from tasks.jobs.reports import build_provider_metadata
|
||||
|
||||
PROVIDER_UID = "provider-uid-123"
|
||||
PROVIDER_ALIAS = "my-provider-alias"
|
||||
|
||||
|
||||
def _provider_row(provider_type: str, alias: str | None = PROVIDER_ALIAS):
|
||||
"""Mimic the Provider DB row attributes read by build_provider_metadata."""
|
||||
return SimpleNamespace(provider=provider_type, uid=PROVIDER_UID, alias=alias)
|
||||
|
||||
|
||||
class TestBuildProviderMetadata:
|
||||
@pytest.mark.parametrize("provider_type", Provider.ProviderChoices.values)
|
||||
def test_every_provider_type_gets_safe_defaults(self, provider_type):
|
||||
stub = build_provider_metadata(_provider_row(provider_type))
|
||||
|
||||
assert stub.type == provider_type
|
||||
assert isinstance(stub.auth_method, str)
|
||||
assert hasattr(stub, "identity")
|
||||
|
||||
def test_aws_identity_account_is_uid(self):
|
||||
stub = build_provider_metadata(_provider_row("aws"))
|
||||
assert stub.identity.account == PROVIDER_UID
|
||||
|
||||
def test_azure_identity_covers_generate_output_accesses(self):
|
||||
stub = build_provider_metadata(_provider_row("azure"))
|
||||
|
||||
# generate_output indexes tenant_ids[0] and reads these directly.
|
||||
assert stub.identity.tenant_ids
|
||||
assert stub.identity.identity_type == ""
|
||||
assert stub.identity.identity_id == ""
|
||||
assert stub.identity.subscriptions == {PROVIDER_UID: PROVIDER_ALIAS}
|
||||
|
||||
def test_gcp_projects_keyed_by_uid(self):
|
||||
stub = build_provider_metadata(_provider_row("gcp"))
|
||||
|
||||
project = stub.projects[PROVIDER_UID]
|
||||
assert project.id == PROVIDER_UID
|
||||
assert project.name == PROVIDER_ALIAS
|
||||
assert project.labels == {}
|
||||
# generate_output calls getattr(project, "organization") without a
|
||||
# default, so the attribute must exist (None skips the org branch).
|
||||
assert project.organization is None
|
||||
|
||||
def test_kubernetes_identity_context_and_cluster(self):
|
||||
stub = build_provider_metadata(_provider_row("kubernetes"))
|
||||
assert stub.identity.context == PROVIDER_UID
|
||||
assert stub.identity.cluster == PROVIDER_UID
|
||||
|
||||
def test_github_identity_is_real_identity_info(self):
|
||||
# generate_output only assigns account fields inside
|
||||
# isinstance(identity, Github*IdentityInfo) branches.
|
||||
stub = build_provider_metadata(_provider_row("github"))
|
||||
assert isinstance(stub.identity, GithubIdentityInfo)
|
||||
assert stub.identity.account_id == PROVIDER_UID
|
||||
assert stub.identity.account_name == PROVIDER_ALIAS
|
||||
|
||||
def test_iac_provider_uid(self):
|
||||
stub = build_provider_metadata(_provider_row("iac"))
|
||||
assert stub.provider_uid == PROVIDER_UID
|
||||
|
||||
def test_alias_falls_back_to_uid(self):
|
||||
stub = build_provider_metadata(_provider_row("azure", alias=None))
|
||||
assert stub.identity.subscriptions == {PROVIDER_UID: PROVIDER_UID}
|
||||
|
||||
|
||||
def _check_metadata_dict(provider_type: str, check_id: str) -> dict:
|
||||
return {
|
||||
"provider": provider_type,
|
||||
"checkid": check_id,
|
||||
"checktitle": "Test check title",
|
||||
"checktype": [],
|
||||
# CheckMetadata validates ServiceName == check_id.split("_")[0]
|
||||
"servicename": check_id.split("_")[0],
|
||||
"subservicename": "",
|
||||
"severity": "high",
|
||||
"resourcetype": "resource-type",
|
||||
"description": "",
|
||||
"risk": "",
|
||||
"relatedurl": "",
|
||||
"remediation": {
|
||||
"recommendation": {"text": "", "url": ""},
|
||||
"code": {"nativeiac": "", "terraform": "", "cli": "", "other": ""},
|
||||
},
|
||||
"resourceidtemplate": "",
|
||||
"categories": [],
|
||||
"dependson": [],
|
||||
"relatedto": [],
|
||||
"notes": "",
|
||||
}
|
||||
|
||||
|
||||
class _FakeFinding:
|
||||
"""Attribute-faithful Finding stand-in.
|
||||
|
||||
A plain object instead of ``Mock``: only the attributes the Django model
|
||||
exposes exist, so any new provider-attribute read in generate_output
|
||||
(e.g. cloudflare's ``getattr(finding, "account_id", ...)``) hits the
|
||||
same missing-attribute path it would hit in production instead of being
|
||||
masked by Mock auto-created attributes.
|
||||
"""
|
||||
|
||||
|
||||
def _finding_model(provider_type: str, check_id: str, region: str):
|
||||
"""Mimic the Django Finding row attributes read by transform_api_finding."""
|
||||
resource = SimpleNamespace(
|
||||
uid="resource-uid",
|
||||
name="resource-name",
|
||||
metadata="{}",
|
||||
details="",
|
||||
region=region,
|
||||
tags=SimpleNamespace(all=lambda: []),
|
||||
)
|
||||
finding = _FakeFinding()
|
||||
finding.resources = SimpleNamespace(first=lambda: resource)
|
||||
finding.check_metadata = _check_metadata_dict(provider_type, check_id)
|
||||
finding.status = "FAIL"
|
||||
finding.status_extended = "failed for testing"
|
||||
finding.muted = False
|
||||
return finding
|
||||
|
||||
|
||||
_FINDING_REGION = "region-x"
|
||||
|
||||
# Expected (account_uid, region) of the transformed finding per provider
|
||||
# type, with resource.region = _FINDING_REGION. Keyed by every
|
||||
# Provider.ProviderChoices value so that adding a new provider type without
|
||||
# extending build_provider_metadata (and this table) fails the test below
|
||||
# instead of breaking PDF generation at runtime.
|
||||
_EXPECTED_TRANSFORM = {
|
||||
"aws": (PROVIDER_UID, _FINDING_REGION),
|
||||
"azure": (PROVIDER_UID, _FINDING_REGION),
|
||||
"gcp": (PROVIDER_UID, _FINDING_REGION),
|
||||
# transform_api_finding strips the "namespace: " prefix and
|
||||
# generate_output re-adds it.
|
||||
"kubernetes": (PROVIDER_UID, f"namespace: {_FINDING_REGION}"),
|
||||
"m365": (PROVIDER_UID, _FINDING_REGION),
|
||||
# For GitHub the owner comes from resource.region.
|
||||
"github": (_FINDING_REGION, _FINDING_REGION),
|
||||
"mongodbatlas": (PROVIDER_UID, _FINDING_REGION),
|
||||
"iac": (PROVIDER_UID, _FINDING_REGION),
|
||||
"oraclecloud": (PROVIDER_UID, _FINDING_REGION),
|
||||
"alibabacloud": (PROVIDER_UID, _FINDING_REGION),
|
||||
# Cloudflare uses the zone name (falls back to resource.name) as region.
|
||||
"cloudflare": (PROVIDER_UID, "resource-name"),
|
||||
"openstack": (PROVIDER_UID, _FINDING_REGION),
|
||||
"image": ("image", _FINDING_REGION),
|
||||
"googleworkspace": (PROVIDER_UID, _FINDING_REGION),
|
||||
"vercel": (PROVIDER_UID, "global"),
|
||||
"okta": (PROVIDER_UID, "global"),
|
||||
}
|
||||
|
||||
|
||||
class TestTransformApiFindingWithMetadataStub:
|
||||
"""transform_api_finding must work end-to-end with the stub — i.e.
|
||||
without a credentialed SDK provider — for EVERY API provider type."""
|
||||
|
||||
@pytest.mark.parametrize("provider_type", Provider.ProviderChoices.values)
|
||||
def test_transform_with_stub(self, provider_type):
|
||||
assert provider_type in _EXPECTED_TRANSFORM, (
|
||||
f"New provider type {provider_type!r}: add a branch to "
|
||||
f"build_provider_metadata covering the attributes read by "
|
||||
f"FindingOutput.generate_output, then add its expected "
|
||||
f"(account_uid, region) here."
|
||||
)
|
||||
expected_account_uid, expected_region = _EXPECTED_TRANSFORM[provider_type]
|
||||
|
||||
stub = build_provider_metadata(_provider_row(provider_type))
|
||||
check_id = f"{provider_type}_test_check"
|
||||
finding_model = _finding_model(provider_type, check_id, _FINDING_REGION)
|
||||
|
||||
output = FindingOutput.transform_api_finding(finding_model, stub)
|
||||
|
||||
assert output.check_id == check_id
|
||||
assert output.status == "FAIL"
|
||||
assert output.account_uid == expected_account_uid
|
||||
assert output.region == expected_region
|
||||
assert output.resource_name
|
||||
assert output.resource_uid
|
||||
@@ -21,11 +21,13 @@ from api.models import (
|
||||
StateChoices,
|
||||
StatusChoices,
|
||||
)
|
||||
from django.db import IntegrityError, OperationalError
|
||||
from prowler.lib.check.models import Severity
|
||||
from prowler.lib.outputs.finding import Status
|
||||
from tasks.jobs.scan import (
|
||||
_ATTACK_SURFACE_MAPPING_CACHE,
|
||||
_aggregate_findings_by_region,
|
||||
_bulk_update_resource_failed_findings_counts,
|
||||
_copy_compliance_requirement_rows,
|
||||
_create_compliance_summaries,
|
||||
_create_finding_delta,
|
||||
@@ -858,6 +860,98 @@ class TestPerformScan:
|
||||
# Assert that failed_findings_count was reset to 0 during the scan
|
||||
assert resource.failed_findings_count == 0
|
||||
|
||||
def test_failed_findings_count_update_retries_deadlock_in_stable_order(
|
||||
self, resources_fixture, monkeypatch
|
||||
):
|
||||
resource1, resource2, _ = resources_fixture
|
||||
tenant_id = str(resource1.tenant_id)
|
||||
resource1.failed_findings_count = 2
|
||||
resource2.failed_findings_count = 3
|
||||
resources_to_update = [resource2, resource1]
|
||||
expected_order = [
|
||||
str(resource.id)
|
||||
for resource in sorted(resources_to_update, key=lambda item: str(item.id))
|
||||
]
|
||||
original_bulk_update = Resource.objects.bulk_update
|
||||
bulk_update_calls = []
|
||||
|
||||
def flaky_bulk_update(objects, fields, batch_size=None):
|
||||
bulk_update_calls.append([str(obj.id) for obj in objects])
|
||||
if len(bulk_update_calls) == 1:
|
||||
raise OperationalError("deadlock detected")
|
||||
return original_bulk_update(objects, fields, batch_size=batch_size)
|
||||
|
||||
monkeypatch.setattr("tasks.jobs.scan.SCAN_DB_BATCH_SIZE", 10)
|
||||
monkeypatch.setattr(Resource.objects, "bulk_update", flaky_bulk_update)
|
||||
|
||||
_bulk_update_resource_failed_findings_counts(
|
||||
tenant_id=tenant_id,
|
||||
scan_id="scan-id",
|
||||
resources_to_update=resources_to_update,
|
||||
)
|
||||
|
||||
resource1.refresh_from_db()
|
||||
resource2.refresh_from_db()
|
||||
assert resource1.failed_findings_count == 2
|
||||
assert resource2.failed_findings_count == 3
|
||||
assert bulk_update_calls == [expected_order, expected_order]
|
||||
|
||||
def test_failed_findings_count_update_does_not_retry_integrity_error(
|
||||
self, resources_fixture, monkeypatch
|
||||
):
|
||||
resource, *_ = resources_fixture
|
||||
resource.failed_findings_count = 2
|
||||
bulk_update_calls = []
|
||||
sleep_calls = []
|
||||
|
||||
def failing_bulk_update(objects, fields, batch_size=None):
|
||||
bulk_update_calls.append([str(obj.id) for obj in objects])
|
||||
raise IntegrityError("constraint violation")
|
||||
|
||||
monkeypatch.setattr(Resource.objects, "bulk_update", failing_bulk_update)
|
||||
monkeypatch.setattr("tasks.jobs.scan.time.sleep", sleep_calls.append)
|
||||
|
||||
with pytest.raises(IntegrityError, match="constraint violation"):
|
||||
_bulk_update_resource_failed_findings_counts(
|
||||
tenant_id=str(resource.tenant_id),
|
||||
scan_id="scan-id",
|
||||
resources_to_update=[resource],
|
||||
)
|
||||
|
||||
assert len(bulk_update_calls) == 1
|
||||
assert sleep_calls == []
|
||||
|
||||
def test_failed_findings_count_update_adds_jitter_to_retry_backoff(
|
||||
self, resources_fixture, monkeypatch
|
||||
):
|
||||
from tasks.jobs import scan as scan_jobs
|
||||
|
||||
resource, *_ = resources_fixture
|
||||
resource.failed_findings_count = 2
|
||||
bulk_update_calls = []
|
||||
sleep_calls = []
|
||||
original_bulk_update = Resource.objects.bulk_update
|
||||
|
||||
def flaky_bulk_update(objects, fields, batch_size=None):
|
||||
bulk_update_calls.append([str(obj.id) for obj in objects])
|
||||
if len(bulk_update_calls) == 1:
|
||||
raise OperationalError("deadlock detected")
|
||||
return original_bulk_update(objects, fields, batch_size=batch_size)
|
||||
|
||||
monkeypatch.setattr(Resource.objects, "bulk_update", flaky_bulk_update)
|
||||
monkeypatch.setattr(scan_jobs, "random", MagicMock())
|
||||
scan_jobs.random.uniform.return_value = 0.037
|
||||
monkeypatch.setattr("tasks.jobs.scan.time.sleep", sleep_calls.append)
|
||||
|
||||
_bulk_update_resource_failed_findings_counts(
|
||||
tenant_id=str(resource.tenant_id),
|
||||
scan_id="scan-id",
|
||||
resources_to_update=[resource],
|
||||
)
|
||||
|
||||
scan_jobs.random.uniform.assert_called_once_with(0, 0.1)
|
||||
assert sleep_calls == [0.137]
|
||||
|
||||
def test_perform_prowler_scan_with_active_mute_rules(
|
||||
self,
|
||||
tenants_fixture,
|
||||
@@ -3558,6 +3652,95 @@ class TestAggregateFindings:
|
||||
regions = {s.region for s in summaries}
|
||||
assert regions == {"us-east-1", "us-west-2"}
|
||||
|
||||
@patch("tasks.jobs.scan.Finding.objects.filter")
|
||||
@patch("tasks.jobs.scan.ScanSummary.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_findings_orders_upserts_by_conflict_key(
|
||||
self, mock_rls_transaction, mock_bulk_create, mock_findings_filter
|
||||
):
|
||||
"""Scan summaries must use a stable lock order for concurrent upserts."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
scan_id = str(uuid.uuid4())
|
||||
counts = {
|
||||
"fail": 1,
|
||||
"_pass": 0,
|
||||
"muted_count": 0,
|
||||
"total": 1,
|
||||
"new": 1,
|
||||
"changed": 0,
|
||||
"unchanged": 0,
|
||||
"fail_new": 1,
|
||||
"fail_changed": 0,
|
||||
"pass_new": 0,
|
||||
"pass_changed": 0,
|
||||
"muted_new": 0,
|
||||
"muted_changed": 0,
|
||||
}
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.values.return_value = mock_queryset
|
||||
mock_queryset.annotate.return_value = [
|
||||
{
|
||||
"check_id": "check-b",
|
||||
"resources__service": "s3",
|
||||
"severity": "high",
|
||||
"resources__region": "us-east-1",
|
||||
**counts,
|
||||
},
|
||||
{
|
||||
"check_id": "check-a",
|
||||
"resources__service": "sqs",
|
||||
"severity": "high",
|
||||
"resources__region": "us-east-1",
|
||||
**counts,
|
||||
},
|
||||
{
|
||||
"check_id": "check-a",
|
||||
"resources__service": "s3",
|
||||
"severity": "medium",
|
||||
"resources__region": "us-east-1",
|
||||
**counts,
|
||||
},
|
||||
{
|
||||
"check_id": "check-a",
|
||||
"resources__service": "s3",
|
||||
"severity": "high",
|
||||
"resources__region": "us-west-2",
|
||||
**counts,
|
||||
},
|
||||
{
|
||||
"check_id": "check-a",
|
||||
"resources__service": "s3",
|
||||
"severity": "high",
|
||||
"resources__region": "us-east-1",
|
||||
**counts,
|
||||
},
|
||||
]
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
mock_findings_filter.return_value = mock_queryset
|
||||
|
||||
aggregate_findings(tenant_id, scan_id)
|
||||
|
||||
summaries = mock_bulk_create.call_args.args[0]
|
||||
assert isinstance(summaries, list)
|
||||
conflict_keys = [
|
||||
(
|
||||
str(summary.tenant_id),
|
||||
str(summary.scan_id),
|
||||
summary.check_id,
|
||||
summary.service,
|
||||
summary.severity,
|
||||
summary.region,
|
||||
)
|
||||
for summary in summaries
|
||||
]
|
||||
assert len(conflict_keys) == 5
|
||||
assert conflict_keys == sorted(conflict_keys)
|
||||
|
||||
@patch("tasks.jobs.scan.Finding.objects.filter")
|
||||
@patch("tasks.jobs.scan.ScanSummary.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
|
||||
@@ -3,6 +3,7 @@ from contextlib import contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
from api.models import (
|
||||
@@ -14,11 +15,13 @@ from api.models import (
|
||||
Task,
|
||||
)
|
||||
from botocore.exceptions import ClientError
|
||||
from celery import states
|
||||
from django_celery_beat.models import IntervalSchedule, PeriodicTask
|
||||
from django_celery_results.models import TaskResult
|
||||
from tasks.jobs.lighthouse_providers import (
|
||||
_create_bedrock_client,
|
||||
_extract_bedrock_credentials,
|
||||
_LighthouseOpenAICompatibleNetworkBackend,
|
||||
)
|
||||
from tasks.tasks import (
|
||||
DJANGO_TMP_OUTPUT_DIRECTORY,
|
||||
@@ -1565,7 +1568,7 @@ class TestCheckLighthouseProviderConnectionTask:
|
||||
(
|
||||
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
{"api_key": "sk-test123"},
|
||||
"https://openrouter.ai/api/v1",
|
||||
"https://93.184.216.34/api/v1",
|
||||
{"connected": True, "error": None},
|
||||
),
|
||||
(
|
||||
@@ -1640,7 +1643,7 @@ class TestCheckLighthouseProviderConnectionTask:
|
||||
(
|
||||
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
{"api_key": "sk-invalid"},
|
||||
"https://openrouter.ai/api/v1",
|
||||
"https://93.184.216.34/api/v1",
|
||||
openai.APIConnectionError(request=MagicMock()),
|
||||
),
|
||||
(
|
||||
@@ -1754,6 +1757,166 @@ class TestCheckLighthouseProviderConnectionTask:
|
||||
provider_cfg.refresh_from_db()
|
||||
assert provider_cfg.is_active is False
|
||||
|
||||
def test_openai_compatible_connection_rejects_metadata_base_url_without_request(
|
||||
self, tenants_fixture
|
||||
):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://169.254.169.254/latest/meta-data",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
eager_result = check_lighthouse_provider_connection_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result["connected"] is False
|
||||
assert "base url" in result["error"].lower()
|
||||
mock_openai.assert_not_called()
|
||||
provider_cfg.refresh_from_db()
|
||||
assert provider_cfg.is_active is False
|
||||
|
||||
def test_openai_compatible_connection_disables_redirects(self, tenants_fixture):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=False,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.return_value = MagicMock()
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = check_lighthouse_provider_connection_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result == {"connected": True, "error": None}
|
||||
http_client = mock_openai.call_args.kwargs["http_client"]
|
||||
assert http_client.follow_redirects is False
|
||||
assert http_client.trust_env is False
|
||||
|
||||
def test_openai_compatible_connection_masks_remote_http_error(
|
||||
self, tenants_fixture
|
||||
):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
remote_body = "<!DOCTYPE HTML><p>remote 404 body</p>"
|
||||
response = httpx.Response(
|
||||
404,
|
||||
request=httpx.Request("GET", "https://provider.example/v1/models"),
|
||||
)
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.side_effect = openai.NotFoundError(
|
||||
remote_body,
|
||||
response=response,
|
||||
body=remote_body,
|
||||
)
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = check_lighthouse_provider_connection_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result == {"connected": False, "error": "Provider connection failed"}
|
||||
assert remote_body not in result["error"]
|
||||
provider_cfg.refresh_from_db()
|
||||
assert provider_cfg.is_active is False
|
||||
|
||||
def test_openai_compatible_connection_masks_remote_auth_error(
|
||||
self, tenants_fixture
|
||||
):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
remote_body = {"error": {"message": "remote auth detail"}}
|
||||
response = httpx.Response(
|
||||
401,
|
||||
request=httpx.Request("GET", "https://provider.example/v1/models"),
|
||||
)
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.side_effect = openai.AuthenticationError(
|
||||
"Unauthorized",
|
||||
response=response,
|
||||
body=remote_body,
|
||||
)
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = check_lighthouse_provider_connection_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result == {"connected": False, "error": "API key is invalid or missing"}
|
||||
assert "remote auth detail" not in result["error"]
|
||||
provider_cfg.refresh_from_db()
|
||||
assert provider_cfg.is_active is False
|
||||
|
||||
def test_openai_compatible_network_backend_uses_validated_ip(self, monkeypatch):
|
||||
backend = _LighthouseOpenAICompatibleNetworkBackend()
|
||||
stream = MagicMock()
|
||||
|
||||
def resolve_to_public_ip(host, port):
|
||||
del host, port
|
||||
return ("93.184.216.34",)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tasks.jobs.lighthouse_providers.resolve_lighthouse_openai_compatible_host",
|
||||
resolve_to_public_ip,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.lighthouse_providers.httpcore.SyncBackend.connect_tcp",
|
||||
return_value=stream,
|
||||
) as mock_connect_tcp:
|
||||
result = backend.connect_tcp("provider.example", 443, timeout=1.0)
|
||||
|
||||
assert result is stream
|
||||
assert mock_connect_tcp.call_args.args[:2] == ("93.184.216.34", 443)
|
||||
assert mock_connect_tcp.call_args.kwargs["timeout"] == 1.0
|
||||
|
||||
def test_check_connection_provider_does_not_exist(self, tenants_fixture):
|
||||
"""Test that checking non-existent provider raises DoesNotExist."""
|
||||
non_existent_id = str(uuid.uuid4())
|
||||
@@ -1783,7 +1946,7 @@ class TestRefreshLighthouseProviderModelsTask:
|
||||
(
|
||||
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
{"api_key": "sk-test123"},
|
||||
"https://openrouter.ai/api/v1",
|
||||
"https://93.184.216.34/api/v1",
|
||||
{"model-1": "Model One", "model-2": "Model Two"},
|
||||
2,
|
||||
),
|
||||
@@ -1863,6 +2026,106 @@ class TestRefreshLighthouseProviderModelsTask:
|
||||
== expected_count
|
||||
)
|
||||
|
||||
def test_refresh_models_rejects_metadata_base_url_without_request(
|
||||
self, tenants_fixture
|
||||
):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://169.254.169.254/latest/meta-data",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.lighthouse_providers._fetch_openai_compatible_models"
|
||||
) as mock_fetch:
|
||||
eager_result = refresh_lighthouse_provider_models_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result["created"] == 0
|
||||
assert result["updated"] == 0
|
||||
assert result["deleted"] == 0
|
||||
assert "base url" in result["error"].lower()
|
||||
mock_fetch.assert_not_called()
|
||||
|
||||
def test_refresh_models_disables_redirects(self, tenants_fixture):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.return_value = MagicMock(data=[])
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = refresh_lighthouse_provider_models_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result["created"] == 0
|
||||
assert result["updated"] == 0
|
||||
assert result["deleted"] == 0
|
||||
http_client = mock_openai.call_args.kwargs["http_client"]
|
||||
assert http_client.follow_redirects is False
|
||||
assert http_client.trust_env is False
|
||||
|
||||
def test_refresh_models_masks_remote_http_error(self, tenants_fixture):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
remote_body = "<!DOCTYPE HTML><p>remote 404 body</p>"
|
||||
response = httpx.Response(
|
||||
404,
|
||||
request=httpx.Request("GET", "https://provider.example/v1/models"),
|
||||
)
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.side_effect = openai.NotFoundError(
|
||||
remote_body,
|
||||
response=response,
|
||||
body=remote_body,
|
||||
)
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = refresh_lighthouse_provider_models_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result["created"] == 0
|
||||
assert result["updated"] == 0
|
||||
assert result["deleted"] == 0
|
||||
assert result["error"] == "Provider connection failed"
|
||||
assert remote_body not in result["error"]
|
||||
|
||||
def test_refresh_models_mixed_operations(self, tenants_fixture):
|
||||
"""Test mixed create, update, and delete operations."""
|
||||
# Create provider configuration
|
||||
@@ -2286,6 +2549,51 @@ class TestCleanupOrphanScheduledScans:
|
||||
assert Scan.objects.filter(id=scheduled_scan.id).exists()
|
||||
assert Scan.objects.filter(id=available_scan_other_task.id).exists()
|
||||
|
||||
def test_cleanup_keeps_db_queued_scheduled_scans(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""DB-queued scheduled scans have a task and must not be deleted as orphans."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status="QUEUED",
|
||||
)
|
||||
queued_task = Task.objects.create(
|
||||
id=task_result.task_id,
|
||||
task_runner_task=task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
queued_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Queued scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
task=queued_task,
|
||||
)
|
||||
scheduled_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
deleted_count = _cleanup_orphan_scheduled_scans(
|
||||
tenant_id=str(tenant.id),
|
||||
provider_id=str(provider.id),
|
||||
scheduler_task_id=periodic_task.id,
|
||||
)
|
||||
|
||||
assert deleted_count == 0
|
||||
assert Scan.objects.filter(id=queued_scan.id).exists()
|
||||
assert Scan.objects.filter(id=scheduled_scan.id).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestPerformScheduledScanTask:
|
||||
@@ -2334,10 +2642,10 @@ class TestPerformScheduledScanTask:
|
||||
)
|
||||
return task_result
|
||||
|
||||
def test_skip_when_scheduled_scan_executing(
|
||||
def test_queues_scheduled_scan_when_scheduled_scan_is_executing(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Skip a scheduled run when another scheduled scan is already executing."""
|
||||
"""Queue a scheduled run when another scheduled scan is executing."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
@@ -2364,8 +2672,16 @@ class TestPerformScheduledScanTask:
|
||||
|
||||
mock_scan.assert_not_called()
|
||||
mock_complete_tasks.assert_not_called()
|
||||
assert result["id"] == str(executing_scan.id)
|
||||
assert result["state"] == StateChoices.EXECUTING
|
||||
assert result["id"] != str(executing_scan.id)
|
||||
assert result["state"] == StateChoices.AVAILABLE
|
||||
queued_scheduled_scan = Scan.objects.get(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
assert result["id"] == str(queued_scheduled_scan.id)
|
||||
assert queued_scheduled_scan.task.task_runner_task.status == "QUEUED"
|
||||
assert (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant.id,
|
||||
@@ -2373,7 +2689,133 @@ class TestPerformScheduledScanTask:
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
).count()
|
||||
== 0
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_queues_scheduled_scan_when_manual_scan_is_pending(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Queue one scheduled run when a manual scan is already dispatched."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
self._create_periodic_task(provider.id, tenant.id)
|
||||
task_id = str(uuid.uuid4())
|
||||
self._create_task_result(tenant.id, task_id)
|
||||
manual_task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status=states.PENDING,
|
||||
)
|
||||
manual_task = Task.objects.create(
|
||||
id=manual_task_result.task_id,
|
||||
task_runner_task=manual_task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
manual_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Manual scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task=manual_task,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan") as mock_scan,
|
||||
patch("tasks.tasks._perform_scan_complete_tasks") as mock_complete_tasks,
|
||||
self._override_task_request(perform_scheduled_scan_task, id=task_id),
|
||||
):
|
||||
result = perform_scheduled_scan_task.run(
|
||||
tenant_id=str(tenant.id), provider_id=str(provider.id)
|
||||
)
|
||||
|
||||
mock_scan.assert_not_called()
|
||||
mock_complete_tasks.assert_not_called()
|
||||
assert result["id"] != str(manual_scan.id)
|
||||
assert result["state"] == StateChoices.AVAILABLE
|
||||
queued_scheduled_scan = Scan.objects.get(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
assert result["id"] == str(queued_scheduled_scan.id)
|
||||
assert queued_scheduled_scan.task.task_runner_task.status == "QUEUED"
|
||||
scheduled_scan = Scan.objects.get(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.SCHEDULED,
|
||||
)
|
||||
assert scheduled_scan.scheduled_at > datetime.now(UTC)
|
||||
|
||||
def test_coalesces_scheduled_scan_when_one_is_already_queued(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
"""Reuse the existing queued scheduled scan instead of adding another."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
periodic_task = self._create_periodic_task(provider.id, tenant.id)
|
||||
task_id = str(uuid.uuid4())
|
||||
self._create_task_result(tenant.id, task_id)
|
||||
manual_task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status=states.PENDING,
|
||||
)
|
||||
manual_task = Task.objects.create(
|
||||
id=manual_task_result.task_id,
|
||||
task_runner_task=manual_task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Manual scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task=manual_task,
|
||||
)
|
||||
queued_task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status="QUEUED",
|
||||
)
|
||||
queued_task = Task.objects.create(
|
||||
id=queued_task_result.task_id,
|
||||
task_runner_task=queued_task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
queued_scheduled_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Daily scheduled scan",
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
scheduler_task_id=periodic_task.id,
|
||||
task=queued_task,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan") as mock_scan,
|
||||
patch("tasks.tasks._perform_scan_complete_tasks") as mock_complete_tasks,
|
||||
self._override_task_request(perform_scheduled_scan_task, id=task_id),
|
||||
):
|
||||
result = perform_scheduled_scan_task.run(
|
||||
tenant_id=str(tenant.id), provider_id=str(provider.id)
|
||||
)
|
||||
|
||||
mock_scan.assert_not_called()
|
||||
mock_complete_tasks.assert_not_called()
|
||||
assert result["id"] == str(queued_scheduled_scan.id)
|
||||
assert (
|
||||
Scan.objects.filter(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state=StateChoices.AVAILABLE,
|
||||
).count()
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_creates_next_scheduled_scan_after_completion(
|
||||
@@ -2435,6 +2877,41 @@ class TestPerformScheduledScanTask:
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_next_scheduled_scan_failure_does_not_mask_completed_scan(
|
||||
self, tenants_fixture, providers_fixture, caplog
|
||||
):
|
||||
"""Keep scheduled scan success when next-run creation fails."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
self._create_periodic_task(provider.id, tenant.id)
|
||||
task_id = str(uuid.uuid4())
|
||||
self._create_task_result(tenant.id, task_id)
|
||||
|
||||
def _complete_scan(tenant_id, scan_id, provider_id):
|
||||
scan_instance = Scan.objects.get(id=scan_id)
|
||||
scan_instance.state = StateChoices.COMPLETED
|
||||
scan_instance.save()
|
||||
return {"status": "ok"}
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan", side_effect=_complete_scan),
|
||||
patch("tasks.tasks._perform_scan_complete_tasks"),
|
||||
patch(
|
||||
"tasks.tasks._get_or_create_next_scheduled_scan",
|
||||
side_effect=RuntimeError("scheduler unavailable"),
|
||||
),
|
||||
patch("tasks.tasks._dispatch_next_queued_provider_scan") as mock_dispatch,
|
||||
self._override_task_request(perform_scheduled_scan_task, id=task_id),
|
||||
caplog.at_level("ERROR"),
|
||||
):
|
||||
result = perform_scheduled_scan_task.run(
|
||||
tenant_id=str(tenant.id), provider_id=str(provider.id)
|
||||
)
|
||||
|
||||
assert result == {"status": "ok"}
|
||||
mock_dispatch.assert_called_once_with(str(tenant.id), str(provider.id))
|
||||
assert "Failed to ensure next scheduled scan" in caplog.text
|
||||
|
||||
def test_dedupes_multiple_scheduled_scans_before_run(
|
||||
self, tenants_fixture, providers_fixture
|
||||
):
|
||||
@@ -2549,6 +3026,104 @@ class TestPerformScanTask:
|
||||
mock_scan.assert_not_called()
|
||||
mock_complete_tasks.assert_not_called()
|
||||
|
||||
def test_dispatches_next_queued_scan_after_completion(
|
||||
self,
|
||||
tenants_fixture,
|
||||
providers_fixture,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Dispatch the next queued scan for the provider after completion."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
current_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Running scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
queued_task_result = TaskResult.objects.create(
|
||||
task_id=str(uuid.uuid4()),
|
||||
task_name="scan-perform",
|
||||
status="QUEUED",
|
||||
)
|
||||
queued_task = Task.objects.create(
|
||||
id=queued_task_result.task_id,
|
||||
task_runner_task=queued_task_result,
|
||||
tenant_id=tenant.id,
|
||||
)
|
||||
queued_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Queued scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
task=queued_task,
|
||||
)
|
||||
|
||||
def _complete_scan(tenant_id, scan_id, provider_id, checks_to_execute=None):
|
||||
scan_instance = Scan.objects.get(id=scan_id)
|
||||
scan_instance.state = StateChoices.COMPLETED
|
||||
scan_instance.save()
|
||||
return {"status": "ok"}
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan", side_effect=_complete_scan),
|
||||
patch("tasks.tasks._perform_scan_complete_tasks"),
|
||||
patch("tasks.tasks.perform_scan_task.apply_async") as mock_apply_async,
|
||||
):
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
result = perform_scan_task.run(
|
||||
tenant_id=str(tenant.id),
|
||||
scan_id=str(current_scan.id),
|
||||
provider_id=str(provider.id),
|
||||
)
|
||||
|
||||
queued_task_result.refresh_from_db()
|
||||
assert result == {"status": "ok"}
|
||||
assert queued_task_result.status == states.PENDING
|
||||
mock_apply_async.assert_called_once_with(
|
||||
kwargs={
|
||||
"tenant_id": str(tenant.id),
|
||||
"scan_id": str(queued_scan.id),
|
||||
"provider_id": str(provider.id),
|
||||
},
|
||||
task_id=str(queued_task.id),
|
||||
)
|
||||
|
||||
def test_dispatch_failure_does_not_mask_completed_scan(
|
||||
self, tenants_fixture, providers_fixture, caplog
|
||||
):
|
||||
"""Keep scan success when queued dispatch fails after completion."""
|
||||
tenant = tenants_fixture[0]
|
||||
provider = providers_fixture[0]
|
||||
current_scan = Scan.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider,
|
||||
name="Running scan",
|
||||
trigger=Scan.TriggerChoices.MANUAL,
|
||||
state=StateChoices.AVAILABLE,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("tasks.tasks.perform_prowler_scan", return_value={"status": "ok"}),
|
||||
patch("tasks.tasks._perform_scan_complete_tasks"),
|
||||
patch(
|
||||
"tasks.tasks._dispatch_next_queued_provider_scan",
|
||||
side_effect=RuntimeError("dispatch unavailable"),
|
||||
) as mock_dispatch,
|
||||
caplog.at_level("ERROR"),
|
||||
):
|
||||
result = perform_scan_task.run(
|
||||
tenant_id=str(tenant.id),
|
||||
scan_id=str(current_scan.id),
|
||||
provider_id=str(provider.id),
|
||||
)
|
||||
|
||||
assert result == {"status": "ok"}
|
||||
mock_dispatch.assert_called_once_with(str(tenant.id), str(provider.id))
|
||||
assert "Failed to dispatch next queued scan" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestReaggregateAllFindingGroupSummaries:
|
||||
|
||||
@@ -103,6 +103,7 @@ def _get_or_create_scheduled_scan(
|
||||
trigger=Scan.TriggerChoices.SCHEDULED,
|
||||
state__in=(StateChoices.SCHEDULED, StateChoices.AVAILABLE),
|
||||
scheduler_task_id=scheduler_task_id,
|
||||
task__isnull=True,
|
||||
).order_by("scheduled_at", "inserted_at")
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
[tool.towncrier]
|
||||
directory = "changelog.d"
|
||||
filename = "CHANGELOG.md"
|
||||
start_string = "<!-- changelog: release notes start -->\n"
|
||||
title_format = "## [{version}] ({name})"
|
||||
issue_format = "[(#{issue})](https://github.com/prowler-cloud/prowler/pull/{issue})"
|
||||
template = "../.github/towncrier/template.md.jinja"
|
||||
underlines = ["", "", ""]
|
||||
ignore = [".gitkeep", "README.md"]
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "added"
|
||||
name = "🚀 Added"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "changed"
|
||||
name = "🔄 Changed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "deprecated"
|
||||
name = "⚠️ Deprecated"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "removed"
|
||||
name = "❌ Removed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "fixed"
|
||||
name = "🐞 Fixed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "security"
|
||||
name = "🔐 Security"
|
||||
showcontent = true
|
||||
@@ -4673,8 +4673,8 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler"
|
||||
version = "5.32.0"
|
||||
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=master#5dac8a0a53272e4db68c476fb969dc03e88beb68" }
|
||||
version = "5.33.0"
|
||||
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=v5.33#76a2d7bfe61a3ac6c96e497eded30a2978cd3ffe" }
|
||||
dependencies = [
|
||||
{ name = "alibabacloud-actiontrail20200706" },
|
||||
{ name = "alibabacloud-credentials" },
|
||||
@@ -4762,7 +4762,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler-api"
|
||||
version = "1.33.0"
|
||||
version = "1.34.2"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "cartography" },
|
||||
@@ -4862,7 +4862,7 @@ requires-dist = [
|
||||
{ name = "matplotlib", specifier = "==3.10.8" },
|
||||
{ name = "neo4j", specifier = "==6.1.0" },
|
||||
{ name = "openai", specifier = "==1.109.1" },
|
||||
{ name = "prowler", git = "https://github.com/prowler-cloud/prowler.git?rev=master" },
|
||||
{ name = "prowler", git = "https://github.com/prowler-cloud/prowler.git?rev=v5.33" },
|
||||
{ name = "psycopg2-binary", specifier = "==2.9.9" },
|
||||
{ name = "pytest-celery", extras = ["redis"], specifier = "==1.3.0" },
|
||||
{ name = "reportlab", specifier = "==4.4.10" },
|
||||
|
||||
@@ -39,6 +39,24 @@ The former build-time variables map to the new runtime variables as follows:
|
||||
|
||||
The build-time-only Sentry variables used for source-map upload — `SENTRY_ORG`, `SENTRY_PROJECT`, `SENTRY_AUTH_TOKEN`, and `SENTRY_RELEASE` — keep their names, as they are not part of the App's runtime configuration.
|
||||
|
||||
## Enabling Third-Party Integrations
|
||||
|
||||
Prowler App gates each optional third-party integration behind an explicit enable flag. When an integration is configured through its new `UI_*` variables, it loads only when its flag is set to the exact string `"true"`; any other value, including unset, leaves it off. This default-off behavior keeps a deployment free of third-party egress unless it opts in. Deployments still using the deprecated legacy variable names keep loading without the flag, for backward compatibility (see [Deprecated Names](#deprecated-names)).
|
||||
|
||||
| Integration | Enable flag | Required configuration when enabled |
|
||||
|-------------|-------------|-------------------------------------|
|
||||
| Sentry (error monitoring) | `UI_SENTRY_ENABLE` | `UI_SENTRY_DSN` |
|
||||
| Google Tag Manager | `UI_GOOGLE_TAG_MANAGER_ENABLE` | `UI_GOOGLE_TAG_MANAGER_ID` |
|
||||
| PostHog (product analytics) | `UI_POSTHOG_ENABLE` | `UI_POSTHOG_KEY` and `UI_POSTHOG_HOST` |
|
||||
|
||||
When an integration is enabled but its required configuration is missing, Prowler App fails fast at server startup with a clear error, so a misconfigured container never starts silently. A new `UI_*` value set while its enable flag is not `"true"` is ignored, and the server logs a one-time startup warning noting that the integration will not load. Legacy names follow the backward-compatible rule described in [Deprecated Names](#deprecated-names).
|
||||
|
||||
PostHog support is currently limited to configuration validation: Prowler App reads and validates the PostHog variables but does not yet load a PostHog client.
|
||||
|
||||
<Warning>
|
||||
Configuring an integration through the new `UI_*` variables now requires its enable flag. A deployment that adopted `UI_SENTRY_DSN` or `UI_GOOGLE_TAG_MANAGER_ID` must also set `UI_SENTRY_ENABLE=true` or `UI_GOOGLE_TAG_MANAGER_ENABLE=true` to keep the integration active. Deployments still using the legacy names (`NEXT_PUBLIC_*`, or `POSTHOG_KEY` and `POSTHOG_HOST`) keep working without the flag.
|
||||
</Warning>
|
||||
|
||||
## Upcoming Breaking Change
|
||||
|
||||
<Warning>
|
||||
@@ -49,5 +67,5 @@ Prowler API, Prowler SDK, and Prowler MCP Server have not yet adopted the conven
|
||||
|
||||
## Deprecated Names
|
||||
|
||||
- **Prowler App:** The bare server-side `SENTRY_DSN` and `SENTRY_ENVIRONMENT` are no longer read; the server and edge runtimes now read `UI_SENTRY_DSN` and `UI_SENTRY_ENVIRONMENT`. The former `NEXT_PUBLIC_*` build-time variables are deprecated but still read at runtime as a fallback when the matching `UI_*` variable is unset. This fallback will be removed in a future release, so set the `UI_*` runtime variables on the running container.
|
||||
- **Prowler App:** The bare server-side `SENTRY_DSN` and `SENTRY_ENVIRONMENT` are no longer read; the server and edge runtimes now read `UI_SENTRY_DSN` and `UI_SENTRY_ENVIRONMENT`. The former `NEXT_PUBLIC_*` names — and, for PostHog, the unprefixed `POSTHOG_KEY` and `POSTHOG_HOST` — are deprecated but stay backward compatible: they are read at runtime regardless of the enable flag, so an existing deployment keeps its integration active without opting in. The new `UI_*` names, by contrast, load only when the matching enable flag is set to `"true"`. These legacy names will be removed in a future release, so migrate to the `UI_*` runtime variables — and set the enable flag — on the running container.
|
||||
- **Prowler API, Prowler SDK, and Prowler MCP Server:** The current, unprefixed variable names are deprecated. They continue to work today and will be removed once the prefixed convention is adopted for each component, as described in [Upcoming Breaking Change](#upcoming-breaking-change).
|
||||
|
||||
@@ -3421,7 +3421,7 @@ Use existing providers as templates, this will help you to understand better the
|
||||
- **Documentation & Maintenance**
|
||||
|
||||
- **README Updates**: Update provider-specific documentation
|
||||
- **Changelog**: Document changes and new features
|
||||
- **Changelog**: Document changes and new features with a fragment under `prowler/changelog.d/` (see the [Pull Request Template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md))
|
||||
- **Examples**: Provide usage examples and common scenarios
|
||||
- **Troubleshooting**: Include common issues and solutions
|
||||
- **Documentation**: Update the provider documentation to include your new tool provider in the examples and implementation guidance.
|
||||
|
||||
@@ -778,7 +778,7 @@ Before opening the pull request:
|
||||
uv run pre-commit run --all-files
|
||||
uv run pytest -n auto
|
||||
```
|
||||
2. Add a changelog entry under the `### 🚀 Added` section of `prowler/CHANGELOG.md`, describing the new framework and the providers it covers.
|
||||
2. Add a changelog fragment `prowler/changelog.d/<slug>.added.md`, describing the new framework and the providers it covers (no PR link in the text; it is attached automatically at release time).
|
||||
3. Follow the [Pull Request Template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md) and set the PR title using Conventional Commits, e.g. `feat(compliance): add My Framework 1.0 for AWS`.
|
||||
4. Request review from the compliance codeowners listed in `.github/CODEOWNERS`.
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
{
|
||||
"group": "Prowler Lighthouse AI",
|
||||
"pages": [
|
||||
"getting-started/products/prowler-lighthouse-ai"
|
||||
"getting-started/products/prowler-cloud-lighthouse"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -154,8 +154,15 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Lighthouse AI",
|
||||
"group": "Lighthouse AI (Prowler Cloud)",
|
||||
"pages": [
|
||||
"user-guide/tutorials/prowler-cloud-lighthouse-multi-llm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Lighthouse AI (Open Source)",
|
||||
"pages": [
|
||||
"getting-started/products/prowler-lighthouse-ai",
|
||||
"user-guide/tutorials/prowler-app-lighthouse",
|
||||
"user-guide/tutorials/prowler-app-lighthouse-multi-llm"
|
||||
]
|
||||
|
||||
@@ -128,8 +128,8 @@ To update the environment file:
|
||||
Edit the `.env` file and change version values:
|
||||
|
||||
```env
|
||||
PROWLER_UI_VERSION="5.31.0"
|
||||
PROWLER_API_VERSION="5.31.0"
|
||||
PROWLER_UI_VERSION="5.32.0"
|
||||
PROWLER_API_VERSION="5.32.0"
|
||||
```
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
---
|
||||
title: 'Overview'
|
||||
---
|
||||
|
||||
import { SubscriptionBanner } from "/snippets/subscription-banner.mdx"
|
||||
|
||||
Prowler Cloud runs an enhanced version of Lighthouse AI in Open Source repository, the Agentic Cloud Defender that helps teams understand, prioritize, and remediate security findings across cloud environments.
|
||||
|
||||
<SubscriptionBanner />
|
||||
|
||||
<img src="/images/prowler-app/lighthouse/prowler-cloud/chat-animation.gif" alt="Lighthouse AI on Prowler Cloud" />
|
||||
|
||||
## What's New
|
||||
|
||||
The Agentic Cloud Defender does more than answer questions, it helps teams **find and remediate what actually matters**, cutting through the noise to focus on the risk that counts. Prowler Cloud sharpens Lighthouse AI with the following improvements over the open-source version:
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Persistent Chat Sessions" icon="clock-rotate-left">
|
||||
Conversations are saved and can be revisited or resumed at any time.
|
||||
</Card>
|
||||
<Card title="GPT-5.5 by Default" icon="microchip">
|
||||
An upgraded default model delivers stronger reasoning and tool calling.
|
||||
</Card>
|
||||
<Card title="Normal and Agentic Views" icon="table-columns">
|
||||
Switch between the standard interface and a chat-first agentic view.
|
||||
</Card>
|
||||
<Card title="Provider Connection Checks" icon="plug-circle-check">
|
||||
Credentials are validated automatically when a provider is configured.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
## Chat View
|
||||
|
||||
Lighthouse AI is no longer a separate section in the left navigation. Prowler Cloud now offers two application views: a normal view for browsing dashboards, findings, and configuration, and an agentic chat view, powered by Lighthouse AI, for conversational, multi-step security analysis. Conversations are saved automatically, so earlier sessions can be reopened and resumed at any time.
|
||||
|
||||
Promoting the chat to a top-level view gives Lighthouse AI the room it needs for a fully agentic workflow and makes the Agentic Cloud Defender a primary way to work in Prowler Cloud.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse/prowler-cloud/main-chat-page.png" alt="Lighthouse AI chat view in Prowler Cloud" />
|
||||
|
||||
### Tool Usage
|
||||
|
||||
Lighthouse AI on Prowler Cloud renders the agent's work as it happens, so responses are easier to follow and to trust. Tool calls and reasoning steps appear in the order they occur within the conversation.
|
||||
|
||||
- **Ordered steps:** Tool calls and reasoning are shown in sequence, reflecting how the agent reached its answer.
|
||||
- **Tool visibility:** The data tools invoked to retrieve findings and other Prowler information are displayed as the agent uses them.
|
||||
- **Thought process:** The agent's reasoning is presented alongside its actions, and can be expanded to review the full chain of steps.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse/prowler-cloud/tool-usage.png" alt="Ordered tool usage and reasoning in the Lighthouse AI chat" />
|
||||
|
||||
## Configuration
|
||||
|
||||
Configure Lighthouse AI on Prowler Cloud from **Configuration** → **Lighthouse AI**:
|
||||
|
||||
1. Click the desired provider (OpenAI, Amazon Bedrock, or OpenAI Compatible).
|
||||
2. Enter the required credentials.
|
||||
3. Click **Save**. The connection is validated automatically before the provider becomes available.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse/prowler-cloud/config-page.png" alt="Lighthouse AI configuration page in Prowler Cloud" />
|
||||
|
||||
### Business Context
|
||||
|
||||
At the top of the configuration page, the optional **Business Context** field lets teams add environment priorities, compliance requirements, and ownership details, so responses align with organizational needs.
|
||||
|
||||
Lighthouse AI on Prowler Cloud supports OpenAI, Amazon Bedrock, and OpenAI-compatible providers, with GPT-5.5 as the default. For per-provider setup and how to switch the default provider or model, see [Using Multiple LLM Providers](/user-guide/tutorials/prowler-cloud-lighthouse-multi-llm).
|
||||
|
||||
## FAQ
|
||||
|
||||
**Which LLM providers are supported?**
|
||||
|
||||
OpenAI (GPT models, including the default GPT-5.5), Amazon Bedrock (Claude, Llama, Titan, and others), and any OpenAI-compatible service such as OpenRouter.
|
||||
|
||||
**Can Lighthouse AI change my cloud environment?**
|
||||
|
||||
No. Lighthouse AI has read-only access to security data and no tools to modify resources, even when the connected cloud credentials would allow changes.
|
||||
|
||||
## Looking for the Open Source Version?
|
||||
|
||||
Lighthouse AI is also available in the self-hosted, open-source Prowler App. For its capabilities, FAQs, and limitations, see the open-source documentation.
|
||||
|
||||
<Card title="Lighthouse AI (Open Source)" icon="github" href="/getting-started/products/prowler-lighthouse-ai">
|
||||
Capabilities, FAQs, and limitations for Lighthouse AI in the open-source Prowler App
|
||||
</Card>
|
||||
|
||||
## Getting Help
|
||||
|
||||
For issues or suggestions with Lighthouse AI on Prowler Cloud, request support at [support.prowler.com](https://support.prowler.com) or [reach out through our Slack channel](https://goto.prowler.com/slack).
|
||||
@@ -8,11 +8,16 @@ import { VersionBadge } from "/snippets/version-badge.mdx"
|
||||
|
||||
Prowler Lighthouse AI is a Cloud Security Analyst chatbot that helps you understand, prioritize, and remediate security findings in your cloud environments. It's designed to provide security expertise for teams without dedicated resources, acting as your 24/7 virtual cloud security analyst.
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-intro.png" alt="Prowler Lighthouse" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/intro.png" alt="Prowler Lighthouse" />
|
||||
|
||||
<Card title="Set Up Lighthouse AI" icon="rocket" href="/user-guide/tutorials/prowler-app-lighthouse#set-up">
|
||||
Learn how to configure Lighthouse AI with your preferred LLM provider
|
||||
</Card>
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Lighthouse AI on Prowler Cloud" icon="cloud" href="/getting-started/products/prowler-cloud-lighthouse">
|
||||
Discover the enhanced Cloud experience: persistent chat sessions, GPT-5.5 by default, a dedicated agentic view, and transparent reasoning
|
||||
</Card>
|
||||
<Card title="Set Up Lighthouse AI" icon="rocket" href="/user-guide/tutorials/prowler-app-lighthouse#set-up">
|
||||
Learn how to configure Lighthouse AI with your preferred LLM provider
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
## Capabilities
|
||||
|
||||
@@ -26,7 +31,7 @@ Ask questions in plain English about your security findings. Examples:
|
||||
- "Show me all S3 buckets with public access."
|
||||
- "What security issues were found in my production accounts?"
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-feature1.png" alt="Natural language querying" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/feature1.png" alt="Natural language querying" />
|
||||
|
||||
### Detailed Remediation Guidance
|
||||
|
||||
@@ -36,7 +41,7 @@ Get tailored step-by-step instructions for fixing security issues:
|
||||
- Commands or console steps to implement fixes
|
||||
- Alternative approaches with different solutions
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-feature2.png" alt="Detailed Remediation" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/feature2.png" alt="Detailed Remediation" />
|
||||
|
||||
### Enhanced Context and Analysis
|
||||
|
||||
@@ -46,9 +51,9 @@ Lighthouse AI can provide additional context to help you understand the findings
|
||||
- Provide risk assessments based on your environment and context
|
||||
- Connect related findings to show broader security patterns
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-config.png" alt="Business Context" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/config.png" alt="Business Context" />
|
||||
|
||||
<img src="/images/prowler-app/lighthouse-feature3.png" alt="Contextual Responses" />
|
||||
<img src="/images/prowler-app/lighthouse/oss/feature3.png" alt="Contextual Responses" />
|
||||
|
||||
## Important Notes
|
||||
|
||||
|
||||
|
Before Width: | Height: | Size: 115 KiB After Width: | Height: | Size: 123 KiB |
|
Before Width: | Height: | Size: 316 KiB After Width: | Height: | Size: 111 KiB |
|
Before Width: | Height: | Size: 304 KiB After Width: | Height: | Size: 97 KiB |
|
Before Width: | Height: | Size: 477 KiB After Width: | Height: | Size: 136 KiB |
|
Before Width: | Height: | Size: 575 KiB After Width: | Height: | Size: 107 KiB |
|
Before Width: | Height: | Size: 324 KiB After Width: | Height: | Size: 136 KiB |
|
Before Width: | Height: | Size: 115 KiB After Width: | Height: | Size: 123 KiB |
|
Before Width: | Height: | Size: 423 KiB After Width: | Height: | Size: 120 KiB |
|
Before Width: | Height: | Size: 275 KiB After Width: | Height: | Size: 121 KiB |
|
Before Width: | Height: | Size: 66 KiB After Width: | Height: | Size: 81 KiB |
|
Before Width: | Height: | Size: 481 KiB After Width: | Height: | Size: 610 KiB |
|
Before Width: | Height: | Size: 456 KiB After Width: | Height: | Size: 593 KiB |
|
Before Width: | Height: | Size: 96 KiB After Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 540 KiB After Width: | Height: | Size: 540 KiB |
|
Before Width: | Height: | Size: 136 KiB After Width: | Height: | Size: 136 KiB |
|
Before Width: | Height: | Size: 147 KiB After Width: | Height: | Size: 147 KiB |
|
Before Width: | Height: | Size: 180 KiB After Width: | Height: | Size: 180 KiB |
|
Before Width: | Height: | Size: 165 KiB After Width: | Height: | Size: 165 KiB |
|
Before Width: | Height: | Size: 347 KiB After Width: | Height: | Size: 347 KiB |
|
Before Width: | Height: | Size: 173 KiB After Width: | Height: | Size: 173 KiB |
|
After Width: | Height: | Size: 236 KiB |
|
After Width: | Height: | Size: 472 KiB |