mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-08-19 09:30:21 +00:00
Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eaee2622a4 | ||
|
|
9b184d1d45 | ||
|
|
c1d18040b7 | ||
|
|
e59b391dea | ||
|
|
8bf926d4e7 | ||
|
|
0c9654db4b | ||
|
|
31261d78f3 | ||
|
|
e8779953cd | ||
|
|
7345e051cf | ||
|
|
c2b0135e35 | ||
|
|
1c4d8e3e75 | ||
|
|
b3562a800f | ||
|
|
cea4244db8 | ||
|
|
52f2da90f6 | ||
|
|
d1d6825159 | ||
|
|
548dd0f35a | ||
|
|
c11fdb9d6d | ||
|
|
9f3a0534a8 | ||
|
|
44b7afdb25 | ||
|
|
fd19a9a048 | ||
|
|
1da1da54d1 | ||
|
|
c123dc3788 | ||
|
|
3517cb331a | ||
|
|
fa365eb106 | ||
|
|
4526b91d3b |
@@ -158,7 +158,7 @@ SENTRY_RELEASE=local
|
||||
# REO_DEV_CLIENT_ID=
|
||||
|
||||
#### Prowler release version ####
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.33.0
|
||||
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.33.2
|
||||
|
||||
# Social login credentials
|
||||
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
|
||||
|
||||
@@ -18,8 +18,8 @@ Please add a detailed description of how to review this PR.
|
||||
|
||||
<summary><b>Community Checklist</b></summary>
|
||||
|
||||
- [ ] This feature/issue is listed in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
|
||||
- [ ] Is it assigned to me, if not, request it via the issue/feature in [here](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](goto.prowler.com/slack)
|
||||
- [ ] This feature/issue is listed in the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or roadmap.prowler.com
|
||||
- [ ] Is it assigned to me, if not, request it via the [open issues](https://github.com/prowler-cloud/prowler/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) or [Prowler Community Slack](https://goto.prowler.com/slack)
|
||||
|
||||
</details>
|
||||
|
||||
@@ -28,7 +28,7 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Review if code is being documented following this specification https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings
|
||||
- [ ] Review if backport is needed.
|
||||
- [ ] Review if is needed to change the [Readme.md](https://github.com/prowler-cloud/prowler/blob/master/README.md)
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/prowler/CHANGELOG.md), if applicable.
|
||||
- [ ] Ensure a changelog fragment is added under [prowler/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/prowler/changelog.d), if applicable.
|
||||
|
||||
#### SDK/CLI
|
||||
- Are there new checks included in this PR? Yes / No
|
||||
@@ -40,7 +40,7 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Mobile (X < 640px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Table (640px > X < 1024px)
|
||||
- [ ] Screenshots/Video of the functionality flow (if applicable) - Desktop (X > 1024px)
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/ui/CHANGELOG.md), if applicable.
|
||||
- [ ] Ensure a changelog fragment is added under [ui/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/ui/changelog.d), if applicable.
|
||||
|
||||
#### API
|
||||
- [ ] All issue/task requirements work as expected on the API
|
||||
@@ -50,7 +50,11 @@ Please add a detailed description of how to review this PR.
|
||||
- [ ] Any other relevant evidence of the implementation (if applicable)
|
||||
- [ ] Verify if API specs need to be regenerated.
|
||||
- [ ] Check if version updates are required (e.g., specs, uv, etc.).
|
||||
- [ ] Ensure new entries are added to [CHANGELOG.md](https://github.com/prowler-cloud/prowler/blob/master/api/CHANGELOG.md), if applicable.
|
||||
- [ ] Ensure a changelog fragment is added under [api/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/api/changelog.d), if applicable.
|
||||
|
||||
#### MCP Server
|
||||
- [ ] All issue/task requirements work as expected on the MCP Server
|
||||
- [ ] Ensure a changelog fragment is added under [mcp_server/changelog.d/](https://github.com/prowler-cloud/prowler/tree/master/mcp_server/changelog.d), if applicable.
|
||||
|
||||
### License
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rename changelog fragments to their PR number before running towncrier.
|
||||
|
||||
For every <slug>.<type>.md in <component_dir>/changelog.d/, find the commit that
|
||||
added it, resolve its PR via the GitHub API (falling back to the squash-commit
|
||||
subject), and `git mv` it to <PR>.<type>.md so towncrier renders the PR link.
|
||||
Unresolvable fragments become +<slug>.<type>.md orphans (rendered without link).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
FRAGMENT_RE = re.compile(
|
||||
r"^(?P<slug>[A-Za-z0-9][A-Za-z0-9._-]*?)"
|
||||
r"\.(?P<type>added|changed|deprecated|removed|fixed|security)"
|
||||
r"(?:\.(?P<counter>[0-9]+))?\.md$"
|
||||
)
|
||||
SUBJECT_PR_RE = re.compile(r" \(#([0-9]+)\)$")
|
||||
IGNORED_FILES = {".gitkeep", "README.md"}
|
||||
API_TIMEOUT_SECONDS = 10
|
||||
|
||||
|
||||
def git(*args: str) -> str:
|
||||
result = subprocess.run(["git", *args], check=True, capture_output=True, text=True)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def find_adding_commit(path: str) -> str | None:
|
||||
"""Find the commit that added a file, following renames.
|
||||
|
||||
Falls back to a plain (no --follow) lookup: rename detection can lose the
|
||||
add event for degenerate content (e.g. files identical to many others).
|
||||
"""
|
||||
sha = git("log", "--follow", "--diff-filter=A", "--format=%H", "-1", "--", path)
|
||||
if not sha:
|
||||
sha = git("log", "--diff-filter=A", "--format=%H", "-1", "--", path)
|
||||
return sha or None
|
||||
|
||||
|
||||
def pr_from_api(repo: str, sha: str) -> int | None:
|
||||
"""Resolve the PR associated with a commit via the GitHub API.
|
||||
|
||||
Returns None on any network/API failure so the caller can fall back to
|
||||
parsing the squash-commit subject.
|
||||
"""
|
||||
url = f"https://api.github.com/repos/{repo}/commits/{sha}/pulls"
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"User-Agent": "prowler-changelog-attribution",
|
||||
}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
request = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=API_TIMEOUT_SECONDS) as response:
|
||||
pulls = json.load(response)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
return None
|
||||
if isinstance(pulls, list) and pulls:
|
||||
return pulls[0].get("number")
|
||||
return None
|
||||
|
||||
|
||||
def pr_from_subject(sha: str) -> int | None:
|
||||
subject = git("log", "-1", "--format=%s", sha)
|
||||
match = SUBJECT_PR_RE.search(subject)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def unique_destination(directory: str, base_name: str, fragment_type: str) -> str:
|
||||
"""Return a non-colliding fragment path, appending a numeric counter if needed."""
|
||||
candidate = os.path.join(directory, f"{base_name}.{fragment_type}.md")
|
||||
counter = 0
|
||||
while os.path.exists(candidate):
|
||||
counter += 1
|
||||
candidate = os.path.join(directory, f"{base_name}.{fragment_type}.{counter}.md")
|
||||
return candidate
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("component_dir", help="Component directory, e.g. prowler")
|
||||
parser.add_argument("--repo", default="prowler-cloud/prowler")
|
||||
parser.add_argument(
|
||||
"--no-api",
|
||||
action="store_true",
|
||||
help="Skip the GitHub API and resolve PRs from commit subjects only",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
fragments_dir = os.path.join(args.component_dir, "changelog.d")
|
||||
if not os.path.isdir(fragments_dir):
|
||||
print(f"::error::Fragments directory not found: {fragments_dir}")
|
||||
return 1
|
||||
|
||||
malformed = []
|
||||
to_process = []
|
||||
for name in sorted(os.listdir(fragments_dir)):
|
||||
if name in IGNORED_FILES or name.startswith("+"):
|
||||
continue
|
||||
match = FRAGMENT_RE.match(name)
|
||||
if not match:
|
||||
malformed.append(name)
|
||||
continue
|
||||
if match.group("slug").isdigit():
|
||||
continue
|
||||
to_process.append((name, match))
|
||||
|
||||
if malformed:
|
||||
for name in malformed:
|
||||
print(
|
||||
f"::error::Malformed fragment filename in {fragments_dir}: {name} "
|
||||
"(expected <slug>.<type>.md with type one of added|changed|"
|
||||
"deprecated|removed|fixed|security)"
|
||||
)
|
||||
return 1
|
||||
|
||||
for name, match in to_process:
|
||||
slug, fragment_type = match.group("slug"), match.group("type")
|
||||
|
||||
path = os.path.join(fragments_dir, name)
|
||||
sha = find_adding_commit(path)
|
||||
pr_number = None
|
||||
if sha:
|
||||
if not args.no_api:
|
||||
pr_number = pr_from_api(args.repo, sha)
|
||||
if pr_number is None:
|
||||
pr_number = pr_from_subject(sha)
|
||||
|
||||
if pr_number is not None:
|
||||
destination = unique_destination(
|
||||
fragments_dir, str(pr_number), fragment_type
|
||||
)
|
||||
else:
|
||||
destination = unique_destination(fragments_dir, f"+{slug}", fragment_type)
|
||||
print(
|
||||
f"::warning::Could not resolve a PR for {path}; renamed to "
|
||||
f"{os.path.basename(destination)} (entry will render without a PR link)"
|
||||
)
|
||||
git("mv", path, destination)
|
||||
print(f"{path} -> {destination}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,13 @@
|
||||
{% set category_order = definitions.keys() %}
|
||||
{% for section, _ in sections.items() %}
|
||||
{% for category in category_order if category in sections[section] %}
|
||||
### {{ definitions[category]['name'] }}
|
||||
|
||||
{% for text, values in sections[section][category].items() -%}
|
||||
- {{ text }}{% if values %} {{ values|join(', ') }}{% endif %}{{ "\n" }}
|
||||
{%- endfor %}
|
||||
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
---
|
||||
{{ "\n" }}
|
||||
@@ -62,6 +62,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -108,6 +108,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
|
||||
@@ -77,6 +77,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -111,6 +111,7 @@ jobs:
|
||||
api/docs/**
|
||||
api/README.md
|
||||
api/CHANGELOG.md
|
||||
api/changelog.d/**
|
||||
api/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -0,0 +1,527 @@
|
||||
name: 'Tools: Compile Changelogs'
|
||||
|
||||
run-name: 'Compile changelogs for Prowler ${{ inputs.prowler_version }}'
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
prowler_version:
|
||||
description: 'Prowler version being released (e.g., 5.31.0)'
|
||||
required: true
|
||||
type: string
|
||||
target_branch:
|
||||
description: 'Branch to compile on (master for minor releases, v5.X for patches)'
|
||||
required: true
|
||||
type: string
|
||||
sdk_version:
|
||||
description: 'SDK version override (empty = mirrors prowler_version; "skip" = hold this component back)'
|
||||
required: false
|
||||
type: string
|
||||
api_version:
|
||||
description: 'API version override (empty = auto-derive 1.<prowler_minor + 1>.<prowler_patch>; "skip" = hold back)'
|
||||
required: false
|
||||
type: string
|
||||
ui_version:
|
||||
description: 'UI version override (empty = auto-derive 1.<prowler_minor>.<prowler_patch>; "skip" = hold back)'
|
||||
required: false
|
||||
type: string
|
||||
mcp_version:
|
||||
description: 'MCP Server version override (empty = auto-derive from pending fragment types; "skip" = hold back)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ inputs.prowler_version }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PROWLER_VERSION: ${{ inputs.prowler_version }}
|
||||
TARGET_BRANCH: ${{ inputs.target_branch }}
|
||||
SDK_VERSION: ${{ inputs.sdk_version }}
|
||||
API_VERSION: ${{ inputs.api_version }}
|
||||
UI_VERSION: ${{ inputs.ui_version }}
|
||||
MCP_VERSION: ${{ inputs.mcp_version }}
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
compile-changelogs:
|
||||
if: github.event_name == 'workflow_dispatch' && github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Block outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ inputs.target_branch }}
|
||||
fetch-depth: 0 # PR attribution resolves each fragment's adding commit from history
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install towncrier
|
||||
run: pip install --no-cache-dir towncrier==25.8.0
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.name 'prowler-bot'
|
||||
git config --global user.email '179230569+prowler-bot@users.noreply.github.com'
|
||||
|
||||
- name: Validate version inputs
|
||||
run: |
|
||||
if [[ ! "$PROWLER_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::Invalid prowler_version syntax: '$PROWLER_VERSION' (must be N.N.N)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$TARGET_BRANCH" != "master" ] && [[ ! "$TARGET_BRANCH" =~ ^v[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::Invalid target_branch syntax: '$TARGET_BRANCH' (must be 'master' or vN.N, e.g. v5.31)"
|
||||
exit 1
|
||||
fi
|
||||
IFS=. read -r prowler_major prowler_minor prowler_patch <<< "$PROWLER_VERSION"
|
||||
prowler_major=$((10#$prowler_major))
|
||||
prowler_minor=$((10#$prowler_minor))
|
||||
prowler_patch=$((10#$prowler_patch))
|
||||
if [ "$prowler_patch" -eq 0 ]; then
|
||||
if [ "$TARGET_BRANCH" != "master" ]; then
|
||||
echo "::error::target_branch must be 'master' for Prowler ${PROWLER_VERSION}; got '${TARGET_BRANCH}'"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
expected_target_branch="v${prowler_major}.${prowler_minor}"
|
||||
if [ "$TARGET_BRANCH" != "$expected_target_branch" ]; then
|
||||
echo "::error::target_branch must be '${expected_target_branch}' for Prowler ${PROWLER_VERSION}; got '${TARGET_BRANCH}'"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
for pair in "sdk_version:$SDK_VERSION" "api_version:$API_VERSION" "ui_version:$UI_VERSION" "mcp_version:$MCP_VERSION"; do
|
||||
input_name="${pair%%:*}"
|
||||
input_value="${pair#*:}"
|
||||
if [ -n "$input_value" ] && [ "$input_value" != "skip" ] && [[ ! "$input_value" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::Invalid $input_name syntax: '$input_value' (must be N.N.N, empty for auto-derivation, or 'skip')"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Compile changelogs
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
component_input() {
|
||||
case "$1" in
|
||||
prowler) echo "$SDK_VERSION" ;;
|
||||
api) echo "$API_VERSION" ;;
|
||||
ui) echo "$UI_VERSION" ;;
|
||||
mcp_server) echo "$MCP_VERSION" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
version_key() {
|
||||
local version="$1"
|
||||
local major minor patch
|
||||
IFS=. read -r major minor patch <<< "$version"
|
||||
printf '%06d.%06d.%06d' "$((10#$major))" "$((10#$minor))" "$((10#$patch))"
|
||||
}
|
||||
|
||||
pending_fragments() {
|
||||
find "$1/changelog.d" -maxdepth 1 -type f ! -name '.gitkeep' ! -name 'README.md' | sort
|
||||
}
|
||||
|
||||
# The component's last released version is the first stamped heading
|
||||
# of its CHANGELOG.md, the same source prepare-release.yml greps.
|
||||
latest_released_version() {
|
||||
grep -m1 -E '^## \[v?[0-9]+\.[0-9]+\.[0-9]+\]' "$1/CHANGELOG.md" | sed -E 's/^## \[v?([0-9]+\.[0-9]+\.[0-9]+)\].*/\1/'
|
||||
}
|
||||
|
||||
has_removed_fragments() {
|
||||
echo "$1" | grep -qE '\.removed(\.[0-9]+)?\.md$'
|
||||
}
|
||||
|
||||
# Resolve every component's effective version before compiling
|
||||
# anything, so a wrong input cannot leave the tree half-compiled.
|
||||
# Empty input = auto-derive (latest released version + semver bump
|
||||
# from the pending fragment types). 'skip' = hold the component back.
|
||||
errors=0
|
||||
compiling=""
|
||||
for component in prowler api ui mcp_server; do
|
||||
input=$(component_input "$component")
|
||||
fragments=$(pending_fragments "$component")
|
||||
|
||||
if [ "$input" = "skip" ]; then
|
||||
if [ -n "$fragments" ]; then
|
||||
echo "::warning::${component}: held back by request; these pending fragments stay for a future release:"
|
||||
echo "$fragments"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ -n "$input" ] && [ -z "$fragments" ]; then
|
||||
echo "::error::${component}: version input '$input' provided but ${component}/changelog.d/ has no pending fragments (wrong input?)"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
if [ -z "$fragments" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
removed_fragments=false
|
||||
if has_removed_fragments "$fragments"; then
|
||||
removed_fragments=true
|
||||
fi
|
||||
current=$(latest_released_version "$component")
|
||||
if [ -z "$current" ]; then
|
||||
echo "::error::${component}: could not read the latest released version from ${component}/CHANGELOG.md; restore the released heading before compiling"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ -n "$input" ]; then
|
||||
effective="$input"
|
||||
mode="explicit"
|
||||
current_key=$(version_key "$current")
|
||||
effective_key=$(version_key "$effective")
|
||||
if [[ "$effective_key" < "$current_key" || "$effective_key" == "$current_key" ]]; then
|
||||
echo "::error::${component}: explicit version '${effective}' must be greater than the latest released version (${current})"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
else
|
||||
if [ "$removed_fragments" = "true" ]; then
|
||||
echo "::error::${component}: pending 'removed' fragments imply a major bump (breaking change); provide its version input explicitly"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
# SDK, UI, and API versions are deterministic mirrors of the
|
||||
# Prowler version (the scheme bump-version.yml codifies): the SDK
|
||||
# mirrors it directly, the UI tracks 1.<minor>.<patch>, and the
|
||||
# API is the independent 1.<minor + 1>.<patch> stream. Only the
|
||||
# MCP Server has its own cadence, derived from fragment types.
|
||||
IFS=. read -r _ prowler_minor prowler_patch <<< "$PROWLER_VERSION"
|
||||
prowler_minor=$((10#$prowler_minor))
|
||||
prowler_patch=$((10#$prowler_patch))
|
||||
case "$component" in
|
||||
prowler) effective="$PROWLER_VERSION" ;;
|
||||
ui) effective="1.${prowler_minor}.${prowler_patch}" ;;
|
||||
api) effective="1.$((prowler_minor + 1)).${prowler_patch}" ;;
|
||||
mcp_server)
|
||||
IFS=. read -r major minor patch <<< "$current"
|
||||
major=$((10#$major))
|
||||
minor=$((10#$minor))
|
||||
patch=$((10#$patch))
|
||||
# Prowler patch releases (vN.N target) are maintenance
|
||||
# releases, so the MCP Server bumps patch regardless of
|
||||
# fragment types; a deliberate exception needs the explicit
|
||||
# version input.
|
||||
if [ "$TARGET_BRANCH" != "master" ]; then
|
||||
effective="${major}.${minor}.$((patch + 1))"
|
||||
if echo "$fragments" | grep -qE '\.(added|deprecated)(\.[0-9]+)?\.md$'; then
|
||||
echo "::warning::${component}: 'added'/'deprecated' fragments are shipping in a Prowler patch; auto-derived a patch bump (${current} -> ${effective}), pass the version input to override"
|
||||
fi
|
||||
elif echo "$fragments" | grep -qE '\.(added|changed|deprecated)(\.[0-9]+)?\.md$'; then
|
||||
effective="${major}.$((minor + 1)).0"
|
||||
else
|
||||
effective="${major}.${minor}.$((patch + 1))"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
current_key=$(version_key "$current")
|
||||
effective_key=$(version_key "$effective")
|
||||
if [[ "$effective_key" < "$current_key" || "$effective_key" == "$current_key" ]]; then
|
||||
echo "::error::${component}: auto-derived version '${effective}' is not greater than the latest released version (${current}); check prowler_version or pass the version input explicitly"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
mode="auto"
|
||||
echo "::notice::${component}: version auto-derived ${current} -> ${effective}"
|
||||
fi
|
||||
|
||||
if [ "$removed_fragments" = "true" ]; then
|
||||
IFS=. read -r current_major _ <<< "$current"
|
||||
current_major=$((10#$current_major))
|
||||
IFS=. read -r effective_major effective_minor effective_patch <<< "$effective"
|
||||
effective_major=$((10#$effective_major))
|
||||
effective_minor=$((10#$effective_minor))
|
||||
effective_patch=$((10#$effective_patch))
|
||||
if [ "$effective_major" -le "$current_major" ] || [ "$effective_minor" -ne 0 ] || [ "$effective_patch" -ne 0 ]; then
|
||||
echo "::error::${component}: removed fragments require a major component release (${current} -> X.0.0 with X > ${current_major}); got ${effective}"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Without the marker the build would insert the new block above the
|
||||
# file header instead of below it.
|
||||
if ! grep -q '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md"; then
|
||||
echo "::error::${component}/CHANGELOG.md is missing the '<!-- changelog: release notes start -->' marker; restore it after the intro line before compiling"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
# A hand-written UNRELEASED block means someone followed the old
|
||||
# convention; its entries would be left out of the compiled block
|
||||
# and out of the release notes extraction.
|
||||
if grep -q '(Prowler UNRELEASED)' "$component/CHANGELOG.md"; then
|
||||
echo "::error::${component}/CHANGELOG.md contains a hand-written '(Prowler UNRELEASED)' block; convert its entries to fragments in ${component}/changelog.d/ and delete the block before compiling"
|
||||
errors=1
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "${effective} ${mode}" > "${RUNNER_TEMP}/version-${component}.txt"
|
||||
compiling="${compiling}${component} "
|
||||
done
|
||||
if [ "$errors" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$compiling" ]; then
|
||||
echo "::error::Nothing to compile: no component has pending fragments to release"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body_file="${RUNNER_TEMP}/compile-changelogs-pr-body.md"
|
||||
{
|
||||
echo "### Description"
|
||||
echo ""
|
||||
echo "Compiles the pending changelog fragments into the per-component \`CHANGELOG.md\` files for Prowler v${PROWLER_VERSION}, replacing the manual stamping PR."
|
||||
echo ""
|
||||
echo "| Component | Version | Fragments consumed |"
|
||||
echo "|---|---|---|"
|
||||
} > "$body_file"
|
||||
|
||||
compiled_components=""
|
||||
for component in prowler api ui mcp_server; do
|
||||
if [ ! -f "${RUNNER_TEMP}/version-${component}.txt" ]; then
|
||||
echo "Skipping ${component} (no pending fragments or held back)"
|
||||
echo "| \`${component}\` | - | 0 |" >> "$body_file"
|
||||
continue
|
||||
fi
|
||||
read -r version mode < "${RUNNER_TEMP}/version-${component}.txt"
|
||||
version_label="$version"
|
||||
if [ "$mode" = "auto" ]; then
|
||||
version_label="${version} (auto)"
|
||||
fi
|
||||
|
||||
count=$(pending_fragments "$component" | wc -l | tr -d ' ')
|
||||
echo "Compiling ${component} ${version} (${count} fragments, ${mode} version)..."
|
||||
|
||||
# Captured before attribution renames them: these original paths are
|
||||
# what the forward-sync deletes on master (backports copy fragments
|
||||
# verbatim, so filenames match across branches).
|
||||
pending_fragments "$component" > "${RUNNER_TEMP}/consumed-${component}.txt"
|
||||
pre_lines=$(wc -l < "$component/CHANGELOG.md")
|
||||
|
||||
# Attribution must run before the build: towncrier renders the
|
||||
# first dotted segment of each filename as the PR number.
|
||||
python .github/scripts/changelog_attribution.py "$component"
|
||||
towncrier build --config "$component/towncrier.toml" --version "$version" --name "Prowler v${PROWLER_VERSION}" --yes
|
||||
|
||||
# The build only inserts lines right after the marker, so the new
|
||||
# stamped block is exactly the added lines following it. Captured
|
||||
# for the forward-sync to master.
|
||||
post_lines=$(wc -l < "$component/CHANGELOG.md")
|
||||
delta=$((post_lines - pre_lines))
|
||||
marker_line=$(grep -n -m1 '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md" | cut -d: -f1)
|
||||
sed -n "$((marker_line + 1)),$((marker_line + delta))p" "$component/CHANGELOG.md" > "${RUNNER_TEMP}/block-${component}.md"
|
||||
|
||||
compiled_components="${compiled_components}${component} "
|
||||
echo "| \`${component}\` | ${version_label} | ${count} |" >> "$body_file"
|
||||
done
|
||||
echo "COMPILED_COMPONENTS=${compiled_components}" >> "$GITHUB_ENV"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "Review that no pending fragment was dropped (the diff must delete every consumed fragment) and that each new version block is correct, then squash-merge."
|
||||
echo ""
|
||||
echo "### License"
|
||||
echo ""
|
||||
echo "By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license."
|
||||
} >> "$body_file"
|
||||
|
||||
echo "PR_BODY_FILE=${body_file}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Create compile PR
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
commit-message: 'chore(changelog): v${{ env.PROWLER_VERSION }}'
|
||||
branch: compile-changelogs-${{ env.PROWLER_VERSION }}
|
||||
base: ${{ env.TARGET_BRANCH }}
|
||||
title: 'chore(changelog): v${{ env.PROWLER_VERSION }}'
|
||||
body-path: ${{ env.PR_BODY_FILE }}
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
labels: |
|
||||
no-changelog
|
||||
skip-sync
|
||||
|
||||
# Patch compiles (target_branch = v5.X) leave master holding the consumed
|
||||
# fragments and missing the new version block. This applies the equivalent
|
||||
# change to master: insert the same stamped blocks under the marker and
|
||||
# delete the consumed fragments, so the next minor compile cannot
|
||||
# re-release entries that already shipped in the patch.
|
||||
- name: Apply forward-sync to master
|
||||
if: env.TARGET_BRANCH != 'master'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
git checkout -B master origin/master
|
||||
|
||||
version_key() {
|
||||
local version="$1"
|
||||
local major minor patch
|
||||
IFS=. read -r major minor patch <<< "$version"
|
||||
printf '%06d.%06d.%06d' "$((10#$major))" "$((10#$minor))" "$((10#$patch))"
|
||||
}
|
||||
|
||||
release_from_heading() {
|
||||
local heading="$1"
|
||||
echo "$heading" | sed -E 's/^## \[[^]]+\] \(Prowler v?([0-9]+\.[0-9]+\.[0-9]+)\).*/\1/'
|
||||
}
|
||||
|
||||
insert_changelog_block_ordered() {
|
||||
local component="$1"
|
||||
local block_file="$2"
|
||||
local changelog="${component}/CHANGELOG.md"
|
||||
local incoming_heading incoming_release incoming_key
|
||||
local marker_line insertion_line duplicate_line total_lines
|
||||
local line heading existing_release existing_key
|
||||
|
||||
marker_line=$(grep -n -m1 '^<!-- changelog: release notes start -->$' "$changelog" | cut -d: -f1)
|
||||
incoming_heading=$(grep -m1 -E '^## \[[^]]+\] \(Prowler v?[0-9]+\.[0-9]+\.[0-9]+\)' "$block_file" || true)
|
||||
if [ -z "$incoming_heading" ]; then
|
||||
echo "::error::${block_file} does not contain a stamped Prowler release heading"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
incoming_release=$(release_from_heading "$incoming_heading")
|
||||
incoming_key=$(version_key "$incoming_release")
|
||||
insertion_line=""
|
||||
duplicate_line=""
|
||||
|
||||
while IFS=: read -r line heading; do
|
||||
existing_release=$(release_from_heading "$heading")
|
||||
existing_key=$(version_key "$existing_release")
|
||||
if [[ "$incoming_key" == "$existing_key" ]]; then
|
||||
duplicate_line="$line"
|
||||
break
|
||||
fi
|
||||
if [[ "$incoming_key" > "$existing_key" ]]; then
|
||||
insertion_line="$line"
|
||||
break
|
||||
fi
|
||||
done < <(grep -n -E '^## \[[^]]+\] \(Prowler v?[0-9]+\.[0-9]+\.[0-9]+\)' "$changelog" || true)
|
||||
|
||||
if [ -n "$duplicate_line" ]; then
|
||||
echo "::error::${changelog} already contains a block for Prowler v${incoming_release} at line ${duplicate_line}; refusing to insert a duplicate"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$insertion_line" ]; then
|
||||
insertion_line=$(($(wc -l < "$changelog") + 1))
|
||||
fi
|
||||
if [ "$insertion_line" -le "$marker_line" ]; then
|
||||
insertion_line=$((marker_line + 1))
|
||||
fi
|
||||
|
||||
# The captured block window can be off by one blank line on either
|
||||
# end (towncrier re-emits the blank after the marker), so strip the
|
||||
# outer blank lines and pad exactly one on each side: the block
|
||||
# must never glue to the marker above or the next heading below.
|
||||
awk '
|
||||
/[^[:space:]]/ { for (i = 0; i < pending; i++) print ""; pending = 0; print; started = 1; next }
|
||||
started { pending++ }
|
||||
' "$block_file" > "${RUNNER_TEMP}/block-normalized.md"
|
||||
|
||||
total_lines=$(wc -l < "$changelog")
|
||||
{
|
||||
head -n "$((insertion_line - 1))" "$changelog"
|
||||
if [ "$insertion_line" -gt 1 ] && [ -n "$(sed -n "$((insertion_line - 1))p" "$changelog")" ]; then
|
||||
echo ""
|
||||
fi
|
||||
cat "${RUNNER_TEMP}/block-normalized.md"
|
||||
if [ "$insertion_line" -le "$total_lines" ]; then
|
||||
echo ""
|
||||
fi
|
||||
tail -n +"$insertion_line" "$changelog"
|
||||
} > "${RUNNER_TEMP}/changelog.tmp"
|
||||
mv "${RUNNER_TEMP}/changelog.tmp" "$changelog"
|
||||
|
||||
echo "::notice::Inserted ${component} changelog block for Prowler v${incoming_release} at line ${insertion_line}"
|
||||
}
|
||||
|
||||
sync_body="${RUNNER_TEMP}/forward-sync-pr-body.md"
|
||||
{
|
||||
echo "### Description"
|
||||
echo ""
|
||||
echo "Forward-syncs the v${PROWLER_VERSION} compiled changelogs from \`${TARGET_BRANCH}\` to \`master\`: inserts the same stamped version blocks under the insertion marker and deletes the consumed fragments, so the next minor compile cannot re-release entries that already shipped in this patch. Opened automatically by the same run that opened the compile PR; review and squash-merge after it."
|
||||
echo ""
|
||||
echo "| Component | Fragments deleted on master | Skipped (only on ${TARGET_BRANCH}) |"
|
||||
echo "|---|---|---|"
|
||||
} > "$sync_body"
|
||||
|
||||
for component in $COMPILED_COMPONENTS; do
|
||||
block_file="${RUNNER_TEMP}/block-${component}.md"
|
||||
consumed_file="${RUNNER_TEMP}/consumed-${component}.txt"
|
||||
|
||||
if ! grep -qm1 '^<!-- changelog: release notes start -->$' "$component/CHANGELOG.md"; then
|
||||
echo "::error::${component}/CHANGELOG.md on master is missing the insertion marker; cannot forward-sync"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
deleted=0
|
||||
skipped=0
|
||||
while IFS= read -r fragment; do
|
||||
if [ -z "$fragment" ]; then
|
||||
continue
|
||||
fi
|
||||
if [ -f "$fragment" ]; then
|
||||
git rm -q "$fragment"
|
||||
deleted=$((deleted + 1))
|
||||
else
|
||||
echo "::notice::${fragment} does not exist on master (change landed only on ${TARGET_BRANCH}); skipping its deletion"
|
||||
skipped=$((skipped + 1))
|
||||
fi
|
||||
done < "$consumed_file"
|
||||
|
||||
insert_changelog_block_ordered "$component" "$block_file"
|
||||
|
||||
echo "| \`${component}\` | ${deleted} | ${skipped} |" >> "$sync_body"
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### License"
|
||||
echo ""
|
||||
echo "By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license."
|
||||
} >> "$sync_body"
|
||||
|
||||
echo "SYNC_BODY_FILE=${sync_body}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Create forward-sync PR
|
||||
if: env.TARGET_BRANCH != 'master'
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
token: ${{ secrets.PROWLER_BOT_ACCESS_TOKEN }}
|
||||
commit-message: 'chore(changelog): v${{ env.PROWLER_VERSION }} forward-sync to master'
|
||||
branch: forward-sync-changelogs-${{ env.PROWLER_VERSION }}
|
||||
base: master
|
||||
title: 'chore(changelog): v${{ env.PROWLER_VERSION }} forward-sync to master'
|
||||
body-path: ${{ env.SYNC_BODY_FILE }}
|
||||
author: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
|
||||
labels: |
|
||||
no-changelog
|
||||
skip-sync
|
||||
@@ -102,6 +102,7 @@ jobs:
|
||||
files_ignore: |
|
||||
mcp_server/README.md
|
||||
mcp_server/CHANGELOG.md
|
||||
mcp_server/changelog.d/**
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check-changes.outputs.any_changed == 'true'
|
||||
|
||||
@@ -19,6 +19,60 @@ concurrency:
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test-changelog-attribution:
|
||||
if: github.repository == 'prowler-cloud/prowler'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: block
|
||||
allowed-endpoints: >
|
||||
api.github.com:443
|
||||
github.com:443
|
||||
objects.githubusercontent.com:443
|
||||
pypi.org:443
|
||||
files.pythonhosted.org:443
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Fetch PR base ref for tj-actions/changed-files
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: git fetch --depth=1 origin "${BASE_REF}"
|
||||
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
files: |
|
||||
.github/scripts/changelog_attribution.py
|
||||
.github/workflows/pr-check-changelog.yml
|
||||
.github/workflows/compile-changelogs.yml
|
||||
.github/towncrier/template.md.jinja
|
||||
*/towncrier.toml
|
||||
tests/github/**
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Test changelog attribution
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: |
|
||||
python3 -m pip install --user --disable-pip-version-check pytest==9.0.3 towncrier==25.8.0
|
||||
python3 -m pytest tests/github
|
||||
|
||||
check-changelog:
|
||||
if: contains(github.event.pull_request.labels.*.name, 'no-changelog') == false
|
||||
runs-on: ubuntu-latest
|
||||
@@ -62,53 +116,160 @@ jobs:
|
||||
uv.lock
|
||||
pyproject.toml
|
||||
|
||||
- name: Check for folder changes and changelog presence
|
||||
- name: Check for folder changes and changelog fragment presence
|
||||
id: check-folders
|
||||
run: |
|
||||
missing_changelogs=""
|
||||
fragment_name_re='^[A-Za-z0-9][A-Za-z0-9._-]*\.(added|changed|deprecated|removed|fixed|security)(\.[0-9]+)?\.md$'
|
||||
manual_pr_link_re='(\[\(#[0-9]+\)\]|\[#[0-9]+\]\(|\(#[0-9]+\)|github\.com/[^[:space:]/]+/[^[:space:]/]+/(pull|issues)/[0-9]+)'
|
||||
folder_alt=$(echo "$MONITORED_FOLDERS" | tr ' ' '|')
|
||||
|
||||
missing_fragments=""
|
||||
invalid_fragments=""
|
||||
linked_fragments=""
|
||||
handwritten_changelogs=""
|
||||
|
||||
all_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n')
|
||||
added=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" | tr ' ' '\n')
|
||||
added_or_renamed=$(printf '%s\n%s' "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES}" | tr ' ' '\n')
|
||||
added_modified_or_renamed=$(printf '%s\n%s\n%s' "${STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_MODIFIED_FILES}" "${STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES}" | tr ' ' '\n')
|
||||
|
||||
# Returns success if the folder has a valid fragment added, modified, or renamed.
|
||||
has_changelog_update() {
|
||||
local folder="$1"
|
||||
if echo "$added_modified_or_renamed" | grep "^${folder}/changelog.d/" | sed "s|^${folder}/changelog.d/||" | grep -qE "$fragment_name_re"; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ "${STEPS_CHANGED_FILES_OUTPUTS_ANY_CHANGED}" == "true" ]]; then
|
||||
# Check monitored folders
|
||||
for folder in $MONITORED_FOLDERS; do
|
||||
# Get files changed in this folder
|
||||
changed_in_folder=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep "^${folder}/" || true)
|
||||
if echo "$all_changed" | grep -q "^${folder}/CHANGELOG.md$"; then
|
||||
echo "Direct CHANGELOG.md edits are not allowed for ${folder}/"
|
||||
handwritten_changelogs="${handwritten_changelogs}- \`${folder}/CHANGELOG.md\`"$'\n'
|
||||
fi
|
||||
|
||||
changed_in_folder=$(echo "$all_changed" | grep "^${folder}/" | grep -v "^${folder}/CHANGELOG.md$" || true)
|
||||
|
||||
if [ -n "$changed_in_folder" ]; then
|
||||
echo "Detected changes in ${folder}/"
|
||||
|
||||
# Check if CHANGELOG.md was updated
|
||||
if ! echo "$changed_in_folder" | grep -q "^${folder}/CHANGELOG.md$"; then
|
||||
echo "No changelog update found for ${folder}/"
|
||||
missing_changelogs="${missing_changelogs}- \`${folder}\`"$'\n'
|
||||
if ! has_changelog_update "$folder"; then
|
||||
echo "No changelog fragment found for ${folder}/"
|
||||
missing_fragments="${missing_fragments}- \`${folder}\`"$'\n'
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Check root-level dependency files (uv.lock, pyproject.toml)
|
||||
# These are associated with the prowler folder changelog
|
||||
root_deps_changed=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
|
||||
root_deps_changed=$(echo "$all_changed" | grep -E "^(uv\.lock|pyproject\.toml)$" || true)
|
||||
if [ -n "$root_deps_changed" ]; then
|
||||
echo "Detected changes in root dependency files: $root_deps_changed"
|
||||
# Check if prowler/CHANGELOG.md was already updated (might have been caught above)
|
||||
prowler_changelog_updated=$(echo "${STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES}" | tr ' ' '\n' | grep "^prowler/CHANGELOG.md$" || true)
|
||||
if [ -z "$prowler_changelog_updated" ]; then
|
||||
if ! has_changelog_update "prowler"; then
|
||||
# Only add if prowler wasn't already flagged
|
||||
if ! echo "$missing_changelogs" | grep -q "prowler"; then
|
||||
echo "No changelog update found for root dependency changes"
|
||||
missing_changelogs="${missing_changelogs}- \`prowler\` (root dependency files changed)"$'\n'
|
||||
if ! echo "$missing_fragments" | grep -q "prowler"; then
|
||||
echo "No changelog fragment found for root dependency changes"
|
||||
missing_fragments="${missing_fragments}- \`prowler\` (root dependency files changed)"$'\n'
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Validate the filename of every fragment added by this PR
|
||||
added_fragments=$(echo "$added_or_renamed" | grep -E "^(${folder_alt})/changelog\.d/" || true)
|
||||
for fragment in $added_fragments; do
|
||||
name=$(basename "$fragment")
|
||||
if [ "$name" = ".gitkeep" ] || [ "$name" = "README.md" ]; then
|
||||
continue
|
||||
fi
|
||||
if ! echo "$name" | grep -qE "$fragment_name_re"; then
|
||||
echo "Invalid fragment filename: $fragment"
|
||||
invalid_fragments="${invalid_fragments}- \`${fragment}\`"$'\n'
|
||||
fi
|
||||
done
|
||||
|
||||
# Lint fragment content: the PR link is attached automatically at
|
||||
# compile time, so a hand-written PR or issue link would be wrong
|
||||
touched_fragments=$(echo "$added_modified_or_renamed" | grep -E "^(${folder_alt})/changelog\.d/" || true)
|
||||
for fragment in $touched_fragments; do
|
||||
name=$(basename "$fragment")
|
||||
if [ "$name" = ".gitkeep" ] || [ "$name" = "README.md" ] || [ ! -f "$fragment" ]; then
|
||||
continue
|
||||
fi
|
||||
if grep -qE "$manual_pr_link_re" "$fragment"; then
|
||||
echo "Fragment contains a hand-written PR or issue link: $fragment"
|
||||
linked_fragments="${linked_fragments}- \`${fragment}\`"$'\n'
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
{
|
||||
echo "missing_changelogs<<EOF"
|
||||
echo -e "${missing_changelogs}"
|
||||
echo "EOF"
|
||||
} >> $GITHUB_OUTPUT
|
||||
# Suggest a slug derived from the branch name for the bot comment
|
||||
suggested_slug=$(echo "$HEAD_REF" | tr '[:upper:]' '[:lower:]' | sed 's|.*/||; s/[^a-z0-9._-]/-/g; s/^[^a-z0-9]*//')
|
||||
if [ -z "$suggested_slug" ]; then
|
||||
suggested_slug="my-change"
|
||||
fi
|
||||
|
||||
fragment_help="A changelog fragment is a small Markdown file named \`<slug>.<type>.md\` under \`<component>/changelog.d/\`, where \`<type>\` is one of \`added\`, \`changed\`, \`deprecated\`, \`removed\`, \`fixed\` or \`security\`. Its content is the changelog entry text, without the PR link (added automatically at release time) and without a trailing period. For example:
|
||||
|
||||
\`\`\`
|
||||
echo 'Entry text describing the change' > <component>/changelog.d/${suggested_slug}.fixed.md
|
||||
\`\`\`
|
||||
|
||||
If this PR does not need a changelog entry, add the \`no-changelog\` label instead."
|
||||
|
||||
if [ -n "$missing_fragments" ] || [ -n "$invalid_fragments" ] || [ -n "$linked_fragments" ] || [ -n "$handwritten_changelogs" ]; then
|
||||
comment_body=""
|
||||
if [ -n "$missing_fragments" ]; then
|
||||
comment_body="⚠️ **Changes detected in the following folders without a changelog fragment:**"$'\n\n'"${missing_fragments}"$'\n'
|
||||
fi
|
||||
if [ -n "$invalid_fragments" ]; then
|
||||
comment_body="${comment_body}⚠️ **Changelog fragment filenames that do not follow the naming convention:**"$'\n\n'"${invalid_fragments}"$'\n'
|
||||
fi
|
||||
if [ -n "$linked_fragments" ]; then
|
||||
comment_body="${comment_body}⚠️ **Changelog fragments containing a hand-written PR or issue link (remove it; the PR link is attached automatically at release time):**"$'\n\n'"${linked_fragments}"$'\n'
|
||||
fi
|
||||
if [ -n "$handwritten_changelogs" ]; then
|
||||
comment_body="${comment_body}⚠️ **Direct \`CHANGELOG.md\` edits are not allowed in regular PRs:**"$'\n\n'"${handwritten_changelogs}"$'\n'
|
||||
fi
|
||||
comment_body="${comment_body}${fragment_help}"
|
||||
else
|
||||
comment_body="✅ All required changelog fragments are present."
|
||||
fi
|
||||
|
||||
write_multiline_output() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
local delimiter
|
||||
|
||||
while true; do
|
||||
delimiter="EOF_$(openssl rand -hex 16)"
|
||||
if ! grep -qxF "$delimiter" <<< "$value"; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo "${name}<<${delimiter}"
|
||||
if [ -n "$value" ]; then
|
||||
printf '%s\n' "$value"
|
||||
fi
|
||||
echo "${delimiter}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
}
|
||||
|
||||
write_multiline_output "missing_fragments" "$missing_fragments"
|
||||
write_multiline_output "invalid_fragments" "$invalid_fragments"
|
||||
write_multiline_output "linked_fragments" "$linked_fragments"
|
||||
write_multiline_output "handwritten_changelogs" "$handwritten_changelogs"
|
||||
write_multiline_output "comment_body" "$comment_body"
|
||||
env:
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ANY_CHANGED: ${{ steps.changed-files.outputs.any_changed }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_MODIFIED_FILES: ${{ steps.changed-files.outputs.modified_files }}
|
||||
STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES: ${{ steps.changed-files.outputs.renamed_files }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
|
||||
- name: Find existing changelog comment
|
||||
if: github.event.pull_request.head.repo.full_name == github.repository
|
||||
@@ -128,14 +289,10 @@ jobs:
|
||||
edit-mode: replace
|
||||
body: |
|
||||
<!-- changelog-check -->
|
||||
${{ steps.check-folders.outputs.missing_changelogs != '' && format('⚠️ **Changes detected in the following folders without a corresponding update to the `CHANGELOG.md`:**
|
||||
${{ steps.check-folders.outputs.comment_body }}
|
||||
|
||||
{0}
|
||||
|
||||
Please add an entry to the corresponding `CHANGELOG.md` file to maintain a clear history of changes.', steps.check-folders.outputs.missing_changelogs) || '✅ All necessary `CHANGELOG.md` files have been updated.' }}
|
||||
|
||||
- name: Fail if changelog is missing
|
||||
if: steps.check-folders.outputs.missing_changelogs != ''
|
||||
- name: Fail if changelog fragment is missing or invalid
|
||||
if: steps.check-folders.outputs.missing_fragments != '' || steps.check-folders.outputs.invalid_fragments != '' || steps.check-folders.outputs.linked_fragments != '' || steps.check-folders.outputs.handwritten_changelogs != ''
|
||||
run: |
|
||||
echo "::error::Missing changelog updates in some folders"
|
||||
echo "::error::Missing, invalid, or disallowed changelog updates"
|
||||
exit 1
|
||||
|
||||
@@ -55,6 +55,7 @@ jobs:
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
|
||||
@@ -12,6 +12,7 @@ on:
|
||||
- '.github/workflows/sdk-codeql.yml'
|
||||
- '.github/codeql/sdk-codeql-config.yml'
|
||||
- '!prowler/CHANGELOG.md'
|
||||
- '!prowler/changelog.d/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
@@ -23,6 +24,7 @@ on:
|
||||
- '.github/workflows/sdk-codeql.yml'
|
||||
- '.github/codeql/sdk-codeql-config.yml'
|
||||
- '!prowler/CHANGELOG.md'
|
||||
- '!prowler/changelog.d/**'
|
||||
schedule:
|
||||
- cron: '00 12 * * *'
|
||||
|
||||
|
||||
@@ -113,6 +113,7 @@ jobs:
|
||||
.github/workflows/sdk-container-checks.yml
|
||||
files_ignore: |
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
|
||||
@@ -73,6 +73,7 @@ jobs:
|
||||
.github/scripts/osv-scan.sh
|
||||
files_ignore: |
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
**/AGENTS.md
|
||||
|
||||
- name: Setup Python with uv
|
||||
|
||||
@@ -77,6 +77,7 @@ jobs:
|
||||
files_ignore: |
|
||||
.github/**
|
||||
prowler/CHANGELOG.md
|
||||
prowler/changelog.d/**
|
||||
docs/**
|
||||
permissions/**
|
||||
api/**
|
||||
|
||||
@@ -10,6 +10,7 @@ on:
|
||||
- '.github/workflows/ui-codeql.yml'
|
||||
- '.github/codeql/ui-codeql-config.yml'
|
||||
- '!ui/CHANGELOG.md'
|
||||
- '!ui/changelog.d/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'master'
|
||||
@@ -19,6 +20,7 @@ on:
|
||||
- '.github/workflows/ui-codeql.yml'
|
||||
- '.github/codeql/ui-codeql-config.yml'
|
||||
- '!ui/CHANGELOG.md'
|
||||
- '!ui/changelog.d/**'
|
||||
schedule:
|
||||
- cron: '00 12 * * *'
|
||||
|
||||
|
||||
@@ -102,6 +102,7 @@ jobs:
|
||||
files: ui/**
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/changelog.d/**
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
|
||||
@@ -64,6 +64,7 @@ jobs:
|
||||
.github/workflows/ui-tests.yml
|
||||
files_ignore: |
|
||||
ui/CHANGELOG.md
|
||||
ui/changelog.d/**
|
||||
ui/README.md
|
||||
ui/AGENTS.md
|
||||
|
||||
|
||||
@@ -2,6 +2,33 @@
|
||||
|
||||
All notable changes to the **Prowler API** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [1.34.2] (Prowler v5.33.2)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Attack Paths graph mutations now retry transient Neptune concurrency and deadline failures, while Neo4j mutations use managed transaction retries [(#11968)](https://github.com/prowler-cloud/prowler/pull/11968)
|
||||
- Attack Paths scans now use bounded child node identifiers for normalized list values in Neo4j and Neptune, preventing Neo4j RANGE index key size failures [(#11969)](https://github.com/prowler-cloud/prowler/pull/11969)
|
||||
- `scan-summary` aggregation now upserts summaries in deterministic conflict-key order, preventing PostgreSQL deadlocks during concurrent reaggregation [(#11971)](https://github.com/prowler-cloud/prowler/pull/11971)
|
||||
|
||||
---
|
||||
|
||||
## [1.34.1] (Prowler v5.33.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Session tokens are rejected after account password updates [(#11914)](https://github.com/prowler-cloud/prowler/pull/11914)
|
||||
- Jira dispatch task results now surface user-facing Jira failure messages [(#11925)](https://github.com/prowler-cloud/prowler/pull/11925)
|
||||
- AWS Attack Paths privilege escalation queries no longer fail on Neo4j with `Aggregation column contains implicit grouping expressions` [(#11939)](https://github.com/prowler-cloud/prowler/pull/11939)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- OpenAI-compatible Lighthouse provider base URLs are restricted before connection checks [(#11940)](https://github.com/prowler-cloud/prowler/pull/11940)
|
||||
- `LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS` environment variable to allow internal hosts as OpenAI-compatible Lighthouse AI base URLs [(#11942)](https://github.com/prowler-cloud/prowler/pull/11942)
|
||||
|
||||
---
|
||||
|
||||
## [1.34.0] (Prowler v5.33.0)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Changelog fragments
|
||||
|
||||
Each PR adds one small file here instead of editing `CHANGELOG.md` directly, so concurrent PRs never conflict.
|
||||
|
||||
- Filename: `<slug>.<type>.md`, e.g. `my-new-check.added.md` (slug is free-form: letters, digits, `.`, `_`, `-`)
|
||||
- `<type>` is one of: `added`, `changed`, `deprecated`, `removed`, `fixed`, `security`
|
||||
- Content: one line with the changelog entry text, without the PR link and without a trailing period (the PR link is attached automatically at release time)
|
||||
- A PR adds as many fragment files as entries it needs, freely mixing types (one file per entry); same-type entries just use different slugs
|
||||
|
||||
Fragments are compiled into `CHANGELOG.md` when a release is prepared. Full conventions: `skills/prowler-changelog/SKILL.md`.
|
||||
+2
-2
@@ -45,7 +45,7 @@ dependencies = [
|
||||
"gunicorn==26.0.0",
|
||||
"uvloop==0.22.1",
|
||||
"lxml==6.1.0",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@master",
|
||||
"prowler @ git+https://github.com/prowler-cloud/prowler.git@v5.33",
|
||||
"psycopg2-binary==2.9.9",
|
||||
"pytest-celery[redis] (==1.3.0)",
|
||||
"sentry-sdk[django] (==2.56.0)",
|
||||
@@ -71,7 +71,7 @@ name = "prowler-api"
|
||||
package-mode = false
|
||||
# Needed for the SDK compatibility
|
||||
requires-python = ">=3.11,<3.13"
|
||||
version = "1.34.0"
|
||||
version = "1.34.2"
|
||||
|
||||
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
|
||||
# target-version tracks this project's lowest supported Python.
|
||||
|
||||
@@ -418,7 +418,8 @@ AWS_APPRUNNER_PRIVESC_UPDATE_SERVICE = AttackPathsQueryDefinition(
|
||||
// Find existing App Runner services with roles attached (potential targets)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'tasks.apprunner.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -523,7 +524,8 @@ AWS_BEDROCK_PRIVESC_INVOKE_CODE_INTERPRETER = AttackPathsQueryDefinition(
|
||||
// Find roles that trust the Bedrock AgentCore service (already attached to existing code interpreters)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'bedrock-agentcore.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -607,7 +609,8 @@ AWS_CLOUDFORMATION_PRIVESC_UPDATE_STACK = AttackPathsQueryDefinition(
|
||||
// Find roles that trust CloudFormation service (already attached to existing stacks)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -753,7 +756,8 @@ AWS_CLOUDFORMATION_PRIVESC_CHANGESET = AttackPathsQueryDefinition(
|
||||
// Find roles that trust CloudFormation service (already attached to existing stacks)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'cloudformation.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -844,7 +848,8 @@ AWS_CODEBUILD_PRIVESC_START_BUILD = AttackPathsQueryDefinition(
|
||||
// Find roles that trust CodeBuild service (already attached to existing projects)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -880,7 +885,8 @@ AWS_CODEBUILD_PRIVESC_START_BUILD_BATCH = AttackPathsQueryDefinition(
|
||||
// Find roles that trust CodeBuild service (already attached to existing projects)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'codebuild.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1096,7 +1102,8 @@ AWS_EC2_PRIVESC_MODIFY_INSTANCE_ATTRIBUTE = AttackPathsQueryDefinition(
|
||||
// Find EC2 instances with instance profiles (potential targets)
|
||||
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1187,7 +1194,8 @@ AWS_EC2_PRIVESC_LAUNCH_TEMPLATE = AttackPathsQueryDefinition(
|
||||
// Find launch templates in the account (potential targets)
|
||||
MATCH path_target = (aws)--(template:LaunchTemplate)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1223,7 +1231,8 @@ AWS_EC2INSTANCECONNECT_PRIVESC_SEND_SSH_PUBLIC_KEY = AttackPathsQueryDefinition(
|
||||
// Find EC2 instances with attached roles (targets for credential theft via IMDS)
|
||||
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1539,7 +1548,8 @@ AWS_ECS_PRIVESC_EXECUTE_COMMAND = AttackPathsQueryDefinition(
|
||||
// Target: roles already attached to running tasks (trust ECS tasks service)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'ecs-tasks.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -1622,7 +1632,8 @@ AWS_GLUE_PRIVESC_UPDATE_DEV_ENDPOINT = AttackPathsQueryDefinition(
|
||||
// Find roles that trust Glue service (already attached to existing dev endpoints)
|
||||
MATCH path_target = (aws)--(target_role:AWSRole)-[:TRUSTS_AWS_PRINCIPAL]->(:AWSPrincipal {{arn: 'glue.amazonaws.com'}})
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -3337,7 +3348,8 @@ AWS_SSM_PRIVESC_START_SESSION = AttackPathsQueryDefinition(
|
||||
// Find EC2 instances with attached roles (targets for credential theft via IMDS)
|
||||
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
@@ -3373,7 +3385,8 @@ AWS_SSM_PRIVESC_SEND_COMMAND = AttackPathsQueryDefinition(
|
||||
// Find EC2 instances with attached roles (targets for credential theft via IMDS)
|
||||
MATCH path_target = (aws)--(ec2:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(target_role:AWSRole)
|
||||
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
UNWIND paths AS p
|
||||
UNWIND nodes(p) AS n
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
@@ -9,17 +11,19 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RetryableSession:
|
||||
"""
|
||||
Wrapper around `neo4j.Session` that retries `neo4j.exceptions.ServiceUnavailable` errors.
|
||||
"""
|
||||
"""Wrapper around ``neo4j.Session`` with a refreshable retry policy."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: Callable[[], neo4j.Session],
|
||||
max_retries: int,
|
||||
retry_if: Callable[[Exception], bool] | None = None,
|
||||
initial_retry_delay_seconds: float = 0,
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._max_retries = max(0, max_retries)
|
||||
self._retry_if = retry_if
|
||||
self._initial_retry_delay_seconds = max(0.0, initial_retry_delay_seconds)
|
||||
self._session = self._session_factory()
|
||||
|
||||
def close(self) -> None:
|
||||
@@ -56,24 +60,47 @@ class RetryableSession:
|
||||
method = getattr(self._session, method_name)
|
||||
return method(*args, **kwargs)
|
||||
|
||||
except (
|
||||
BrokenPipeError,
|
||||
ConnectionResetError,
|
||||
neo4j.exceptions.ServiceUnavailable,
|
||||
) as exc: # pragma: no cover - depends on infra
|
||||
except Exception as exc:
|
||||
if not self._should_retry(exc):
|
||||
raise
|
||||
|
||||
last_exc = exc
|
||||
attempt += 1
|
||||
|
||||
if attempt > self._max_retries:
|
||||
raise
|
||||
|
||||
delay = self._retry_delay(attempt)
|
||||
logger.warning(
|
||||
f"Neo4j session {method_name} failed with {type(exc).__name__} ({attempt}/{self._max_retries} attempts). Retrying..."
|
||||
"Graph session %s failed with %s; retry %s/%s in %.3fs",
|
||||
method_name,
|
||||
type(exc).__name__,
|
||||
attempt,
|
||||
self._max_retries,
|
||||
delay,
|
||||
)
|
||||
self._refresh_session()
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
|
||||
raise last_exc if last_exc else RuntimeError("Unexpected retry loop exit")
|
||||
|
||||
def _should_retry(self, exc: Exception) -> bool:
|
||||
if isinstance(
|
||||
exc,
|
||||
(
|
||||
BrokenPipeError,
|
||||
ConnectionResetError,
|
||||
neo4j.exceptions.ServiceUnavailable,
|
||||
),
|
||||
):
|
||||
return True
|
||||
return self._retry_if(exc) if self._retry_if else False
|
||||
|
||||
def _retry_delay(self, attempt: int) -> float:
|
||||
max_delay = self._initial_retry_delay_seconds * (2**attempt)
|
||||
return random.uniform(max_delay / 2, max_delay) if max_delay else 0
|
||||
|
||||
def _refresh_session(self) -> None:
|
||||
if self._session is not None:
|
||||
try:
|
||||
|
||||
@@ -42,6 +42,10 @@ def delete_batches(
|
||||
batch_size: int,
|
||||
drop_t0: float,
|
||||
) -> tuple[int, int]:
|
||||
def delete_batch(tx: Any) -> int:
|
||||
record = tx.run(query, {"batch_size": batch_size}).single()
|
||||
return (record[count_key] if record else 0) or 0
|
||||
|
||||
deleted_total = initial_total
|
||||
batches = 0
|
||||
while True:
|
||||
@@ -56,8 +60,7 @@ def delete_batches(
|
||||
deleted_total,
|
||||
time.perf_counter() - drop_t0,
|
||||
)
|
||||
record = session.run(query, {"batch_size": batch_size}).single()
|
||||
deleted = (record[count_key] if record else 0) or 0
|
||||
deleted = session.execute_write(delete_batch)
|
||||
if deleted == 0:
|
||||
return deleted_total, batches
|
||||
|
||||
|
||||
@@ -355,7 +355,7 @@ class Neo4jSink(SinkDatabase):
|
||||
f"ON (n.`{PROVIDER_ELEMENT_ID_PROPERTY}`)"
|
||||
)
|
||||
with self.get_session(database) as session:
|
||||
session.run(query).consume()
|
||||
session.execute_write(lambda tx: tx.run(query).consume())
|
||||
|
||||
def write_nodes(
|
||||
self,
|
||||
@@ -377,7 +377,7 @@ class Neo4jSink(SinkDatabase):
|
||||
SET n += row.props
|
||||
"""
|
||||
with self.get_session(database) as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
@@ -403,7 +403,7 @@ class Neo4jSink(SinkDatabase):
|
||||
SET r += row.props
|
||||
"""
|
||||
with self.get_session(database) as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
# For compatibility with test harnesses that patch the concrete driver
|
||||
def get_driver(self) -> neo4j.Driver:
|
||||
|
||||
@@ -59,17 +59,28 @@ CONNECTION_TIMEOUT = env.int("NEPTUNE_CONNECTION_TIMEOUT", default=10)
|
||||
# Roll connections hourly so SigV4 rotations and cert refreshes don't strand long-lived pool entries
|
||||
MAX_CONNECTION_LIFETIME = env.int("NEPTUNE_MAX_CONNECTION_LIFETIME", default=3600)
|
||||
MAX_CONNECTION_POOL_SIZE = env.int("NEPTUNE_MAX_CONNECTION_POOL_SIZE", default=50)
|
||||
NEPTUNE_WRITE_RETRY_DELAY_SECONDS = 2
|
||||
|
||||
READ_EXCEPTION_CODES = [
|
||||
"Neo.ClientError.Statement.AccessMode",
|
||||
"Neo.ClientError.Procedure.ProcedureNotFound",
|
||||
]
|
||||
CLIENT_STATEMENT_EXCEPTION_PREFIX = "Neo.ClientError.Statement."
|
||||
RETRYABLE_WRITE_ERROR_PREFIXES = (
|
||||
"Operation failed due to conflicting concurrent operations",
|
||||
"Operation terminated (deadline exceeded)",
|
||||
)
|
||||
|
||||
# Refresh 60s before the 5-minute SigV4 window closes
|
||||
SIGV4_TOKEN_LIFETIME_MINUTES = 4
|
||||
|
||||
|
||||
def _is_retryable_write_error(exc: Exception) -> bool:
|
||||
if not isinstance(exc, neo4j.exceptions.Neo4jError):
|
||||
return False
|
||||
return bool(exc.message and exc.message.startswith(RETRYABLE_WRITE_ERROR_PREFIXES))
|
||||
|
||||
|
||||
class NeptuneSink(SinkDatabase):
|
||||
"""Neptune-backed sink. Single database; isolation is label-based."""
|
||||
|
||||
@@ -205,11 +216,16 @@ class NeptuneSink(SinkDatabase):
|
||||
|
||||
session_wrapper: RetryableSession | None = None
|
||||
try:
|
||||
is_write_session = default_access_mode != neo4j.READ_ACCESS
|
||||
session_wrapper = RetryableSession(
|
||||
session_factory=lambda: driver.session(
|
||||
default_access_mode=default_access_mode
|
||||
),
|
||||
max_retries=SERVICE_UNAVAILABLE_MAX_RETRIES,
|
||||
retry_if=_is_retryable_write_error if is_write_session else None,
|
||||
initial_retry_delay_seconds=(
|
||||
NEPTUNE_WRITE_RETRY_DELAY_SECONDS if is_write_session else 0
|
||||
),
|
||||
)
|
||||
yield session_wrapper
|
||||
|
||||
@@ -405,7 +421,7 @@ class NeptuneSink(SinkDatabase):
|
||||
SET n.`{PROVIDER_ELEMENT_ID_PROPERTY}` = row.provider_element_id
|
||||
"""
|
||||
with self.get_session() as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
def write_relationships(
|
||||
self,
|
||||
@@ -429,7 +445,7 @@ class NeptuneSink(SinkDatabase):
|
||||
SET r += row.props
|
||||
"""
|
||||
with self.get_session() as session:
|
||||
session.run(query, {"rows": rows}).consume()
|
||||
session.execute_write(lambda tx: tx.run(query, {"rows": rows}).consume())
|
||||
|
||||
# Test helpers
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.34.0
|
||||
version: 1.34.2
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
|
||||
class SSEChannelManager(DefaultChannelManager):
|
||||
"""Connect `django-eventstream` to the platform's SSE viewsets."""
|
||||
|
||||
def get_channels_for_request(self, request: Request, view_kwargs: dict) -> set[str]: # noqa: vulture
|
||||
def get_channels_for_request(self, request: Request, view_kwargs: dict) -> set[str]:
|
||||
"""Return the request's channels scoped to the active JWT tenant.
|
||||
|
||||
Args:
|
||||
@@ -30,6 +30,7 @@ class SSEChannelManager(DefaultChannelManager):
|
||||
The subset of `request.sse_channels` whose embedded tenant
|
||||
matches the active request tenant.
|
||||
"""
|
||||
_ = view_kwargs
|
||||
try:
|
||||
request_tenant_id = UUID(str(getattr(request, "tenant_id", None)))
|
||||
except (TypeError, ValueError):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
@@ -8,6 +9,10 @@ from conftest import TEST_PASSWORD, get_api_tokens, get_authorization_header
|
||||
from django.urls import reverse
|
||||
from drf_simple_apikey.crypto import get_crypto
|
||||
from rest_framework.test import APIClient
|
||||
from rest_framework_simplejwt.token_blacklist.models import (
|
||||
BlacklistedToken,
|
||||
OutstandingToken,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -103,6 +108,118 @@ def test_refresh_token(create_test_user, tenants_fixture):
|
||||
assert new_refresh_response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_password_change_invalidates_existing_tokens(create_test_user, tenants_fixture):
|
||||
client = APIClient()
|
||||
new_password = "ChangedSecret123@"
|
||||
|
||||
access_token, refresh_token = get_api_tokens(
|
||||
client, create_test_user.email, TEST_PASSWORD
|
||||
)
|
||||
auth_headers = get_authorization_header(access_token)
|
||||
outstanding_token_ids = list(
|
||||
OutstandingToken.objects.filter(user=create_test_user).values_list(
|
||||
"id", flat=True
|
||||
)
|
||||
)
|
||||
assert outstanding_token_ids
|
||||
assert not BlacklistedToken.objects.filter(
|
||||
token_id__in=outstanding_token_ids
|
||||
).exists()
|
||||
|
||||
password_change_payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(create_test_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
}
|
||||
}
|
||||
password_change_response = client.patch(
|
||||
reverse("user-detail", kwargs={"pk": create_test_user.id}),
|
||||
data=json.dumps(password_change_payload),
|
||||
headers=auth_headers,
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
assert password_change_response.status_code == 200, password_change_response.json()
|
||||
assert BlacklistedToken.objects.filter(
|
||||
token_id__in=outstanding_token_ids
|
||||
).count() == len(outstanding_token_ids)
|
||||
|
||||
old_access_response = client.get(reverse("user-me"), headers=auth_headers)
|
||||
assert old_access_response.status_code == 401
|
||||
|
||||
old_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "tokens-refresh",
|
||||
"attributes": {"refresh": refresh_token},
|
||||
}
|
||||
},
|
||||
format="vnd.api+json",
|
||||
)
|
||||
assert old_refresh_response.status_code == 400
|
||||
|
||||
new_access_token, _ = get_api_tokens(client, create_test_user.email, new_password)
|
||||
new_access_response = client.get(
|
||||
reverse("user-me"), headers=get_authorization_header(new_access_token)
|
||||
)
|
||||
assert new_access_response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_password_change_invalidates_rotated_refresh_token(
|
||||
create_test_user, tenants_fixture
|
||||
):
|
||||
client = APIClient()
|
||||
new_password = "ChangedSecret123@"
|
||||
|
||||
access_token, refresh_token = get_api_tokens(
|
||||
client, create_test_user.email, TEST_PASSWORD
|
||||
)
|
||||
rotated_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "tokens-refresh",
|
||||
"attributes": {"refresh": refresh_token},
|
||||
}
|
||||
},
|
||||
format="vnd.api+json",
|
||||
)
|
||||
assert rotated_refresh_response.status_code == 200
|
||||
rotated_refresh_token = rotated_refresh_response.json()["data"]["attributes"][
|
||||
"refresh"
|
||||
]
|
||||
|
||||
password_change_payload = {
|
||||
"data": {
|
||||
"type": "users",
|
||||
"id": str(create_test_user.id),
|
||||
"attributes": {"password": new_password},
|
||||
}
|
||||
}
|
||||
password_change_response = client.patch(
|
||||
reverse("user-detail", kwargs={"pk": create_test_user.id}),
|
||||
data=json.dumps(password_change_payload),
|
||||
headers=get_authorization_header(access_token),
|
||||
content_type="application/vnd.api+json",
|
||||
)
|
||||
assert password_change_response.status_code == 200, password_change_response.json()
|
||||
|
||||
old_rotated_refresh_response = client.post(
|
||||
reverse("token-refresh"),
|
||||
data={
|
||||
"data": {
|
||||
"type": "tokens-refresh",
|
||||
"attributes": {"refresh": rotated_refresh_token},
|
||||
}
|
||||
},
|
||||
format="vnd.api+json",
|
||||
)
|
||||
assert old_rotated_refresh_response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_user_me_when_inviting_users(create_test_user, tenants_fixture, roles_fixture):
|
||||
client = APIClient()
|
||||
@@ -189,6 +306,8 @@ def test_user_me_when_inviting_users(create_test_user, tenants_fixture, roles_fi
|
||||
class TestTokenSwitchTenant:
|
||||
def test_switch_tenant_with_valid_token(self, tenants_fixture, providers_fixture):
|
||||
client = APIClient()
|
||||
aws_provider = providers_fixture[0]
|
||||
assert aws_provider
|
||||
|
||||
test_user = "test_email@prowler.com"
|
||||
test_password = "Test_password1@"
|
||||
@@ -1403,6 +1522,8 @@ class TestAPIKeyMultiTenantWorkflows:
|
||||
Verifies RLS enforcement after authentication ensures tenant isolation.
|
||||
"""
|
||||
client = APIClient()
|
||||
aws_provider = providers_fixture[0]
|
||||
assert aws_provider
|
||||
|
||||
user1 = User.objects.create_user(
|
||||
name="tenant1_user",
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from api.attack_paths.retryable_session import RetryableSession
|
||||
from neo4j.exceptions import ServiceUnavailable
|
||||
|
||||
|
||||
class TestRetryableSession:
|
||||
@patch("api.attack_paths.retryable_session.time.sleep")
|
||||
@patch("api.attack_paths.retryable_session.random.uniform", return_value=3.0)
|
||||
def test_custom_retry_uses_backoff_and_a_fresh_session(
|
||||
self, mock_uniform, mock_sleep
|
||||
):
|
||||
retryable_error = RuntimeError("retryable")
|
||||
first_session = MagicMock()
|
||||
first_session.execute_write.side_effect = retryable_error
|
||||
second_session = MagicMock()
|
||||
second_session.execute_write.return_value = "success"
|
||||
session_factory = MagicMock(side_effect=[first_session, second_session])
|
||||
work = MagicMock()
|
||||
|
||||
session = RetryableSession(
|
||||
session_factory=session_factory,
|
||||
max_retries=3,
|
||||
retry_if=lambda exc: exc is retryable_error,
|
||||
initial_retry_delay_seconds=2,
|
||||
)
|
||||
|
||||
assert session.execute_write(work) == "success"
|
||||
assert session_factory.call_count == 2
|
||||
first_session.close.assert_called_once_with()
|
||||
mock_uniform.assert_called_once_with(2.0, 4.0)
|
||||
mock_sleep.assert_called_once_with(3.0)
|
||||
|
||||
def test_connection_errors_remain_retryable(self):
|
||||
first_session = MagicMock()
|
||||
first_session.run.side_effect = ServiceUnavailable("unavailable")
|
||||
second_session = MagicMock()
|
||||
second_session.run.return_value = "success"
|
||||
session_factory = MagicMock(side_effect=[first_session, second_session])
|
||||
|
||||
session = RetryableSession(session_factory=session_factory, max_retries=1)
|
||||
|
||||
assert session.run("RETURN 1") == "success"
|
||||
first_session.close.assert_called_once_with()
|
||||
|
||||
def test_non_retryable_error_is_raised_without_refreshing_session(self):
|
||||
error = RuntimeError("do not retry")
|
||||
driver_session = MagicMock()
|
||||
driver_session.execute_write.side_effect = error
|
||||
session_factory = MagicMock(return_value=driver_session)
|
||||
session = RetryableSession(
|
||||
session_factory=session_factory,
|
||||
max_retries=3,
|
||||
retry_if=lambda _: False,
|
||||
initial_retry_delay_seconds=2,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
session.execute_write(MagicMock())
|
||||
|
||||
assert exc_info.value is error
|
||||
session_factory.assert_called_once_with()
|
||||
driver_session.close.assert_not_called()
|
||||
|
||||
def test_retry_exhaustion_raises_the_last_error(self):
|
||||
error = RuntimeError("still retryable")
|
||||
driver_sessions = [MagicMock() for _ in range(3)]
|
||||
for driver_session in driver_sessions:
|
||||
driver_session.execute_write.side_effect = error
|
||||
session_factory = MagicMock(side_effect=driver_sessions)
|
||||
session = RetryableSession(
|
||||
session_factory=session_factory,
|
||||
max_retries=2,
|
||||
retry_if=lambda _: True,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
session.execute_write(MagicMock())
|
||||
|
||||
assert exc_info.value is error
|
||||
assert session_factory.call_count == 3
|
||||
driver_sessions[0].close.assert_called_once_with()
|
||||
driver_sessions[1].close.assert_called_once_with()
|
||||
driver_sessions[2].close.assert_not_called()
|
||||
@@ -6,18 +6,20 @@ builds dual writer/reader Bolt drivers.
|
||||
"""
|
||||
|
||||
import json
|
||||
from importlib import import_module
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import neo4j
|
||||
import pytest
|
||||
|
||||
# Prime patch-target resolution. `api.attack_paths.sink/__init__.py` doesn't
|
||||
# eagerly import these submodules (they're loaded on demand inside the
|
||||
# factory), so `mock.patch("api.attack_paths.sink.<sub>.…")` would fail with
|
||||
# AttributeError on first call. Importing here registers them as attributes
|
||||
# of the package before any decorator runs.
|
||||
import_module("api.attack_paths.sink.neo4j")
|
||||
import_module("api.attack_paths.sink.neptune")
|
||||
from api.attack_paths import sink as sink_module
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.attack_paths.sink import factory
|
||||
from api.attack_paths.sink.neo4j import DATABASE_NOT_FOUND_CODE, Neo4jSink
|
||||
from api.attack_paths.sink.neptune import (
|
||||
NEPTUNE_WRITE_RETRY_DELAY_SECONDS,
|
||||
NeptuneSink,
|
||||
_is_retryable_write_error,
|
||||
_NeptuneAuthToken,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -26,8 +28,6 @@ def reset_sink_state():
|
||||
|
||||
The cache lives in `api.attack_paths.sink.factory`, not on the package.
|
||||
"""
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
original_backend = factory._backend
|
||||
original_secondary = dict(factory._secondary_backends)
|
||||
factory._backend = None
|
||||
@@ -40,29 +40,20 @@ def reset_sink_state():
|
||||
|
||||
class TestSinkFactory:
|
||||
def test_default_resolves_to_neo4j(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neo4j"
|
||||
assert factory._resolve_setting() == "neo4j"
|
||||
|
||||
def test_neptune_resolves_correctly(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neptune"
|
||||
assert factory._resolve_setting() == "neptune"
|
||||
|
||||
def test_invalid_value_raises(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "foo"
|
||||
with pytest.raises(RuntimeError, match="ATTACK_PATHS_SINK_DATABASE"):
|
||||
factory._resolve_setting()
|
||||
|
||||
@patch("api.attack_paths.sink.neo4j.neo4j.GraphDatabase.driver")
|
||||
def test_init_builds_neo4j_backend_by_default(self, mock_driver, settings):
|
||||
from api.attack_paths import sink as sink_module
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neo4j"
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
@@ -85,9 +76,6 @@ class TestSinkFactory:
|
||||
def test_init_builds_neptune_backend(
|
||||
self, mock_driver, mock_auth_provider, settings
|
||||
):
|
||||
from api.attack_paths import sink as sink_module
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neptune"
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
@@ -116,8 +104,6 @@ class TestSinkFactory:
|
||||
def test_neptune_reader_falls_back_to_writer(
|
||||
self, mock_driver, mock_auth_provider, settings
|
||||
):
|
||||
from api.attack_paths import sink as sink_module
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neptune"
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
@@ -144,8 +130,6 @@ class TestGetBackendForScan:
|
||||
def test_legacy_scan_in_neo4j_process_uses_active_backend(
|
||||
self, mock_driver, settings
|
||||
):
|
||||
from api.attack_paths import sink as sink_module
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neo4j"
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
@@ -164,8 +148,6 @@ class TestGetBackendForScan:
|
||||
assert backend is sink_module.get_backend()
|
||||
|
||||
def test_neptune_scan_on_neo4j_process_uses_neptune_secondary(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neo4j"
|
||||
active_neo4j = MagicMock(name="neo4j-active")
|
||||
factory._backend = active_neo4j
|
||||
@@ -190,6 +172,29 @@ def _count_result(key: str, count: int) -> MagicMock:
|
||||
return MagicMock(single=MagicMock(return_value={key: count}))
|
||||
|
||||
|
||||
def _run_managed_write(session: MagicMock) -> MagicMock:
|
||||
transaction = MagicMock()
|
||||
session.execute_write.call_args.args[0](transaction)
|
||||
return transaction
|
||||
|
||||
|
||||
def _managed_write_session(
|
||||
results: list[MagicMock],
|
||||
) -> tuple[MagicMock, list[MagicMock]]:
|
||||
session = MagicMock()
|
||||
transactions: list[MagicMock] = []
|
||||
result_iter = iter(results)
|
||||
|
||||
def execute_write(work):
|
||||
transaction = MagicMock()
|
||||
transaction.run.return_value = next(result_iter)
|
||||
transactions.append(transaction)
|
||||
return work(transaction)
|
||||
|
||||
session.execute_write.side_effect = execute_write
|
||||
return session, transactions
|
||||
|
||||
|
||||
def _directed_drop_results(
|
||||
outgoing_rels: int,
|
||||
incoming_rels: int,
|
||||
@@ -207,31 +212,26 @@ def _directed_drop_results(
|
||||
|
||||
class TestNeo4jSinkSyncWrites:
|
||||
def test_ensure_sync_indexes_runs_create_index_idempotent(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.return_value = MagicMock()
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
sink.ensure_sync_indexes("db-tenant-x")
|
||||
|
||||
query = session.run.call_args.args[0]
|
||||
transaction = _run_managed_write(session)
|
||||
query = transaction.run.call_args.args[0]
|
||||
assert "CREATE INDEX" in query
|
||||
assert "IF NOT EXISTS" in query
|
||||
assert "`_ProviderResource`" in query
|
||||
assert "`_provider_element_id`" in query
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
def test_write_nodes_skips_empty_batch(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
with patch.object(sink, "get_session") as get_session:
|
||||
sink.write_nodes("db-tenant-x", "`AWSUser`", [])
|
||||
get_session.assert_not_called()
|
||||
|
||||
def test_write_nodes_merges_on_provider_resource_label(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
@@ -241,15 +241,15 @@ class TestNeo4jSinkSyncWrites:
|
||||
[{"provider_element_id": "p:e", "props": {"k": "v"}}],
|
||||
)
|
||||
|
||||
query, params = session.run.call_args.args
|
||||
transaction = _run_managed_write(session)
|
||||
query, params = transaction.run.call_args.args
|
||||
assert "MERGE (n:`_ProviderResource`" in query
|
||||
assert "`_provider_element_id`: row.provider_element_id" in query
|
||||
assert "SET n:`AWSUser`:`_ProviderResource`" in query
|
||||
assert params == {"rows": [{"provider_element_id": "p:e", "props": {"k": "v"}}]}
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
def test_write_relationships_scopes_endpoints_by_provider_label(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
provider_id = "00000000-0000-0000-0000-000000000abc"
|
||||
@@ -268,24 +268,22 @@ class TestNeo4jSinkSyncWrites:
|
||||
],
|
||||
)
|
||||
|
||||
query = session.run.call_args.args[0]
|
||||
transaction = _run_managed_write(session)
|
||||
query = transaction.run.call_args.args[0]
|
||||
assert ":`_Provider_00000000000000000000000000000abc`" in query
|
||||
assert ":RESOURCE" in query.replace("`", "")
|
||||
assert "MERGE (s)-[r:`RESOURCE`" in query
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
|
||||
class TestNeptuneSinkSyncWrites:
|
||||
def test_ensure_sync_indexes_is_noop(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
with patch.object(sink, "get_session") as get_session:
|
||||
sink.ensure_sync_indexes("ignored")
|
||||
get_session.assert_not_called()
|
||||
|
||||
def test_write_nodes_merges_on_neptune_id_with_provider_resource_label(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
@@ -295,16 +293,16 @@ class TestNeptuneSinkSyncWrites:
|
||||
[{"provider_element_id": "p:e", "props": {"k": "v"}}],
|
||||
)
|
||||
|
||||
query = session.run.call_args.args[0]
|
||||
transaction = _run_managed_write(session)
|
||||
query = transaction.run.call_args.args[0]
|
||||
# Neptune assigns a default `vertex` label to any unlabeled node,
|
||||
# so the MERGE must pin a real label at creation time.
|
||||
assert "MERGE (n:`_ProviderResource` {`~id`: row.provider_element_id})" in query
|
||||
assert "SET n:`AWSUser`" in query
|
||||
assert "SET n.`_provider_element_id` = row.provider_element_id" in query
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
def test_write_relationships_matches_endpoints_by_id(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
@@ -322,30 +320,86 @@ class TestNeptuneSinkSyncWrites:
|
||||
],
|
||||
)
|
||||
|
||||
query = session.run.call_args.args[0]
|
||||
transaction = _run_managed_write(session)
|
||||
query = transaction.run.call_args.args[0]
|
||||
assert "MATCH (s) WHERE id(s) = row.start_element_id" in query
|
||||
assert "MATCH (e) WHERE id(e) = row.end_element_id" in query
|
||||
assert "MERGE (s)-[r:`RESOURCE`" in query
|
||||
transaction.run.return_value.consume.assert_called_once_with()
|
||||
|
||||
|
||||
class TestNeptuneRetryPolicy:
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
"Operation failed due to conflicting concurrent operations "
|
||||
+ "(please retry), 0 transactions are currently rolling back.",
|
||||
"Operation terminated (deadline exceeded)",
|
||||
],
|
||||
)
|
||||
def test_observed_transient_write_errors_are_retryable(self, message):
|
||||
error = MagicMock(spec=neo4j.exceptions.Neo4jError)
|
||||
error.message = message
|
||||
|
||||
assert _is_retryable_write_error(error) is True
|
||||
|
||||
def test_unrelated_database_error_is_not_retryable(self):
|
||||
error = MagicMock(spec=neo4j.exceptions.Neo4jError)
|
||||
error.message = "Operation terminated (out of memory)"
|
||||
|
||||
assert _is_retryable_write_error(error) is False
|
||||
|
||||
def test_non_neo4j_error_is_not_retryable(self):
|
||||
error = RuntimeError(
|
||||
"Operation failed due to conflicting concurrent operations"
|
||||
)
|
||||
|
||||
assert _is_retryable_write_error(error) is False
|
||||
|
||||
@patch("api.attack_paths.sink.neptune.RetryableSession")
|
||||
def test_writer_session_enables_neptune_retry_policy(self, retryable_session):
|
||||
sink = NeptuneSink()
|
||||
driver = MagicMock()
|
||||
with patch.object(sink, "_get_writer", return_value=driver):
|
||||
with sink.get_session():
|
||||
pass
|
||||
|
||||
kwargs = retryable_session.call_args.kwargs
|
||||
assert kwargs["retry_if"] is _is_retryable_write_error
|
||||
assert (
|
||||
kwargs["initial_retry_delay_seconds"] == NEPTUNE_WRITE_RETRY_DELAY_SECONDS
|
||||
)
|
||||
|
||||
@patch("api.attack_paths.sink.neptune.RetryableSession")
|
||||
def test_reader_session_does_not_enable_write_retry_policy(self, retryable_session):
|
||||
sink = NeptuneSink()
|
||||
driver = MagicMock()
|
||||
with patch.object(sink, "_get_reader", return_value=driver):
|
||||
with sink.get_session(default_access_mode=neo4j.READ_ACCESS):
|
||||
pass
|
||||
|
||||
kwargs = retryable_session.call_args.kwargs
|
||||
assert kwargs["retry_if"] is None
|
||||
assert kwargs["initial_retry_delay_seconds"] == 0
|
||||
|
||||
|
||||
class TestNeptuneSinkDropSubgraph:
|
||||
def test_drop_subgraph_deletes_directed_rels_before_nodes_in_bounded_batches(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
session.run.side_effect = _directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
session, transactions = _managed_write_session(
|
||||
_directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
)
|
||||
)
|
||||
|
||||
with patch.object(sink, "get_session", return_value=_session_ctx(session)):
|
||||
deleted = sink.drop_subgraph("ignored", "provider-1")
|
||||
|
||||
assert deleted == 10
|
||||
assert session.run.call_count == 6
|
||||
queries = [call.args[0] for call in session.run.call_args_list]
|
||||
assert session.execute_write.call_count == 6
|
||||
queries = [transaction.run.call_args.args[0] for transaction in transactions]
|
||||
|
||||
assert ")-[r]->()" in queries[0]
|
||||
assert ")<-[r]-()" in queries[2]
|
||||
@@ -362,14 +416,13 @@ class TestNeo4jSinkDropSubgraph:
|
||||
"""Neo4j drop deletes relationships then nodes in batches (no ``DETACH DELETE``)."""
|
||||
|
||||
def test_drop_subgraph_deletes_directed_rels_before_nodes_in_bounded_batches(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.side_effect = _directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
session, transactions = _managed_write_session(
|
||||
_directed_drop_results(
|
||||
outgoing_rels=50,
|
||||
incoming_rels=30,
|
||||
nodes=10,
|
||||
)
|
||||
)
|
||||
|
||||
provider_id = "00000000-0000-0000-0000-000000000abc"
|
||||
@@ -378,9 +431,9 @@ class TestNeo4jSinkDropSubgraph:
|
||||
|
||||
# Only phase-2 node counts contribute to the return value.
|
||||
assert deleted == 10
|
||||
assert session.run.call_count == 6
|
||||
assert session.execute_write.call_count == 6
|
||||
|
||||
queries = [call.args[0] for call in session.run.call_args_list]
|
||||
queries = [transaction.run.call_args.args[0] for transaction in transactions]
|
||||
# Regression guard: the memory blow-up was caused by DETACH DELETE.
|
||||
assert all("DETACH DELETE" not in query for query in queries)
|
||||
assert all("DISTINCT r" not in query for query in queries)
|
||||
@@ -399,12 +452,9 @@ class TestNeo4jSinkDropSubgraph:
|
||||
assert last_rel < first_node
|
||||
|
||||
def test_drop_subgraph_returns_zero_when_database_does_not_exist(self):
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.attack_paths.sink.neo4j import DATABASE_NOT_FOUND_CODE, Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.side_effect = GraphDatabaseQueryException(
|
||||
session.execute_write.side_effect = GraphDatabaseQueryException(
|
||||
message="db missing", code=DATABASE_NOT_FOUND_CODE
|
||||
)
|
||||
|
||||
@@ -418,8 +468,6 @@ class TestSinkHasProviderData:
|
||||
"""``has_provider_data`` is the read-path probe used by API views."""
|
||||
|
||||
def test_neo4j_returns_true_when_provider_node_exists(self):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.return_value.single.return_value = MagicMock()
|
||||
@@ -433,9 +481,6 @@ class TestSinkHasProviderData:
|
||||
assert ":`_Provider_00000000000000000000000000000abc`" in query
|
||||
|
||||
def test_neo4j_returns_false_when_database_does_not_exist(self):
|
||||
from api.attack_paths.database import GraphDatabaseQueryException
|
||||
from api.attack_paths.sink.neo4j import DATABASE_NOT_FOUND_CODE, Neo4jSink
|
||||
|
||||
sink = Neo4jSink()
|
||||
session = MagicMock()
|
||||
session.run.side_effect = GraphDatabaseQueryException(
|
||||
@@ -448,8 +493,6 @@ class TestSinkHasProviderData:
|
||||
assert present is False
|
||||
|
||||
def test_neptune_returns_true_when_provider_node_exists(self):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
session = MagicMock()
|
||||
session.run.return_value.single.return_value = MagicMock()
|
||||
@@ -463,8 +506,6 @@ class TestGetBackendForScanCutover:
|
||||
"""``get_backend_for_scan`` keeps old-sink scans queryable after cutover."""
|
||||
|
||||
def test_legacy_scan_on_neptune_process_uses_neo4j_secondary(self, settings):
|
||||
from api.attack_paths.sink import factory
|
||||
|
||||
settings.ATTACK_PATHS_SINK_DATABASE = "neptune"
|
||||
active_neptune = MagicMock(name="neptune-active")
|
||||
factory._backend = active_neptune
|
||||
@@ -487,8 +528,6 @@ class TestSinkVerifyConnectivity:
|
||||
|
||||
@patch("api.attack_paths.sink.neo4j.neo4j.GraphDatabase.driver")
|
||||
def test_neo4j_verifies_its_driver(self, mock_driver, settings):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
"neo4j": {
|
||||
@@ -513,8 +552,6 @@ class TestSinkVerifyConnectivity:
|
||||
def test_neptune_verifies_reader_not_writer(
|
||||
self, mock_driver, mock_auth_provider, settings
|
||||
):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
"neptune": {
|
||||
@@ -548,8 +585,6 @@ class TestSinkInitToleratesUnreachableSink:
|
||||
|
||||
@patch("api.attack_paths.sink.neo4j.neo4j.GraphDatabase.driver")
|
||||
def test_neo4j_init_continues_when_verify_fails(self, mock_driver, settings):
|
||||
from api.attack_paths.sink.neo4j import Neo4jSink
|
||||
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
"neo4j": {
|
||||
@@ -573,8 +608,6 @@ class TestSinkInitToleratesUnreachableSink:
|
||||
def test_neptune_init_continues_when_verify_fails(
|
||||
self, mock_driver, mock_auth_provider, settings
|
||||
):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
settings.DATABASES = {
|
||||
**settings.DATABASES,
|
||||
"neptune": {
|
||||
@@ -601,8 +634,6 @@ class TestNeptuneAdminNoOps:
|
||||
|
||||
@pytest.mark.parametrize("method", ["create_database", "drop_database"])
|
||||
def test_admin_ops_return_none_without_touching_a_session(self, method):
|
||||
from api.attack_paths.sink.neptune import NeptuneSink
|
||||
|
||||
sink = NeptuneSink()
|
||||
with patch.object(sink, "get_session") as get_session:
|
||||
assert getattr(sink, method)("ignored") is None
|
||||
@@ -617,8 +648,6 @@ class TestNeptuneAuthToken:
|
||||
def test_host_header_includes_non_default_port(self, mock_boto, mock_sigv4):
|
||||
# Neptune runs on 8182; the SigV4 canonical Host must keep the port or
|
||||
# the signature is rejected.
|
||||
from api.attack_paths.sink.neptune import _NeptuneAuthToken
|
||||
|
||||
credentials = MagicMock()
|
||||
credentials.get_frozen_credentials.return_value = MagicMock()
|
||||
mock_boto.return_value.get_credentials.return_value = credentials
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
from api.validators import (
|
||||
resolve_lighthouse_openai_compatible_host,
|
||||
validate_lighthouse_openai_compatible_base_url,
|
||||
)
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.test import override_settings
|
||||
|
||||
|
||||
def test_lighthouse_base_url_rejects_http_scheme():
|
||||
with pytest.raises(ValidationError, match="HTTPS"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"http://openrouter.ai/api/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://openrouter.ai:0/api/v1",
|
||||
"https://openrouter.ai:-1/api/v1",
|
||||
"https://openrouter.ai:65536/api/v1",
|
||||
"https://openrouter.ai:invalid/api/v1",
|
||||
],
|
||||
)
|
||||
def test_lighthouse_base_url_rejects_invalid_port(base_url):
|
||||
with pytest.raises(ValidationError, match="port is invalid"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
base_url,
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("port", [1, 65535])
|
||||
def test_lighthouse_base_url_accepts_valid_port_boundaries(port):
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
f"https://openrouter.ai:{port}/api/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_rejects_localhost():
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://localhost/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ip_address", ["10.0.0.1", "172.16.0.1", "192.168.1.1"])
|
||||
def test_lighthouse_base_url_rejects_private_ip_literal(ip_address):
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
f"https://{ip_address}/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_rejects_metadata_ip_literal():
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://169.254.169.254/latest/meta-data",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://[::ffff:169.254.169.254]/v1",
|
||||
"https://[64:ff9b::a9fe:a9fe]/v1",
|
||||
"https://[2002:a9fe:a9fe::]/v1",
|
||||
],
|
||||
)
|
||||
def test_lighthouse_base_url_rejects_embedded_non_global_ip(base_url):
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
base_url,
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://[::ffff:93.184.216.34]/v1",
|
||||
"https://[64:ff9b::5db8:d822]/v1",
|
||||
"https://[2002:5db8:d822::]/v1",
|
||||
],
|
||||
)
|
||||
def test_lighthouse_base_url_accepts_embedded_public_ip(base_url):
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
base_url,
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_accepts_hostname_without_dns_resolution():
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://openrouter.ai/api/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_rejects_post_dns_internal_address(monkeypatch):
|
||||
def resolve_to_metadata(*_args, **_kwargs):
|
||||
return [
|
||||
(
|
||||
socket.AF_INET,
|
||||
socket.SOCK_STREAM,
|
||||
6,
|
||||
"",
|
||||
("169.254.169.254", 443),
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr("api.validators.socket.getaddrinfo", resolve_to_metadata)
|
||||
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://metadata.example.test/v1"
|
||||
)
|
||||
|
||||
|
||||
def test_lighthouse_base_url_accepts_public_resolved_address(monkeypatch):
|
||||
def resolve_to_public(*_args, **_kwargs):
|
||||
return [
|
||||
(
|
||||
socket.AF_INET,
|
||||
socket.SOCK_STREAM,
|
||||
6,
|
||||
"",
|
||||
("93.184.216.34", 443),
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr("api.validators.socket.getaddrinfo", resolve_to_public)
|
||||
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url("https://openrouter.ai/api/v1")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"]
|
||||
)
|
||||
def test_lighthouse_base_url_accepts_allowlisted_host_without_resolution(monkeypatch):
|
||||
def fail_resolution(*_args, **_kwargs):
|
||||
raise AssertionError("allowlisted hosts must not be resolved")
|
||||
|
||||
monkeypatch.setattr("api.validators.socket.getaddrinfo", fail_resolution)
|
||||
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://custom-openai.internal/v1"
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"]
|
||||
)
|
||||
def test_lighthouse_resolve_returns_allowlisted_hostname_unpinned():
|
||||
assert resolve_lighthouse_openai_compatible_host(
|
||||
"Custom-OpenAI.internal.", 443
|
||||
) == ("custom-openai.internal",)
|
||||
|
||||
|
||||
@override_settings(LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["localhost"])
|
||||
def test_lighthouse_base_url_accepts_allowlisted_blocked_host():
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://localhost/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["10.0.0.1"])
|
||||
def test_lighthouse_base_url_accepts_allowlisted_private_ip_literal():
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://10.0.0.1/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=[" Custom-OpenAI.Internal. "]
|
||||
)
|
||||
def test_lighthouse_allowlist_entries_are_normalized():
|
||||
assert (
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://custom-openai.internal/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"]
|
||||
)
|
||||
def test_lighthouse_base_url_rejects_host_not_in_allowlist():
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://localhost/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@override_settings(LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=[""])
|
||||
def test_lighthouse_allowlist_ignores_empty_entries():
|
||||
with pytest.raises(ValidationError, match="external public endpoint"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"https://localhost/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=["custom-openai.internal"]
|
||||
)
|
||||
def test_lighthouse_base_url_allowlisted_host_still_requires_https():
|
||||
with pytest.raises(ValidationError, match="HTTPS"):
|
||||
validate_lighthouse_openai_compatible_base_url(
|
||||
"http://custom-openai.internal/v1",
|
||||
resolve_dns=False,
|
||||
)
|
||||
@@ -17130,6 +17130,76 @@ class TestLighthouseProviderConfigViewSet:
|
||||
error_detail = str(resp.json()).lower()
|
||||
assert "base_url" in error_detail
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://127.0.0.1/v1",
|
||||
"https://169.254.169.254/latest/meta-data",
|
||||
],
|
||||
)
|
||||
def test_openai_compatible_rejects_internal_base_url_on_create(
|
||||
self, authenticated_client, base_url
|
||||
):
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai_compatible",
|
||||
"base_url": base_url,
|
||||
"credentials": {"api_key": "compat-key"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert resp.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "base_url" in str(resp.json()).lower()
|
||||
|
||||
def test_openai_compatible_rejects_internal_base_url_on_update(
|
||||
self, authenticated_client
|
||||
):
|
||||
create_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai_compatible",
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
"credentials": {"api_key": "compat-key-123"},
|
||||
},
|
||||
}
|
||||
}
|
||||
create_resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=create_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert create_resp.status_code == status.HTTP_201_CREATED
|
||||
provider_id = create_resp.json()["data"]["id"]
|
||||
|
||||
patch_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"id": provider_id,
|
||||
"attributes": {
|
||||
"base_url": "https://169.254.169.254/latest/meta-data",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
patch_resp = authenticated_client.patch(
|
||||
reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}),
|
||||
data=patch_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
assert patch_resp.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "base_url" in str(patch_resp.json()).lower()
|
||||
|
||||
def test_openai_compatible_invalid_credentials(self, authenticated_client):
|
||||
payload = {
|
||||
"data": {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
from api.db_router import MainRouter
|
||||
from rest_framework_simplejwt.token_blacklist.models import (
|
||||
BlacklistedToken,
|
||||
OutstandingToken,
|
||||
)
|
||||
|
||||
|
||||
def blacklist_user_refresh_tokens(user_id):
|
||||
outstanding_token_ids = list(
|
||||
OutstandingToken.objects.using(MainRouter.admin_db)
|
||||
.filter(user_id=user_id)
|
||||
.values_list("id", flat=True)
|
||||
)
|
||||
if outstanding_token_ids:
|
||||
BlacklistedToken.objects.using(MainRouter.admin_db).bulk_create(
|
||||
[BlacklistedToken(token_id=token_id) for token_id in outstanding_token_ids],
|
||||
ignore_conflicts=True,
|
||||
)
|
||||
@@ -38,6 +38,7 @@ from api.models import (
|
||||
UserRoleRelationship,
|
||||
)
|
||||
from api.rls import Tenant
|
||||
from api.v1.serializer_utils.authentication import blacklist_user_refresh_tokens
|
||||
from api.v1.serializer_utils.integrations import (
|
||||
AWSCredentialSerializer,
|
||||
IntegrationConfigField,
|
||||
@@ -56,12 +57,13 @@ from api.v1.serializer_utils.lighthouse import (
|
||||
)
|
||||
from api.v1.serializer_utils.processors import ProcessorConfigField
|
||||
from api.v1.serializer_utils.providers import ProviderSecretField
|
||||
from api.validators import validate_lighthouse_openai_compatible_base_url
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import authenticate
|
||||
from django.contrib.auth.models import update_last_login
|
||||
from django.contrib.auth.password_validation import validate_password
|
||||
from django.core.exceptions import ValidationError as DjangoValidationError
|
||||
from django.db import IntegrityError
|
||||
from django.db import IntegrityError, transaction
|
||||
from drf_spectacular.utils import extend_schema_field
|
||||
from jwt.exceptions import InvalidKeyError
|
||||
from prowler.lib.mutelist.mutelist import Mutelist
|
||||
@@ -72,11 +74,28 @@ from rest_framework_json_api.relations import SerializerMethodResourceRelatedFie
|
||||
from rest_framework_json_api.serializers import ValidationError
|
||||
from rest_framework_simplejwt.exceptions import TokenError
|
||||
from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
|
||||
from rest_framework_simplejwt.settings import api_settings
|
||||
from rest_framework_simplejwt.tokens import RefreshToken
|
||||
from rest_framework_simplejwt.utils import get_md5_hash_password
|
||||
|
||||
# Base
|
||||
|
||||
|
||||
def _validate_lighthouse_base_url_without_dns(base_url: str) -> None:
|
||||
try:
|
||||
validate_lighthouse_openai_compatible_base_url(base_url, resolve_dns=False)
|
||||
except DjangoValidationError as error:
|
||||
raise ValidationError({"base_url": error.messages[0]}) from error
|
||||
|
||||
|
||||
def _reraise_lighthouse_credentials_errors(error: ValidationError) -> None:
|
||||
details = error.detail.copy()
|
||||
for key, value in details.items():
|
||||
error.detail[f"credentials/{key}"] = value
|
||||
del error.detail[key]
|
||||
raise error
|
||||
|
||||
|
||||
class BaseModelSerializerV1(serializers.ModelSerializer):
|
||||
def get_root_meta(self, _resource, _many):
|
||||
return {"version": "v1"}
|
||||
@@ -232,6 +251,18 @@ class TokenRefreshSerializer(BaseSerializerV1):
|
||||
try:
|
||||
# Validate the refresh token
|
||||
refresh = RefreshToken(refresh_token)
|
||||
if api_settings.CHECK_REVOKE_TOKEN:
|
||||
user_id = refresh.payload.get(api_settings.USER_ID_CLAIM)
|
||||
try:
|
||||
user = User.objects.using(MainRouter.admin_db).get(
|
||||
**{api_settings.USER_ID_FIELD: user_id}
|
||||
)
|
||||
except User.DoesNotExist:
|
||||
raise TokenError("User not found.") from None
|
||||
if refresh.get(api_settings.REVOKE_TOKEN_CLAIM) != (
|
||||
get_md5_hash_password(user.password)
|
||||
):
|
||||
raise TokenError("The user's password has been changed.")
|
||||
# Generate new access token
|
||||
access_token = refresh.access_token
|
||||
|
||||
@@ -405,7 +436,13 @@ class UserUpdateSerializer(BaseWriteSerializer):
|
||||
password = validated_data.pop("password", None)
|
||||
if password:
|
||||
validate_password(password, user=instance)
|
||||
instance.set_password(password)
|
||||
with transaction.atomic(using=MainRouter.admin_db):
|
||||
instance.set_password(password)
|
||||
for attr, value in validated_data.items():
|
||||
setattr(instance, attr, value)
|
||||
blacklist_user_refresh_tokens(instance.id)
|
||||
instance.save(using=MainRouter.admin_db)
|
||||
return instance
|
||||
return super().update(instance, validated_data)
|
||||
|
||||
|
||||
@@ -3624,11 +3661,7 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
elif (
|
||||
provider_type == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
|
||||
):
|
||||
@@ -3637,27 +3670,20 @@ class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerialize
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
elif (
|
||||
provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE
|
||||
):
|
||||
if not base_url:
|
||||
raise ValidationError({"base_url": "Base URL is required."})
|
||||
_validate_lighthouse_base_url_without_dns(base_url)
|
||||
try:
|
||||
OpenAICompatibleCredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
|
||||
return super().validate(attrs)
|
||||
|
||||
@@ -3720,11 +3746,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
elif (
|
||||
credentials is not None
|
||||
and provider_type
|
||||
@@ -3748,11 +3770,7 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
|
||||
# Then enforce invariants about not changing the auth method
|
||||
# If the existing config uses an API key, forbid introducing access keys.
|
||||
@@ -3779,24 +3797,23 @@ class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
|
||||
}
|
||||
)
|
||||
elif (
|
||||
credentials is not None
|
||||
and provider_type
|
||||
provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE
|
||||
):
|
||||
if base_url is None:
|
||||
pass
|
||||
elif not base_url:
|
||||
effective_base_url = (
|
||||
base_url if "base_url" in attrs else getattr(self.instance, "base_url")
|
||||
)
|
||||
if not effective_base_url:
|
||||
raise ValidationError({"base_url": "Base URL cannot be empty."})
|
||||
try:
|
||||
OpenAICompatibleCredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
if "base_url" in attrs:
|
||||
_validate_lighthouse_base_url_without_dns(effective_base_url)
|
||||
if credentials is not None:
|
||||
try:
|
||||
OpenAICompatibleCredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
_reraise_lighthouse_credentials_errors(e)
|
||||
|
||||
return super().validate(attrs)
|
||||
|
||||
|
||||
@@ -1,14 +1,155 @@
|
||||
import ipaddress
|
||||
import socket
|
||||
import string
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.utils.translation import gettext as _
|
||||
|
||||
LIGHTHOUSE_OPENAI_COMPATIBLE_ALLOWED_SCHEMES = frozenset({"https"})
|
||||
LIGHTHOUSE_NAT64_WELL_KNOWN_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
|
||||
LIGHTHOUSE_BLOCKED_METADATA_HOSTS = frozenset(
|
||||
{
|
||||
"169.254.169.254",
|
||||
"169.254.170.2",
|
||||
"fd00:ec2::254",
|
||||
"localhost",
|
||||
"metadata.google.internal",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_hostname(hostname: str) -> str:
|
||||
return hostname.rstrip(".").lower()
|
||||
|
||||
|
||||
def _lighthouse_openai_compatible_allowed_hosts() -> frozenset[str]:
|
||||
return frozenset(
|
||||
_normalize_hostname(allowed_host.strip())
|
||||
for allowed_host in settings.LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS
|
||||
if allowed_host and allowed_host.strip()
|
||||
)
|
||||
|
||||
|
||||
def _validate_lighthouse_public_ip(address: str) -> None:
|
||||
ip_address = ipaddress.ip_address(address)
|
||||
if isinstance(ip_address, ipaddress.IPv6Address):
|
||||
# Classify transition addresses by their effective IPv4 destination.
|
||||
embedded_ip_address = ip_address.ipv4_mapped or ip_address.sixtofour
|
||||
if (
|
||||
embedded_ip_address is None
|
||||
and ip_address in LIGHTHOUSE_NAT64_WELL_KNOWN_PREFIX
|
||||
):
|
||||
embedded_ip_address = ipaddress.IPv4Address(int(ip_address) & 0xFFFFFFFF)
|
||||
if embedded_ip_address is not None:
|
||||
ip_address = embedded_ip_address
|
||||
if not ip_address.is_global:
|
||||
raise ValidationError(
|
||||
_("Base URL must use an external public endpoint."),
|
||||
code="lighthouse_base_url_not_public",
|
||||
)
|
||||
|
||||
|
||||
def resolve_lighthouse_openai_compatible_host(
|
||||
hostname: str,
|
||||
port: int,
|
||||
*,
|
||||
resolve_dns: bool = True,
|
||||
) -> tuple[str, ...]:
|
||||
"""Return public IP addresses that are safe for Lighthouse outbound use."""
|
||||
hostname = _normalize_hostname(hostname)
|
||||
if hostname in _lighthouse_openai_compatible_allowed_hosts():
|
||||
# Operator-allowlisted hosts skip the public-endpoint checks; returning
|
||||
# the hostname makes the network backend connect through regular DNS
|
||||
# resolution instead of pinned addresses.
|
||||
return (hostname,)
|
||||
|
||||
if hostname in LIGHTHOUSE_BLOCKED_METADATA_HOSTS or hostname.endswith(".localhost"):
|
||||
raise ValidationError(
|
||||
_("Base URL must use an external public endpoint."),
|
||||
code="lighthouse_base_url_blocked_host",
|
||||
)
|
||||
|
||||
try:
|
||||
_validate_lighthouse_public_ip(hostname)
|
||||
except ValueError:
|
||||
if not resolve_dns:
|
||||
return ()
|
||||
else:
|
||||
return (hostname,)
|
||||
|
||||
try:
|
||||
resolved_addresses = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM)
|
||||
except socket.gaierror as error:
|
||||
raise ValidationError(
|
||||
_("Base URL host could not be resolved."),
|
||||
code="lighthouse_base_url_resolution_failed",
|
||||
) from error
|
||||
|
||||
if not resolved_addresses:
|
||||
raise ValidationError(
|
||||
_("Base URL host could not be resolved."),
|
||||
code="lighthouse_base_url_resolution_failed",
|
||||
)
|
||||
|
||||
public_addresses: list[str] = []
|
||||
for resolved_address in resolved_addresses:
|
||||
socket_address = resolved_address[4]
|
||||
resolved_ip_address = socket_address[0]
|
||||
_validate_lighthouse_public_ip(resolved_ip_address)
|
||||
if resolved_ip_address not in public_addresses:
|
||||
public_addresses.append(resolved_ip_address)
|
||||
|
||||
return tuple(public_addresses)
|
||||
|
||||
|
||||
def validate_lighthouse_openai_compatible_base_url(
|
||||
base_url: str,
|
||||
*,
|
||||
resolve_dns: bool = True,
|
||||
) -> None:
|
||||
"""Validate an OpenAI-compatible Lighthouse base URL before outbound use."""
|
||||
parsed = urlparse(str(base_url))
|
||||
if parsed.scheme.lower() not in LIGHTHOUSE_OPENAI_COMPATIBLE_ALLOWED_SCHEMES:
|
||||
raise ValidationError(
|
||||
_("Base URL must use HTTPS."),
|
||||
code="lighthouse_base_url_invalid_scheme",
|
||||
)
|
||||
|
||||
if not parsed.hostname:
|
||||
raise ValidationError(
|
||||
_("Base URL must include a host."),
|
||||
code="lighthouse_base_url_missing_host",
|
||||
)
|
||||
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError as error:
|
||||
raise ValidationError(
|
||||
_("Base URL port is invalid."),
|
||||
code="lighthouse_base_url_invalid_port",
|
||||
) from error
|
||||
|
||||
if port is not None and not 1 <= port <= 65535:
|
||||
raise ValidationError(
|
||||
_("Base URL port is invalid."),
|
||||
code="lighthouse_base_url_invalid_port",
|
||||
)
|
||||
|
||||
resolve_lighthouse_openai_compatible_host(
|
||||
parsed.hostname,
|
||||
port or 443,
|
||||
resolve_dns=resolve_dns,
|
||||
)
|
||||
|
||||
|
||||
class MaximumLengthValidator:
|
||||
def __init__(self, max_length=72):
|
||||
self.max_length = max_length
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if len(password) > self.max_length:
|
||||
raise ValidationError(
|
||||
_(
|
||||
@@ -31,6 +172,7 @@ class SpecialCharactersValidator:
|
||||
self.min_special_characters = min_special_characters
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if (
|
||||
sum(1 for char in password if char in self.special_characters)
|
||||
< self.min_special_characters
|
||||
@@ -55,6 +197,7 @@ class UppercaseValidator:
|
||||
self.min_uppercase = min_uppercase
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if sum(1 for char in password if char.isupper()) < self.min_uppercase:
|
||||
raise ValidationError(
|
||||
_(
|
||||
@@ -75,6 +218,7 @@ class LowercaseValidator:
|
||||
self.min_lowercase = min_lowercase
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if sum(1 for char in password if char.islower()) < self.min_lowercase:
|
||||
raise ValidationError(
|
||||
_(
|
||||
@@ -95,6 +239,7 @@ class NumericValidator:
|
||||
self.min_numeric = min_numeric
|
||||
|
||||
def validate(self, password, user=None):
|
||||
del user
|
||||
if sum(1 for char in password if char.isdigit()) < self.min_numeric:
|
||||
raise ValidationError(
|
||||
_(
|
||||
|
||||
@@ -230,6 +230,7 @@ SIMPLE_JWT = {
|
||||
"JTI_CLAIM": "jti",
|
||||
"USER_ID_FIELD": "id",
|
||||
"USER_ID_CLAIM": "sub",
|
||||
"CHECK_REVOKE_TOKEN": True,
|
||||
# Issuer and Audience claims, for the moment we will keep these values as default values, they may change in the
|
||||
# future.
|
||||
"AUDIENCE": env.str("DJANGO_JWT_AUDIENCE", "https://api.prowler.com"),
|
||||
@@ -316,6 +317,15 @@ ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
|
||||
# Valid values: "neo4j" (default, OSS and local dev), "neptune" (hosted).
|
||||
ATTACK_PATHS_SINK_DATABASE = env.str("ATTACK_PATHS_SINK_DATABASE", default="neo4j")
|
||||
|
||||
# Lighthouse AI
|
||||
# Comma-separated hostnames (or IP literals) that bypass the SSRF validation
|
||||
# applied to OpenAI-compatible provider base URLs, so self-hosted deployments
|
||||
# can point Lighthouse AI at internal endpoints. Empty by default: every base
|
||||
# URL must resolve to a public endpoint.
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS = env.list(
|
||||
"LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS", default=[]
|
||||
)
|
||||
|
||||
# Orphan task recovery feature flags. The master switch is OFF by default, so task
|
||||
# recovery is opt-in; enable it with DJANGO_TASK_RECOVERY_ENABLED=true. The per-group
|
||||
# toggles default to enabled, so once the master is on every group recovers unless a
|
||||
|
||||
@@ -19,6 +19,7 @@ import json
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
import neo4j
|
||||
@@ -392,11 +393,11 @@ def _build_child_props(
|
||||
def _build_child_id(provider_id: str, child_label: str, value_key: str) -> str:
|
||||
"""Deterministic `_provider_element_id` for a list-item child node.
|
||||
|
||||
Dedupes within (tenant, provider): multiple parents referencing the same
|
||||
value share one child node via the existing MERGE-on-_provider_element_id
|
||||
index in both sinks.
|
||||
Hashing the value keeps the ID bounded while preserving deduplication within
|
||||
each provider and child label.
|
||||
"""
|
||||
return f"{provider_id}::{child_label}::{value_key}"
|
||||
value_digest = sha256(value_key.encode("utf-8")).hexdigest()
|
||||
return f"{provider_id}::{child_label}::{value_digest}"
|
||||
|
||||
|
||||
def _build_catalog_index(
|
||||
|
||||
@@ -14,6 +14,7 @@ from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
|
||||
from prowler.lib.outputs.csv.csv import CSV
|
||||
from prowler.lib.outputs.finding import Finding as FindingOutput
|
||||
from prowler.lib.outputs.html.html import HTML
|
||||
from prowler.lib.outputs.jira.exceptions.exceptions import JiraBaseException
|
||||
from prowler.lib.outputs.ocsf.ocsf import OCSF
|
||||
from prowler.providers.aws.aws_provider import AwsProvider
|
||||
from prowler.providers.aws.lib.s3.s3 import S3
|
||||
@@ -26,6 +27,8 @@ from tasks.utils import batched
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
JIRA_GENERIC_SEND_ERROR = "Failed to create Jira issue."
|
||||
|
||||
|
||||
def get_s3_client_from_integration(
|
||||
integration: Integration,
|
||||
@@ -483,6 +486,7 @@ def send_findings_to_jira(
|
||||
jira_integration = initialize_prowler_integration(integration)
|
||||
|
||||
num_tickets_created = 0
|
||||
error_messages = []
|
||||
for finding_id in finding_ids:
|
||||
with rls_transaction(tenant_id):
|
||||
finding_instance = (
|
||||
@@ -512,35 +516,54 @@ def send_findings_to_jira(
|
||||
recommendation = remediation.get("recommendation", {})
|
||||
remediation_code = remediation.get("code", {})
|
||||
|
||||
# Send the individual finding to Jira
|
||||
result = jira_integration.send_finding(
|
||||
check_id=finding_instance.check_id,
|
||||
check_title=check_metadata.get("checktitle", ""),
|
||||
severity=finding_instance.severity,
|
||||
status=finding_instance.status,
|
||||
status_extended=finding_instance.status_extended or "",
|
||||
provider=finding_instance.scan.provider.provider,
|
||||
region=region,
|
||||
resource_uid=resource_uid,
|
||||
resource_name=resource_name,
|
||||
risk=check_metadata.get("risk", ""),
|
||||
recommendation_text=recommendation.get("text", ""),
|
||||
recommendation_url=recommendation.get("url", ""),
|
||||
remediation_code_native_iac=remediation_code.get("nativeiac", ""),
|
||||
remediation_code_terraform=remediation_code.get("terraform", ""),
|
||||
remediation_code_cli=remediation_code.get("cli", ""),
|
||||
remediation_code_other=remediation_code.get("other", ""),
|
||||
resource_tags=resource_tags,
|
||||
compliance=finding_instance.compliance or {},
|
||||
project_key=project_key,
|
||||
issue_type=issue_type,
|
||||
)
|
||||
try:
|
||||
# Send the individual finding to Jira
|
||||
result = jira_integration.send_finding(
|
||||
check_id=finding_instance.check_id,
|
||||
check_title=check_metadata.get("checktitle", ""),
|
||||
severity=finding_instance.severity,
|
||||
status=finding_instance.status,
|
||||
status_extended=finding_instance.status_extended or "",
|
||||
provider=finding_instance.scan.provider.provider,
|
||||
region=region,
|
||||
resource_uid=resource_uid,
|
||||
resource_name=resource_name,
|
||||
risk=check_metadata.get("risk", ""),
|
||||
recommendation_text=recommendation.get("text", ""),
|
||||
recommendation_url=recommendation.get("url", ""),
|
||||
remediation_code_native_iac=remediation_code.get("nativeiac", ""),
|
||||
remediation_code_terraform=remediation_code.get("terraform", ""),
|
||||
remediation_code_cli=remediation_code.get("cli", ""),
|
||||
remediation_code_other=remediation_code.get("other", ""),
|
||||
resource_tags=resource_tags,
|
||||
compliance=finding_instance.compliance or {},
|
||||
project_key=project_key,
|
||||
issue_type=issue_type,
|
||||
)
|
||||
except JiraBaseException as error:
|
||||
error_message = error.message or JIRA_GENERIC_SEND_ERROR
|
||||
logger.exception(
|
||||
"Failed to send finding %s to Jira: %s", finding_id, error_message
|
||||
)
|
||||
error_messages.append(error_message)
|
||||
continue
|
||||
except Exception:
|
||||
logger.exception("Failed to send finding %s to Jira", finding_id)
|
||||
error_messages.append(JIRA_GENERIC_SEND_ERROR)
|
||||
continue
|
||||
|
||||
if result:
|
||||
num_tickets_created += 1
|
||||
else:
|
||||
logger.error(f"Failed to send finding {finding_id} to Jira")
|
||||
error_message = JIRA_GENERIC_SEND_ERROR
|
||||
logger.error(error_message)
|
||||
error_messages.append(error_message)
|
||||
|
||||
return {
|
||||
result = {
|
||||
"created_count": num_tickets_created,
|
||||
"failed_count": len(finding_ids) - num_tickets_created,
|
||||
}
|
||||
if error_messages:
|
||||
result["error"] = "; ".join(dict.fromkeys(error_messages))
|
||||
|
||||
return result
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import ssl
|
||||
from collections.abc import Iterable
|
||||
|
||||
import boto3
|
||||
import httpcore
|
||||
import httpx
|
||||
import openai
|
||||
from api.models import LighthouseProviderConfiguration, LighthouseProviderModels
|
||||
from api.validators import (
|
||||
resolve_lighthouse_openai_compatible_host,
|
||||
validate_lighthouse_openai_compatible_base_url,
|
||||
)
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
@@ -43,6 +52,90 @@ EXCLUDED_OPENAI_MODEL_SUBSTRINGS = (
|
||||
"-instruct", # Legacy instruct models (gpt-3.5-turbo-instruct, etc.)
|
||||
)
|
||||
|
||||
OPENAI_COMPATIBLE_AUTHENTICATION_ERROR = "API key is invalid or missing"
|
||||
OPENAI_COMPATIBLE_CONNECTION_ERROR = "Provider connection failed"
|
||||
|
||||
|
||||
class _OpenAICompatibleProviderError(Exception):
|
||||
"""Sanitized OpenAI-compatible provider error safe for task results."""
|
||||
|
||||
|
||||
def _sanitize_openai_compatible_error(error: Exception) -> str:
|
||||
status_code = getattr(error, "status_code", None)
|
||||
if status_code is None:
|
||||
response = getattr(error, "response", None)
|
||||
status_code = getattr(response, "status_code", None)
|
||||
|
||||
if status_code == 401:
|
||||
return OPENAI_COMPATIBLE_AUTHENTICATION_ERROR
|
||||
return OPENAI_COMPATIBLE_CONNECTION_ERROR
|
||||
|
||||
|
||||
class _LighthouseOpenAICompatibleNetworkBackend(httpcore.SyncBackend):
|
||||
"""Validate and pin DNS results immediately before TCP connections."""
|
||||
|
||||
def connect_tcp(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
timeout: float | None = None,
|
||||
local_address: str | None = None,
|
||||
socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None,
|
||||
) -> httpcore.NetworkStream:
|
||||
resolved_addresses = resolve_lighthouse_openai_compatible_host(host, port)
|
||||
last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None
|
||||
|
||||
for address in resolved_addresses:
|
||||
try:
|
||||
return super().connect_tcp(
|
||||
address,
|
||||
port,
|
||||
timeout=timeout,
|
||||
local_address=local_address,
|
||||
socket_options=socket_options,
|
||||
)
|
||||
except (httpcore.ConnectError, httpcore.ConnectTimeout) as error:
|
||||
last_error = error
|
||||
|
||||
if last_error:
|
||||
raise last_error
|
||||
raise httpcore.ConnectError("No resolved addresses are available")
|
||||
|
||||
|
||||
class _LighthouseOpenAICompatibleHTTPTransport(httpx.HTTPTransport):
|
||||
"""HTTP transport that connects only to validated public IP addresses."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._pool = httpcore.ConnectionPool(
|
||||
ssl_context=ssl.create_default_context(),
|
||||
network_backend=_LighthouseOpenAICompatibleNetworkBackend(),
|
||||
)
|
||||
|
||||
|
||||
def _create_openai_compatible_http_client() -> httpx.Client:
|
||||
"""Create the restricted HTTP client used for OpenAI-compatible providers."""
|
||||
return httpx.Client(
|
||||
follow_redirects=False,
|
||||
trust_env=False,
|
||||
transport=_LighthouseOpenAICompatibleHTTPTransport(),
|
||||
)
|
||||
|
||||
|
||||
def _list_openai_compatible_models(base_url: str, api_key: str):
|
||||
validate_lighthouse_openai_compatible_base_url(base_url)
|
||||
try:
|
||||
with _create_openai_compatible_http_client() as http_client:
|
||||
client = openai.OpenAI(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
http_client=http_client,
|
||||
)
|
||||
return client.models.list()
|
||||
except Exception as error:
|
||||
raise _OpenAICompatibleProviderError(
|
||||
_sanitize_openai_compatible_error(error)
|
||||
) from error
|
||||
|
||||
|
||||
def _extract_error_message(e: Exception) -> str:
|
||||
"""
|
||||
@@ -114,6 +207,7 @@ def _extract_openai_compatible_params(
|
||||
return None
|
||||
if not isinstance(base_url, str) or not base_url:
|
||||
return None
|
||||
validate_lighthouse_openai_compatible_base_url(base_url, resolve_dns=False)
|
||||
return {"base_url": base_url, "api_key": api_key}
|
||||
|
||||
|
||||
@@ -285,13 +379,7 @@ def check_lighthouse_provider_connection(provider_config_id: str) -> dict:
|
||||
"error": "Base URL or API key is invalid or missing",
|
||||
}
|
||||
|
||||
# Test connection using OpenAI SDK with custom base_url
|
||||
# Note: base_url should include version (e.g., https://openrouter.ai/api/v1)
|
||||
client = openai.OpenAI(
|
||||
api_key=params["api_key"],
|
||||
base_url=params["base_url"],
|
||||
)
|
||||
_ = client.models.list()
|
||||
_ = _list_openai_compatible_models(params["base_url"], params["api_key"])
|
||||
|
||||
else:
|
||||
return {"connected": False, "error": "Unsupported provider type"}
|
||||
@@ -361,8 +449,7 @@ def _fetch_openai_compatible_models(base_url: str, api_key: str) -> dict[str, st
|
||||
|
||||
Note: base_url should include version (e.g., https://openrouter.ai/api/v1)
|
||||
"""
|
||||
client = openai.OpenAI(api_key=api_key, base_url=base_url)
|
||||
models = client.models.list()
|
||||
models = _list_openai_compatible_models(base_url, api_key)
|
||||
|
||||
available_models: dict[str, str] = {}
|
||||
for model in models.data:
|
||||
|
||||
@@ -1464,7 +1464,7 @@ def aggregate_findings(tenant_id: str, scan_id: str):
|
||||
)
|
||||
|
||||
with rls_transaction(tenant_id):
|
||||
scan_aggregations = {
|
||||
scan_aggregations = [
|
||||
ScanSummary(
|
||||
tenant_id=tenant_id,
|
||||
scan_id=scan_id,
|
||||
@@ -1489,9 +1489,18 @@ def aggregate_findings(tenant_id: str, scan_id: str):
|
||||
for agg in aggregation
|
||||
if agg["resources__service"] is not None
|
||||
and agg["resources__region"] is not None
|
||||
}
|
||||
# Upsert so re-runs (post-mute reaggregation) don't trip
|
||||
# `unique_scan_summary`; race-safe under concurrent writers.
|
||||
]
|
||||
# Needed sort so concurrent upserts acquire locks consistently
|
||||
scan_aggregations.sort(
|
||||
key=lambda summary: (
|
||||
summary.tenant_id,
|
||||
summary.scan_id,
|
||||
summary.check_id,
|
||||
summary.service,
|
||||
summary.severity,
|
||||
summary.region,
|
||||
)
|
||||
)
|
||||
ScanSummary.objects.bulk_create(
|
||||
scan_aggregations,
|
||||
batch_size=3000,
|
||||
|
||||
@@ -1902,6 +1902,55 @@ def _make_session_ctx(session, call_order=None, name=None):
|
||||
return ctx
|
||||
|
||||
|
||||
class TestBuildChildId:
|
||||
def test_large_value_is_hashed_and_preserved_as_child_data(self):
|
||||
value = "x" * 22_796
|
||||
spec = sync_module.NormalizedList(
|
||||
"SomeLabel",
|
||||
"values",
|
||||
"SomeLabelValuesItem",
|
||||
"HAS_VALUES",
|
||||
)
|
||||
record = {
|
||||
"element_id": "elem-1",
|
||||
"labels": ["SomeLabel"],
|
||||
"props": {"values": [value]},
|
||||
}
|
||||
|
||||
_, parent, children, relationships = sync_module._node_to_sync_dict(
|
||||
record,
|
||||
"prov-1",
|
||||
sync_module._build_catalog_index([spec]),
|
||||
)
|
||||
|
||||
child = children[0]["row"]
|
||||
child_id = child["provider_element_id"]
|
||||
prefix = "prov-1::SomeLabelValuesItem::"
|
||||
assert parent["provider_element_id"] == "prov-1:elem-1"
|
||||
assert child["props"]["value"] == value
|
||||
assert len(child_id) == len(prefix) + 64
|
||||
assert value not in child_id
|
||||
assert relationships[0]["row"]["end_element_id"] == child_id
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider_id", "child_label", "value_key"),
|
||||
[
|
||||
("prov-2", "ChildLabel", "value"),
|
||||
("prov-1", "OtherChildLabel", "value"),
|
||||
("prov-1", "ChildLabel", "other-value"),
|
||||
],
|
||||
)
|
||||
def test_each_identity_component_changes_id(
|
||||
self, provider_id, child_label, value_key
|
||||
):
|
||||
child_id = sync_module._build_child_id("prov-1", "ChildLabel", "value")
|
||||
|
||||
assert sync_module._build_child_id("prov-1", "ChildLabel", "value") == child_id
|
||||
assert (
|
||||
sync_module._build_child_id(provider_id, child_label, value_key) != child_id
|
||||
)
|
||||
|
||||
|
||||
class TestSyncNodes:
|
||||
def test_iter_sink_batches_rejects_zero_batch_size(self):
|
||||
with pytest.raises(
|
||||
|
||||
@@ -5,6 +5,10 @@ from api.db_router import READ_REPLICA_ALIAS, MainRouter
|
||||
from api.models import Integration
|
||||
from api.utils import prowler_integration_connection_test
|
||||
from django.db import OperationalError
|
||||
from prowler.lib.outputs.jira.exceptions.exceptions import (
|
||||
JiraRefreshTokenError,
|
||||
JiraRequiredCustomFieldsError,
|
||||
)
|
||||
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
|
||||
from prowler.providers.common.models import Connection
|
||||
from tasks.jobs.integrations import (
|
||||
@@ -1830,10 +1834,213 @@ class TestJiraIntegration:
|
||||
)
|
||||
|
||||
# Assertions
|
||||
assert result == {"created_count": 2, "failed_count": 1}
|
||||
assert result == {
|
||||
"created_count": 2,
|
||||
"failed_count": 1,
|
||||
"error": "Failed to create Jira issue.",
|
||||
}
|
||||
|
||||
# Verify error was logged for the failed finding
|
||||
mock_logger.error.assert_called_with("Failed to send finding finding-2 to Jira")
|
||||
mock_logger.error.assert_called_with("Failed to create Jira issue.")
|
||||
|
||||
@patch("tasks.jobs.integrations.rls_transaction")
|
||||
@patch("tasks.jobs.integrations.Finding")
|
||||
@patch("tasks.jobs.integrations.Integration")
|
||||
@patch("tasks.jobs.integrations.initialize_prowler_integration")
|
||||
@patch("tasks.jobs.integrations.logger")
|
||||
def test_send_findings_to_jira_preserves_exception_message(
|
||||
self,
|
||||
mock_logger,
|
||||
mock_initialize_integration,
|
||||
mock_integration_model,
|
||||
mock_finding_model,
|
||||
mock_rls_transaction,
|
||||
):
|
||||
"""Test Jira send exceptions are returned for UI polling."""
|
||||
tenant_id = "tenant-123"
|
||||
integration_id = "integration-456"
|
||||
project_key = "PROJ"
|
||||
issue_type = "Task"
|
||||
finding_ids = ["finding-1"]
|
||||
error_message = "Jira project requires custom fields: Team is required"
|
||||
|
||||
mock_rls_transaction.return_value.__enter__ = MagicMock()
|
||||
mock_rls_transaction.return_value.__exit__ = MagicMock()
|
||||
|
||||
integration = MagicMock()
|
||||
mock_integration_model.objects.get.return_value = integration
|
||||
|
||||
mock_jira_integration = MagicMock()
|
||||
|
||||
mock_jira_integration.send_finding.side_effect = JiraRequiredCustomFieldsError(
|
||||
message=error_message
|
||||
)
|
||||
mock_initialize_integration.return_value = mock_jira_integration
|
||||
|
||||
finding = MagicMock()
|
||||
finding.id = "finding-1"
|
||||
finding.check_id = "check_001"
|
||||
finding.severity = "high"
|
||||
finding.status = "FAIL"
|
||||
finding.status_extended = "Resource is not compliant"
|
||||
finding.compliance = {}
|
||||
finding.resources.exists.return_value = False
|
||||
finding.resources.first.return_value = None
|
||||
finding.scan.provider.provider = "aws"
|
||||
finding.check_metadata = {
|
||||
"checktitle": "Check Title",
|
||||
"risk": "High risk",
|
||||
"remediation": {"recommendation": {}, "code": {}},
|
||||
}
|
||||
mock_select_related = mock_finding_model.all_objects.select_related.return_value
|
||||
mock_finding_query = mock_select_related.prefetch_related.return_value
|
||||
mock_finding_query.get.return_value = finding
|
||||
|
||||
result = send_findings_to_jira(
|
||||
tenant_id, integration_id, project_key, issue_type, finding_ids
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"created_count": 0,
|
||||
"failed_count": 1,
|
||||
"error": error_message,
|
||||
}
|
||||
mock_logger.exception.assert_called_with(
|
||||
"Failed to send finding %s to Jira: %s",
|
||||
"finding-1",
|
||||
error_message,
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.integrations.rls_transaction")
|
||||
@patch("tasks.jobs.integrations.Finding")
|
||||
@patch("tasks.jobs.integrations.Integration")
|
||||
@patch("tasks.jobs.integrations.initialize_prowler_integration")
|
||||
@patch("tasks.jobs.integrations.logger")
|
||||
def test_send_findings_to_jira_preserves_refresh_token_error_message(
|
||||
self,
|
||||
mock_logger,
|
||||
mock_initialize_integration,
|
||||
mock_integration_model,
|
||||
mock_finding_model,
|
||||
mock_rls_transaction,
|
||||
):
|
||||
"""Test Jira refresh token exceptions return their UI-friendly message."""
|
||||
tenant_id = "tenant-123"
|
||||
integration_id = "integration-456"
|
||||
project_key = "PROJ"
|
||||
issue_type = "Task"
|
||||
finding_ids = ["finding-1"]
|
||||
error_message = "Failed to refresh the access token"
|
||||
|
||||
mock_rls_transaction.return_value.__enter__ = MagicMock()
|
||||
mock_rls_transaction.return_value.__exit__ = MagicMock()
|
||||
|
||||
integration = MagicMock()
|
||||
mock_integration_model.objects.get.return_value = integration
|
||||
|
||||
mock_jira_integration = MagicMock()
|
||||
|
||||
mock_jira_integration.send_finding.side_effect = JiraRefreshTokenError(
|
||||
message=error_message
|
||||
)
|
||||
mock_initialize_integration.return_value = mock_jira_integration
|
||||
|
||||
finding = MagicMock()
|
||||
finding.id = "finding-1"
|
||||
finding.check_id = "check_001"
|
||||
finding.severity = "high"
|
||||
finding.status = "FAIL"
|
||||
finding.status_extended = "Resource is not compliant"
|
||||
finding.compliance = {}
|
||||
finding.resources.exists.return_value = False
|
||||
finding.resources.first.return_value = None
|
||||
finding.scan.provider.provider = "aws"
|
||||
finding.check_metadata = {
|
||||
"checktitle": "Check Title",
|
||||
"risk": "High risk",
|
||||
"remediation": {"recommendation": {}, "code": {}},
|
||||
}
|
||||
mock_select_related = mock_finding_model.all_objects.select_related.return_value
|
||||
mock_finding_query = mock_select_related.prefetch_related.return_value
|
||||
mock_finding_query.get.return_value = finding
|
||||
|
||||
result = send_findings_to_jira(
|
||||
tenant_id, integration_id, project_key, issue_type, finding_ids
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"created_count": 0,
|
||||
"failed_count": 1,
|
||||
"error": error_message,
|
||||
}
|
||||
mock_logger.exception.assert_called_with(
|
||||
"Failed to send finding %s to Jira: %s",
|
||||
"finding-1",
|
||||
error_message,
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.integrations.rls_transaction")
|
||||
@patch("tasks.jobs.integrations.Finding")
|
||||
@patch("tasks.jobs.integrations.Integration")
|
||||
@patch("tasks.jobs.integrations.initialize_prowler_integration")
|
||||
@patch("tasks.jobs.integrations.logger")
|
||||
def test_send_findings_to_jira_sanitizes_unexpected_exception_message(
|
||||
self,
|
||||
mock_logger,
|
||||
mock_initialize_integration,
|
||||
mock_integration_model,
|
||||
mock_finding_model,
|
||||
mock_rls_transaction,
|
||||
):
|
||||
"""Test unexpected Jira send exceptions do not leak raw details to UI."""
|
||||
tenant_id = "tenant-123"
|
||||
integration_id = "integration-456"
|
||||
project_key = "PROJ"
|
||||
issue_type = "Task"
|
||||
finding_ids = ["finding-1"]
|
||||
|
||||
mock_rls_transaction.return_value.__enter__ = MagicMock()
|
||||
mock_rls_transaction.return_value.__exit__ = MagicMock()
|
||||
|
||||
integration = MagicMock()
|
||||
mock_integration_model.objects.get.return_value = integration
|
||||
|
||||
mock_jira_integration = MagicMock()
|
||||
mock_jira_integration.send_finding.side_effect = Exception("token=secret-value")
|
||||
mock_initialize_integration.return_value = mock_jira_integration
|
||||
|
||||
finding = MagicMock()
|
||||
finding.id = "finding-1"
|
||||
finding.check_id = "check_001"
|
||||
finding.severity = "high"
|
||||
finding.status = "FAIL"
|
||||
finding.status_extended = "Resource is not compliant"
|
||||
finding.compliance = {}
|
||||
finding.resources.exists.return_value = False
|
||||
finding.resources.first.return_value = None
|
||||
finding.scan.provider.provider = "aws"
|
||||
finding.check_metadata = {
|
||||
"checktitle": "Check Title",
|
||||
"risk": "High risk",
|
||||
"remediation": {"recommendation": {}, "code": {}},
|
||||
}
|
||||
mock_select_related = mock_finding_model.all_objects.select_related.return_value
|
||||
mock_finding_query = mock_select_related.prefetch_related.return_value
|
||||
mock_finding_query.get.return_value = finding
|
||||
|
||||
result = send_findings_to_jira(
|
||||
tenant_id, integration_id, project_key, issue_type, finding_ids
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"created_count": 0,
|
||||
"failed_count": 1,
|
||||
"error": "Failed to create Jira issue.",
|
||||
}
|
||||
assert "secret-value" not in result["error"]
|
||||
mock_logger.exception.assert_called_with(
|
||||
"Failed to send finding %s to Jira", "finding-1"
|
||||
)
|
||||
|
||||
@patch("tasks.jobs.integrations.rls_transaction")
|
||||
@patch("tasks.jobs.integrations.Finding")
|
||||
|
||||
@@ -3652,6 +3652,95 @@ class TestAggregateFindings:
|
||||
regions = {s.region for s in summaries}
|
||||
assert regions == {"us-east-1", "us-west-2"}
|
||||
|
||||
@patch("tasks.jobs.scan.Finding.objects.filter")
|
||||
@patch("tasks.jobs.scan.ScanSummary.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
def test_aggregate_findings_orders_upserts_by_conflict_key(
|
||||
self, mock_rls_transaction, mock_bulk_create, mock_findings_filter
|
||||
):
|
||||
"""Scan summaries must use a stable lock order for concurrent upserts."""
|
||||
tenant_id = str(uuid.uuid4())
|
||||
scan_id = str(uuid.uuid4())
|
||||
counts = {
|
||||
"fail": 1,
|
||||
"_pass": 0,
|
||||
"muted_count": 0,
|
||||
"total": 1,
|
||||
"new": 1,
|
||||
"changed": 0,
|
||||
"unchanged": 0,
|
||||
"fail_new": 1,
|
||||
"fail_changed": 0,
|
||||
"pass_new": 0,
|
||||
"pass_changed": 0,
|
||||
"muted_new": 0,
|
||||
"muted_changed": 0,
|
||||
}
|
||||
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.values.return_value = mock_queryset
|
||||
mock_queryset.annotate.return_value = [
|
||||
{
|
||||
"check_id": "check-b",
|
||||
"resources__service": "s3",
|
||||
"severity": "high",
|
||||
"resources__region": "us-east-1",
|
||||
**counts,
|
||||
},
|
||||
{
|
||||
"check_id": "check-a",
|
||||
"resources__service": "sqs",
|
||||
"severity": "high",
|
||||
"resources__region": "us-east-1",
|
||||
**counts,
|
||||
},
|
||||
{
|
||||
"check_id": "check-a",
|
||||
"resources__service": "s3",
|
||||
"severity": "medium",
|
||||
"resources__region": "us-east-1",
|
||||
**counts,
|
||||
},
|
||||
{
|
||||
"check_id": "check-a",
|
||||
"resources__service": "s3",
|
||||
"severity": "high",
|
||||
"resources__region": "us-west-2",
|
||||
**counts,
|
||||
},
|
||||
{
|
||||
"check_id": "check-a",
|
||||
"resources__service": "s3",
|
||||
"severity": "high",
|
||||
"resources__region": "us-east-1",
|
||||
**counts,
|
||||
},
|
||||
]
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.__enter__.return_value = None
|
||||
ctx.__exit__.return_value = False
|
||||
mock_rls_transaction.return_value = ctx
|
||||
mock_findings_filter.return_value = mock_queryset
|
||||
|
||||
aggregate_findings(tenant_id, scan_id)
|
||||
|
||||
summaries = mock_bulk_create.call_args.args[0]
|
||||
assert isinstance(summaries, list)
|
||||
conflict_keys = [
|
||||
(
|
||||
str(summary.tenant_id),
|
||||
str(summary.scan_id),
|
||||
summary.check_id,
|
||||
summary.service,
|
||||
summary.severity,
|
||||
summary.region,
|
||||
)
|
||||
for summary in summaries
|
||||
]
|
||||
assert len(conflict_keys) == 5
|
||||
assert conflict_keys == sorted(conflict_keys)
|
||||
|
||||
@patch("tasks.jobs.scan.Finding.objects.filter")
|
||||
@patch("tasks.jobs.scan.ScanSummary.objects.bulk_create")
|
||||
@patch("tasks.jobs.scan.rls_transaction")
|
||||
|
||||
@@ -3,6 +3,7 @@ from contextlib import contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
from api.models import (
|
||||
@@ -20,6 +21,7 @@ from django_celery_results.models import TaskResult
|
||||
from tasks.jobs.lighthouse_providers import (
|
||||
_create_bedrock_client,
|
||||
_extract_bedrock_credentials,
|
||||
_LighthouseOpenAICompatibleNetworkBackend,
|
||||
)
|
||||
from tasks.tasks import (
|
||||
DJANGO_TMP_OUTPUT_DIRECTORY,
|
||||
@@ -1566,7 +1568,7 @@ class TestCheckLighthouseProviderConnectionTask:
|
||||
(
|
||||
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
{"api_key": "sk-test123"},
|
||||
"https://openrouter.ai/api/v1",
|
||||
"https://93.184.216.34/api/v1",
|
||||
{"connected": True, "error": None},
|
||||
),
|
||||
(
|
||||
@@ -1641,7 +1643,7 @@ class TestCheckLighthouseProviderConnectionTask:
|
||||
(
|
||||
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
{"api_key": "sk-invalid"},
|
||||
"https://openrouter.ai/api/v1",
|
||||
"https://93.184.216.34/api/v1",
|
||||
openai.APIConnectionError(request=MagicMock()),
|
||||
),
|
||||
(
|
||||
@@ -1755,6 +1757,166 @@ class TestCheckLighthouseProviderConnectionTask:
|
||||
provider_cfg.refresh_from_db()
|
||||
assert provider_cfg.is_active is False
|
||||
|
||||
def test_openai_compatible_connection_rejects_metadata_base_url_without_request(
|
||||
self, tenants_fixture
|
||||
):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://169.254.169.254/latest/meta-data",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
eager_result = check_lighthouse_provider_connection_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result["connected"] is False
|
||||
assert "base url" in result["error"].lower()
|
||||
mock_openai.assert_not_called()
|
||||
provider_cfg.refresh_from_db()
|
||||
assert provider_cfg.is_active is False
|
||||
|
||||
def test_openai_compatible_connection_disables_redirects(self, tenants_fixture):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=False,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.return_value = MagicMock()
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = check_lighthouse_provider_connection_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result == {"connected": True, "error": None}
|
||||
http_client = mock_openai.call_args.kwargs["http_client"]
|
||||
assert http_client.follow_redirects is False
|
||||
assert http_client.trust_env is False
|
||||
|
||||
def test_openai_compatible_connection_masks_remote_http_error(
|
||||
self, tenants_fixture
|
||||
):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
remote_body = "<!DOCTYPE HTML><p>remote 404 body</p>"
|
||||
response = httpx.Response(
|
||||
404,
|
||||
request=httpx.Request("GET", "https://provider.example/v1/models"),
|
||||
)
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.side_effect = openai.NotFoundError(
|
||||
remote_body,
|
||||
response=response,
|
||||
body=remote_body,
|
||||
)
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = check_lighthouse_provider_connection_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result == {"connected": False, "error": "Provider connection failed"}
|
||||
assert remote_body not in result["error"]
|
||||
provider_cfg.refresh_from_db()
|
||||
assert provider_cfg.is_active is False
|
||||
|
||||
def test_openai_compatible_connection_masks_remote_auth_error(
|
||||
self, tenants_fixture
|
||||
):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
remote_body = {"error": {"message": "remote auth detail"}}
|
||||
response = httpx.Response(
|
||||
401,
|
||||
request=httpx.Request("GET", "https://provider.example/v1/models"),
|
||||
)
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.side_effect = openai.AuthenticationError(
|
||||
"Unauthorized",
|
||||
response=response,
|
||||
body=remote_body,
|
||||
)
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = check_lighthouse_provider_connection_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result == {"connected": False, "error": "API key is invalid or missing"}
|
||||
assert "remote auth detail" not in result["error"]
|
||||
provider_cfg.refresh_from_db()
|
||||
assert provider_cfg.is_active is False
|
||||
|
||||
def test_openai_compatible_network_backend_uses_validated_ip(self, monkeypatch):
|
||||
backend = _LighthouseOpenAICompatibleNetworkBackend()
|
||||
stream = MagicMock()
|
||||
|
||||
def resolve_to_public_ip(host, port):
|
||||
del host, port
|
||||
return ("93.184.216.34",)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tasks.jobs.lighthouse_providers.resolve_lighthouse_openai_compatible_host",
|
||||
resolve_to_public_ip,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.lighthouse_providers.httpcore.SyncBackend.connect_tcp",
|
||||
return_value=stream,
|
||||
) as mock_connect_tcp:
|
||||
result = backend.connect_tcp("provider.example", 443, timeout=1.0)
|
||||
|
||||
assert result is stream
|
||||
assert mock_connect_tcp.call_args.args[:2] == ("93.184.216.34", 443)
|
||||
assert mock_connect_tcp.call_args.kwargs["timeout"] == 1.0
|
||||
|
||||
def test_check_connection_provider_does_not_exist(self, tenants_fixture):
|
||||
"""Test that checking non-existent provider raises DoesNotExist."""
|
||||
non_existent_id = str(uuid.uuid4())
|
||||
@@ -1784,7 +1946,7 @@ class TestRefreshLighthouseProviderModelsTask:
|
||||
(
|
||||
LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
{"api_key": "sk-test123"},
|
||||
"https://openrouter.ai/api/v1",
|
||||
"https://93.184.216.34/api/v1",
|
||||
{"model-1": "Model One", "model-2": "Model Two"},
|
||||
2,
|
||||
),
|
||||
@@ -1864,6 +2026,106 @@ class TestRefreshLighthouseProviderModelsTask:
|
||||
== expected_count
|
||||
)
|
||||
|
||||
def test_refresh_models_rejects_metadata_base_url_without_request(
|
||||
self, tenants_fixture
|
||||
):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://169.254.169.254/latest/meta-data",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
|
||||
with patch(
|
||||
"tasks.jobs.lighthouse_providers._fetch_openai_compatible_models"
|
||||
) as mock_fetch:
|
||||
eager_result = refresh_lighthouse_provider_models_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result["created"] == 0
|
||||
assert result["updated"] == 0
|
||||
assert result["deleted"] == 0
|
||||
assert "base url" in result["error"].lower()
|
||||
mock_fetch.assert_not_called()
|
||||
|
||||
def test_refresh_models_disables_redirects(self, tenants_fixture):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.return_value = MagicMock(data=[])
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = refresh_lighthouse_provider_models_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result["created"] == 0
|
||||
assert result["updated"] == 0
|
||||
assert result["deleted"] == 0
|
||||
http_client = mock_openai.call_args.kwargs["http_client"]
|
||||
assert http_client.follow_redirects is False
|
||||
assert http_client.trust_env is False
|
||||
|
||||
def test_refresh_models_masks_remote_http_error(self, tenants_fixture):
|
||||
provider_cfg = LighthouseProviderConfiguration(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type=LighthouseProviderConfiguration.LLMProviderChoices.OPENAI_COMPATIBLE,
|
||||
base_url="https://93.184.216.34/api/v1",
|
||||
is_active=True,
|
||||
)
|
||||
provider_cfg.credentials_decoded = {"api_key": "compatible-key"}
|
||||
provider_cfg.save()
|
||||
remote_body = "<!DOCTYPE HTML><p>remote 404 body</p>"
|
||||
response = httpx.Response(
|
||||
404,
|
||||
request=httpx.Request("GET", "https://provider.example/v1/models"),
|
||||
)
|
||||
|
||||
with patch("tasks.jobs.lighthouse_providers.openai.OpenAI") as mock_openai:
|
||||
mock_client = MagicMock()
|
||||
mock_client.models.list.side_effect = openai.NotFoundError(
|
||||
remote_body,
|
||||
response=response,
|
||||
body=remote_body,
|
||||
)
|
||||
mock_openai.return_value = mock_client
|
||||
|
||||
eager_result = refresh_lighthouse_provider_models_task.apply(
|
||||
kwargs={
|
||||
"provider_config_id": str(provider_cfg.id),
|
||||
"tenant_id": str(tenants_fixture[0].id),
|
||||
}
|
||||
)
|
||||
|
||||
assert eager_result.successful()
|
||||
result = eager_result.result
|
||||
assert result["created"] == 0
|
||||
assert result["updated"] == 0
|
||||
assert result["deleted"] == 0
|
||||
assert result["error"] == "Provider connection failed"
|
||||
assert remote_body not in result["error"]
|
||||
|
||||
def test_refresh_models_mixed_operations(self, tenants_fixture):
|
||||
"""Test mixed create, update, and delete operations."""
|
||||
# Create provider configuration
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
[tool.towncrier]
|
||||
directory = "changelog.d"
|
||||
filename = "CHANGELOG.md"
|
||||
start_string = "<!-- changelog: release notes start -->\n"
|
||||
title_format = "## [{version}] ({name})"
|
||||
issue_format = "[(#{issue})](https://github.com/prowler-cloud/prowler/pull/{issue})"
|
||||
template = "../.github/towncrier/template.md.jinja"
|
||||
underlines = ["", "", ""]
|
||||
ignore = [".gitkeep", "README.md"]
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "added"
|
||||
name = "🚀 Added"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "changed"
|
||||
name = "🔄 Changed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "deprecated"
|
||||
name = "⚠️ Deprecated"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "removed"
|
||||
name = "❌ Removed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "fixed"
|
||||
name = "🐞 Fixed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "security"
|
||||
name = "🔐 Security"
|
||||
showcontent = true
|
||||
Generated
+4
-4
@@ -4673,8 +4673,8 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler"
|
||||
version = "5.32.0"
|
||||
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=master#5dac8a0a53272e4db68c476fb969dc03e88beb68" }
|
||||
version = "5.33.0"
|
||||
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=v5.33#76a2d7bfe61a3ac6c96e497eded30a2978cd3ffe" }
|
||||
dependencies = [
|
||||
{ name = "alibabacloud-actiontrail20200706" },
|
||||
{ name = "alibabacloud-credentials" },
|
||||
@@ -4762,7 +4762,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler-api"
|
||||
version = "1.34.0"
|
||||
version = "1.34.2"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "cartography" },
|
||||
@@ -4862,7 +4862,7 @@ requires-dist = [
|
||||
{ name = "matplotlib", specifier = "==3.10.8" },
|
||||
{ name = "neo4j", specifier = "==6.1.0" },
|
||||
{ name = "openai", specifier = "==1.109.1" },
|
||||
{ name = "prowler", git = "https://github.com/prowler-cloud/prowler.git?rev=master" },
|
||||
{ name = "prowler", git = "https://github.com/prowler-cloud/prowler.git?rev=v5.33" },
|
||||
{ name = "psycopg2-binary", specifier = "==2.9.9" },
|
||||
{ name = "pytest-celery", extras = ["redis"], specifier = "==1.3.0" },
|
||||
{ name = "reportlab", specifier = "==4.4.10" },
|
||||
|
||||
@@ -3421,7 +3421,7 @@ Use existing providers as templates, this will help you to understand better the
|
||||
- **Documentation & Maintenance**
|
||||
|
||||
- **README Updates**: Update provider-specific documentation
|
||||
- **Changelog**: Document changes and new features
|
||||
- **Changelog**: Document changes and new features with a fragment under `prowler/changelog.d/` (see the [Pull Request Template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md))
|
||||
- **Examples**: Provide usage examples and common scenarios
|
||||
- **Troubleshooting**: Include common issues and solutions
|
||||
- **Documentation**: Update the provider documentation to include your new tool provider in the examples and implementation guidance.
|
||||
|
||||
@@ -778,7 +778,7 @@ Before opening the pull request:
|
||||
uv run pre-commit run --all-files
|
||||
uv run pytest -n auto
|
||||
```
|
||||
2. Add a changelog entry under the `### 🚀 Added` section of `prowler/CHANGELOG.md`, describing the new framework and the providers it covers.
|
||||
2. Add a changelog fragment `prowler/changelog.d/<slug>.added.md`, describing the new framework and the providers it covers (no PR link in the text; it is attached automatically at release time).
|
||||
3. Follow the [Pull Request Template](https://github.com/prowler-cloud/prowler/blob/master/.github/pull_request_template.md) and set the PR title using Conventional Commits, e.g. `feat(compliance): add My Framework 1.0 for AWS`.
|
||||
4. Request review from the compliance codeowners listed in `.github/CODEOWNERS`.
|
||||
|
||||
|
||||
@@ -210,7 +210,9 @@ For more detailed guidance on subscription management and permissions:
|
||||
The following security checks require the `ProwlerRole` permissions for execution. Ensure the role is assigned to the identity assumed by Prowler before running these checks:
|
||||
|
||||
- `app_function_access_keys_configured`
|
||||
- `app_function_application_insights_enabled`
|
||||
- `app_function_ftps_deployment_disabled`
|
||||
- `app_function_latest_runtime_version`
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -128,6 +128,25 @@ To connect a provider:
|
||||
3. Configure in Lighthouse AI:
|
||||
- **API Key**: OpenRouter API key
|
||||
- **Base URL**: `https://openrouter.ai/api/v1`
|
||||
|
||||
### Base URL Validation
|
||||
|
||||
To prevent server-side request forgery (SSRF), Prowler API validates the base URL before connecting to it:
|
||||
|
||||
- The URL must use HTTPS.
|
||||
- The host must resolve to a public IP address. Private, loopback, link-local, and cloud metadata addresses are rejected.
|
||||
|
||||
<Warning>
|
||||
This validation can break configurations that point to internal endpoints, such as a self-hosted Ollama server. This is intentional: it fixes a security issue where the Prowler API could be directed to internal services. Internal endpoints must now be allowed explicitly through `LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS`.
|
||||
</Warning>
|
||||
|
||||
To allow internal endpoints, set a comma-separated list of hostnames or IP addresses in the Prowler API environment (for Docker Compose deployments, the shared `.env` file):
|
||||
|
||||
```bash
|
||||
LIGHTHOUSE_AI_OPENAI_COMPATIBLE_ALLOWED_HOSTS=custom-openai.internal,10.0.0.20
|
||||
```
|
||||
|
||||
Hosts in this list skip the public-endpoint validation. HTTPS is still required, so the endpoint needs a certificate the Prowler API trusts.
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
All notable changes to the **Prowler MCP Server** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [0.7.2] (Prowler v5.28.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Changelog fragments
|
||||
|
||||
Each PR adds one small file here instead of editing `CHANGELOG.md` directly, so concurrent PRs never conflict.
|
||||
|
||||
- Filename: `<slug>.<type>.md`, e.g. `my-new-check.added.md` (slug is free-form: letters, digits, `.`, `_`, `-`)
|
||||
- `<type>` is one of: `added`, `changed`, `deprecated`, `removed`, `fixed`, `security`
|
||||
- Content: one line with the changelog entry text, without the PR link and without a trailing period (the PR link is attached automatically at release time)
|
||||
- A PR adds as many fragment files as entries it needs, freely mixing types (one file per entry); same-type entries just use different slugs
|
||||
|
||||
Fragments are compiled into `CHANGELOG.md` when a release is prepared. Full conventions: `skills/prowler-changelog/SKILL.md`.
|
||||
@@ -0,0 +1,39 @@
|
||||
[tool.towncrier]
|
||||
directory = "changelog.d"
|
||||
filename = "CHANGELOG.md"
|
||||
start_string = "<!-- changelog: release notes start -->\n"
|
||||
title_format = "## [{version}] ({name})"
|
||||
issue_format = "[(#{issue})](https://github.com/prowler-cloud/prowler/pull/{issue})"
|
||||
template = "../.github/towncrier/template.md.jinja"
|
||||
underlines = ["", "", ""]
|
||||
ignore = [".gitkeep", "README.md"]
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "added"
|
||||
name = "🚀 Added"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "changed"
|
||||
name = "🔄 Changed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "deprecated"
|
||||
name = "⚠️ Deprecated"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "removed"
|
||||
name = "❌ Removed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "fixed"
|
||||
name = "🐞 Fixed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "security"
|
||||
name = "🔐 Security"
|
||||
showcontent = true
|
||||
@@ -2,6 +2,30 @@
|
||||
|
||||
All notable changes to the **Prowler SDK** are documented in this file.
|
||||
|
||||
<!-- changelog: release notes start -->
|
||||
|
||||
## [5.33.2] (Prowler v5.33.2)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- EC2 AMI loading now targets Amazon-owned AMIs used by audited instances, reducing AWS API calls during EC2 scans [(#11958)](https://github.com/prowler-cloud/prowler/pull/11958)
|
||||
- `ec2_instance_account_imdsv2_enabled` findings now use regional resource ARNs, preventing findings from different AWS Regions from collapsing into one resource [(#11966)](https://github.com/prowler-cloud/prowler/pull/11966)
|
||||
|
||||
---
|
||||
|
||||
## [5.33.1] (Prowler v5.33.1)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- ECS task definition resource limits now select the latest task definitions by registration date instead of relying on ARN ordering [(#11891)](https://github.com/prowler-cloud/prowler/pull/11891)
|
||||
- `dlm_ebs_snapshot_lifecycle_policy_exists` no longer initializes the full EC2 inventory just to detect EBS snapshots, avoiding slow scans when checking DLM lifecycle policies [(#11900)](https://github.com/prowler-cloud/prowler/pull/11900)
|
||||
- `dms_instance_no_public_access` no longer initializes the full EC2 service when there are no DMS replication instances [(#11902)](https://github.com/prowler-cloud/prowler/pull/11902)
|
||||
- `organizations_scp_check_deny_regions` no longer reports false `FAIL` for AWS Organizations that restrict regions with Allow-based SCPs; the Allow path now checks the statement `Effect` instead of an always-false comparison that made it unreachable [(#11915)](https://github.com/prowler-cloud/prowler/pull/11915)
|
||||
- Jira issue creation failures now preserve safe structured response details from Jira [(#11925)](https://github.com/prowler-cloud/prowler/pull/11925)
|
||||
- Azure Function App optional permission failures now log as warnings, and Function App environment variable fields use the correct spelling internally [(#11926)](https://github.com/prowler-cloud/prowler/pull/11926)
|
||||
|
||||
---
|
||||
|
||||
## [5.33.0] (Prowler v5.33.0)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Changelog fragments
|
||||
|
||||
Each PR adds one small file here instead of editing `CHANGELOG.md` directly, so concurrent PRs never conflict.
|
||||
|
||||
- Filename: `<slug>.<type>.md`, e.g. `my-new-check.added.md` (slug is free-form: letters, digits, `.`, `_`, `-`)
|
||||
- `<type>` is one of: `added`, `changed`, `deprecated`, `removed`, `fixed`, `security`
|
||||
- Content: one line with the changelog entry text, without the PR link and without a trailing period (the PR link is attached automatically at release time)
|
||||
- A PR adds as many fragment files as entries it needs, freely mixing types (one file per entry); same-type entries just use different slugs
|
||||
|
||||
Fragments are compiled into `CHANGELOG.md` when a release is prepared. Full conventions: `skills/prowler-changelog/SKILL.md`.
|
||||
@@ -49,7 +49,7 @@ class _MutableTimestamp:
|
||||
|
||||
timestamp = _MutableTimestamp(datetime.today())
|
||||
timestamp_utc = _MutableTimestamp(datetime.now(timezone.utc))
|
||||
prowler_version = "5.33.0"
|
||||
prowler_version = "5.33.2"
|
||||
html_logo_url = "https://github.com/prowler-cloud/prowler/"
|
||||
square_logo_img = "https://raw.githubusercontent.com/prowler-cloud/prowler/dc7d2d5aeb92fdf12e8604f42ef6472cd3e8e889/docs/img/prowler-logo-black.png"
|
||||
aws_logo = "https://user-images.githubusercontent.com/38561120/235953920-3e3fba08-0795-41dc-b480-9bea57db9f2e.png"
|
||||
|
||||
@@ -51,6 +51,39 @@ class JiraConnection(Connection):
|
||||
issue_types: dict = None
|
||||
|
||||
|
||||
def _format_jira_issue_creation_error(response_json: object, status_code: int) -> str:
|
||||
"""Build a safe Jira issue creation error message from structured fields.
|
||||
|
||||
Args:
|
||||
response_json: Parsed Jira response body.
|
||||
status_code: HTTP status code returned by Jira.
|
||||
|
||||
Returns:
|
||||
Safe issue creation error message for user-facing propagation.
|
||||
"""
|
||||
message_parts = []
|
||||
|
||||
if not isinstance(response_json, dict):
|
||||
return f"Failed to create Jira issue: Jira returned status code {status_code}."
|
||||
|
||||
errors = response_json.get("errors")
|
||||
if isinstance(errors, dict):
|
||||
message_parts.extend(
|
||||
f"'{field}': '{message}'" for field, message in errors.items() if message
|
||||
)
|
||||
|
||||
error_messages = response_json.get("errorMessages")
|
||||
if isinstance(error_messages, list):
|
||||
message_parts.extend(str(message) for message in error_messages if message)
|
||||
elif isinstance(error_messages, str) and error_messages:
|
||||
message_parts.append(error_messages)
|
||||
|
||||
if message_parts:
|
||||
return f"Failed to create Jira issue: {'; '.join(message_parts)}"
|
||||
|
||||
return f"Failed to create Jira issue: Jira returned status code {status_code}."
|
||||
|
||||
|
||||
class MarkdownToADFConverter:
|
||||
"""Helper to convert Markdown strings into Atlassian Document Format blocks."""
|
||||
|
||||
@@ -2060,6 +2093,7 @@ class Jira:
|
||||
Raises:
|
||||
- JiraRefreshTokenError: Failed to refresh the access token
|
||||
- JiraRefreshTokenResponseError: Failed to refresh the access token, response code did not match 200
|
||||
- JiraNoTokenError: Failed to get an access token
|
||||
- JiraCreateIssueError: Failed to create an issue in Jira
|
||||
- JiraSendFindingsResponseError: Failed to send the finding to Jira
|
||||
- JiraRequiredCustomFieldsError: Jira project requires custom fields that are not supported
|
||||
@@ -2155,31 +2189,45 @@ class Jira:
|
||||
try:
|
||||
response_json = response.json()
|
||||
except (ValueError, requests.exceptions.JSONDecodeError):
|
||||
response_error = f"Failed to send finding: {response.status_code} - {response.text}"
|
||||
response_error = _format_jira_issue_creation_error(
|
||||
{}, response.status_code
|
||||
)
|
||||
logger.error(response_error)
|
||||
return False
|
||||
raise JiraSendFindingsResponseError(
|
||||
message=response_error, file=os.path.basename(__file__)
|
||||
)
|
||||
|
||||
# Check if the error is due to required custom fields
|
||||
if response.status_code == 400 and "errors" in response_json:
|
||||
if (
|
||||
response.status_code == 400
|
||||
and isinstance(response_json, dict)
|
||||
and "errors" in response_json
|
||||
):
|
||||
errors = response_json.get("errors", {})
|
||||
# Look for custom field errors (fields starting with "customfield_")
|
||||
custom_field_errors = {
|
||||
k: v for k, v in errors.items() if k.startswith("customfield_")
|
||||
}
|
||||
custom_field_errors = {}
|
||||
if isinstance(errors, dict):
|
||||
custom_field_errors = {
|
||||
k: v
|
||||
for k, v in errors.items()
|
||||
if k.startswith("customfield_")
|
||||
}
|
||||
if custom_field_errors:
|
||||
custom_fields_formatted = ", ".join(
|
||||
[f"'{k}': '{v}'" for k, v in custom_field_errors.items()]
|
||||
)
|
||||
logger.error(
|
||||
f"Jira project requires custom fields that are not supported: {custom_fields_formatted}"
|
||||
raise JiraRequiredCustomFieldsError(
|
||||
message=f"Jira project requires custom fields that are not supported: {custom_fields_formatted}",
|
||||
file=os.path.basename(__file__),
|
||||
)
|
||||
return False
|
||||
|
||||
response_error = (
|
||||
f"Failed to send finding: {response.status_code} - {response_json}"
|
||||
response_error = _format_jira_issue_creation_error(
|
||||
response_json, response.status_code
|
||||
)
|
||||
logger.error(response_error)
|
||||
return False
|
||||
raise JiraSendFindingsResponseError(
|
||||
message=response_error, file=os.path.basename(__file__)
|
||||
)
|
||||
else:
|
||||
try:
|
||||
response_json = response.json()
|
||||
@@ -2191,13 +2239,17 @@ class Jira:
|
||||
return True
|
||||
except JiraRequiredCustomFieldsError as custom_fields_error:
|
||||
logger.error(f"Custom fields error: {custom_fields_error}")
|
||||
return False
|
||||
raise custom_fields_error
|
||||
except JiraSendFindingsResponseError as response_error:
|
||||
logger.error(f"Jira response error: {response_error}")
|
||||
raise response_error
|
||||
except JiraRefreshTokenError as refresh_error:
|
||||
logger.error(f"Token refresh error: {refresh_error}")
|
||||
return False
|
||||
raise refresh_error
|
||||
except JiraRefreshTokenResponseError as response_error:
|
||||
logger.error(f"Token response error: {response_error}")
|
||||
return False
|
||||
raise response_error
|
||||
except JiraNoTokenError as no_token_error:
|
||||
raise no_token_error
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send finding: {e}")
|
||||
return False
|
||||
|
||||
+2
-3
@@ -1,6 +1,5 @@
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.providers.aws.services.dlm.dlm_client import dlm_client
|
||||
from prowler.providers.aws.services.ec2.ec2_client import ec2_client
|
||||
|
||||
|
||||
class dlm_ebs_snapshot_lifecycle_policy_exists(Check):
|
||||
@@ -8,8 +7,8 @@ class dlm_ebs_snapshot_lifecycle_policy_exists(Check):
|
||||
findings = []
|
||||
for region in dlm_client.lifecycle_policies:
|
||||
if (
|
||||
region in ec2_client.regions_with_snapshots
|
||||
and ec2_client.regions_with_snapshots[region]
|
||||
region in dlm_client.regions_with_snapshots
|
||||
and dlm_client.regions_with_snapshots[region]
|
||||
):
|
||||
report = Check_Report_AWS(
|
||||
metadata=self.metadata(),
|
||||
|
||||
@@ -9,7 +9,13 @@ class DLM(AWSService):
|
||||
# Call AWSService's __init__
|
||||
super().__init__(__class__.__name__, provider)
|
||||
self.lifecycle_policies = {}
|
||||
self.regions_with_snapshots = {}
|
||||
self.__threading_call__(self._get_lifecycle_policies)
|
||||
ec2_regional_clients = provider.generate_regional_clients("ec2") or {}
|
||||
self.__threading_call__(
|
||||
self._get_regions_with_snapshots,
|
||||
iterator=ec2_regional_clients.values(),
|
||||
)
|
||||
|
||||
def _get_lifecycle_policy_arn_template(self, region):
|
||||
return (
|
||||
@@ -35,6 +41,34 @@ class DLM(AWSService):
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
def _get_regions_with_snapshots(self, regional_client):
|
||||
logger.info("DLM - Checking regions with self-owned EBS snapshots...")
|
||||
try:
|
||||
self.regions_with_snapshots[regional_client.region] = False
|
||||
next_token = None
|
||||
while True:
|
||||
describe_snapshots_args = {
|
||||
"OwnerIds": ["self"],
|
||||
"MaxResults": 5,
|
||||
}
|
||||
if next_token:
|
||||
describe_snapshots_args["NextToken"] = next_token
|
||||
|
||||
snapshots = regional_client.describe_snapshots(
|
||||
**describe_snapshots_args
|
||||
)
|
||||
if snapshots.get("Snapshots"):
|
||||
self.regions_with_snapshots[regional_client.region] = True
|
||||
break
|
||||
|
||||
next_token = snapshots.get("NextToken")
|
||||
if not next_token:
|
||||
break
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
|
||||
class LifecyclePolicy(BaseModel):
|
||||
id: str
|
||||
|
||||
+7
-2
@@ -1,9 +1,14 @@
|
||||
from prowler.lib.check.models import Check, Check_Report_AWS
|
||||
from prowler.providers.aws.services.dms.dms_client import dms_client
|
||||
from prowler.providers.aws.services.ec2.ec2_client import ec2_client
|
||||
from prowler.providers.aws.services.ec2.lib.security_groups import check_security_group
|
||||
|
||||
|
||||
def _get_ec2_client():
|
||||
from prowler.providers.aws.services.ec2.ec2_client import ec2_client
|
||||
|
||||
return ec2_client
|
||||
|
||||
|
||||
class dms_instance_no_public_access(Check):
|
||||
def execute(self):
|
||||
findings = []
|
||||
@@ -19,7 +24,7 @@ class dms_instance_no_public_access(Check):
|
||||
if instance.security_groups:
|
||||
report.status = "PASS"
|
||||
report.status_extended = f"DMS Replication Instance {instance.id} is set as publicly accessible but filtered with security groups."
|
||||
for security_group in ec2_client.security_groups.values():
|
||||
for security_group in _get_ec2_client().security_groups.values():
|
||||
if security_group.id in instance.security_groups:
|
||||
for ingress_rule in security_group.ingress_rules:
|
||||
if check_security_group(
|
||||
|
||||
@@ -18,4 +18,4 @@ class ec2_ami_public(Check):
|
||||
|
||||
findings.append(report)
|
||||
|
||||
return findings
|
||||
return findings
|
||||
|
||||
+5
-1
@@ -14,7 +14,11 @@ class ec2_instance_account_imdsv2_enabled(Check):
|
||||
metadata=self.metadata(),
|
||||
resource=instance_metadata_default,
|
||||
)
|
||||
report.resource_arn = ec2_client.account_arn_template
|
||||
report.resource_arn = (
|
||||
f"arn:{ec2_client.audited_partition}:ec2:"
|
||||
f"{instance_metadata_default.region}:"
|
||||
f"{ec2_client.audited_account}:account"
|
||||
)
|
||||
report.resource_id = ec2_client.audited_account
|
||||
if instance_metadata_default.http_tokens == "required":
|
||||
report.status = "PASS"
|
||||
|
||||
+5
-4
@@ -26,11 +26,12 @@ class ec2_instance_with_outdated_ami(Check):
|
||||
List[Check_Report_AWS]: A list containing the results of the check for each instance.
|
||||
"""
|
||||
findings = []
|
||||
images_by_id = getattr(ec2_client, "images_by_id", None)
|
||||
if images_by_id is None:
|
||||
images_by_id = {image.id: image for image in ec2_client.images}
|
||||
|
||||
for instance in ec2_client.instances:
|
||||
ami = next(
|
||||
(image for image in ec2_client.images if image.id == instance.image_id),
|
||||
None,
|
||||
)
|
||||
ami = images_by_id.get(instance.image_id)
|
||||
if ami and ami.owner == "amazon":
|
||||
report = Check_Report_AWS(metadata=self.metadata(), resource=instance)
|
||||
report.status = "PASS"
|
||||
|
||||
@@ -13,6 +13,8 @@ from prowler.lib.resource_limit import (
|
||||
from prowler.lib.scan_filters.scan_filters import is_resource_filtered
|
||||
from prowler.providers.aws.lib.service.service import AWSService
|
||||
|
||||
DESCRIBE_IMAGES_IMAGE_IDS_BATCH_SIZE = 200
|
||||
|
||||
|
||||
class EC2(AWSService):
|
||||
def __init__(self, provider):
|
||||
@@ -39,6 +41,7 @@ class EC2(AWSService):
|
||||
self.network_interfaces = {}
|
||||
self.__threading_call__(self._describe_network_interfaces)
|
||||
self.images = []
|
||||
self.images_by_id = {}
|
||||
self.__threading_call__(self._describe_images)
|
||||
self.volumes = []
|
||||
self.__threading_call__(self._describe_volumes)
|
||||
@@ -372,36 +375,90 @@ class EC2(AWSService):
|
||||
|
||||
def _describe_images(self, regional_client):
|
||||
try:
|
||||
for owner in ["self", "amazon"]:
|
||||
try:
|
||||
for image in regional_client.describe_images(
|
||||
Owners=[owner], IncludeDeprecated=True
|
||||
)["Images"]:
|
||||
arn = f"arn:{self.audited_partition}:ec2:{regional_client.region}:{self.audited_account}:image/{image['ImageId']}"
|
||||
if not self.audit_resources or (
|
||||
is_resource_filtered(arn, self.audit_resources)
|
||||
):
|
||||
self.images.append(
|
||||
Image(
|
||||
id=image["ImageId"],
|
||||
arn=arn,
|
||||
name=image.get("Name", ""),
|
||||
public=image.get("Public", False),
|
||||
region=regional_client.region,
|
||||
tags=image.get("Tags"),
|
||||
deprecation_time=image.get("DeprecationTime"),
|
||||
owner=owner,
|
||||
)
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
try:
|
||||
for image in regional_client.describe_images(
|
||||
Owners=["self"], IncludeDeprecated=True
|
||||
)["Images"]:
|
||||
self._add_image(image, regional_client.region, "self")
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
amazon_image_ids = sorted(
|
||||
{
|
||||
instance.image_id
|
||||
for instance in self.instances
|
||||
if instance.region == regional_client.region
|
||||
and instance.image_id
|
||||
and instance.image_id not in self.images_by_id
|
||||
}
|
||||
)
|
||||
|
||||
for image_batch in self._get_image_id_batches(amazon_image_ids):
|
||||
for image in self._describe_images_by_id(regional_client, image_batch):
|
||||
if self._is_amazon_image(image):
|
||||
self._add_image(image, regional_client.region, "amazon")
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
|
||||
def _add_image(self, image, region, owner):
|
||||
arn = f"arn:{self.audited_partition}:ec2:{region}:{self.audited_account}:image/{image['ImageId']}"
|
||||
if not self.audit_resources or (
|
||||
is_resource_filtered(arn, self.audit_resources)
|
||||
):
|
||||
ec2_image = Image(
|
||||
id=image["ImageId"],
|
||||
arn=arn,
|
||||
name=image.get("Name", ""),
|
||||
public=image.get("Public", False),
|
||||
region=region,
|
||||
tags=image.get("Tags"),
|
||||
deprecation_time=image.get("DeprecationTime"),
|
||||
owner=owner,
|
||||
)
|
||||
self.images.append(ec2_image)
|
||||
self.images_by_id[ec2_image.id] = ec2_image
|
||||
|
||||
def _describe_images_by_id(self, regional_client, image_ids):
|
||||
try:
|
||||
return regional_client.describe_images(
|
||||
ImageIds=image_ids, IncludeDeprecated=True
|
||||
)["Images"]
|
||||
except ClientError as error:
|
||||
if error.response["Error"]["Code"] == "InvalidAMIID.NotFound":
|
||||
if len(image_ids) == 1:
|
||||
logger.warning(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
return []
|
||||
|
||||
midpoint = len(image_ids) // 2
|
||||
return self._describe_images_by_id(
|
||||
regional_client, image_ids[:midpoint]
|
||||
) + self._describe_images_by_id(regional_client, image_ids[midpoint:])
|
||||
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
return []
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _get_image_id_batches(image_ids):
|
||||
for index in range(0, len(image_ids), DESCRIBE_IMAGES_IMAGE_IDS_BATCH_SIZE):
|
||||
yield image_ids[index : index + DESCRIBE_IMAGES_IMAGE_IDS_BATCH_SIZE]
|
||||
|
||||
@staticmethod
|
||||
def _is_amazon_image(image):
|
||||
return image.get("ImageOwnerAlias") == "amazon"
|
||||
|
||||
def _describe_volumes(self, regional_client):
|
||||
try:
|
||||
describe_volumes_paginator = regional_client.get_paginator(
|
||||
|
||||
@@ -81,12 +81,13 @@ class ECS(AWSService):
|
||||
|
||||
Resources already fetched are memoized in ``self.task_definitions`` and
|
||||
reused across checks (checks run sequentially, so no locking is needed).
|
||||
The configured resource limit bounds ``describe_task_definition`` calls.
|
||||
Task definitions are described before applying the configured resource
|
||||
limit because AWS exposes ``registeredAt`` only through
|
||||
``describe_task_definition``. The limit bounds the task definitions
|
||||
exposed to checks for analysis.
|
||||
"""
|
||||
task_definitions = []
|
||||
for arn, region in limit_resources(
|
||||
self._list_task_definition_arns(), self.task_definition_limit
|
||||
):
|
||||
for arn, region in self._list_task_definition_arns():
|
||||
task_definition = self.task_definitions.get(arn)
|
||||
if task_definition is None:
|
||||
task_definition = TaskDefinition(
|
||||
@@ -102,12 +103,42 @@ class ECS(AWSService):
|
||||
|
||||
self.__threading_call__(self._describe_task_definition, task_definitions)
|
||||
|
||||
for arn, _ in limit_resources(
|
||||
self._list_task_definition_arns(), self.task_definition_limit
|
||||
):
|
||||
task_definition = self.task_definitions[arn]
|
||||
selected_task_definitions = list(
|
||||
limit_resources(
|
||||
self._sort_task_definitions_by_registration_date(
|
||||
self.task_definitions.values()
|
||||
),
|
||||
self.task_definition_limit,
|
||||
)
|
||||
)
|
||||
self.task_definitions = {
|
||||
task_definition.arn: task_definition
|
||||
for task_definition in selected_task_definitions
|
||||
}
|
||||
for task_definition in selected_task_definitions:
|
||||
yield task_definition
|
||||
|
||||
@staticmethod
|
||||
def _sort_task_definitions_by_registration_date(task_definitions):
|
||||
task_definitions = list(task_definitions)
|
||||
if not any(
|
||||
task_definition.registered_at for task_definition in task_definitions
|
||||
):
|
||||
return task_definitions
|
||||
|
||||
return sorted(
|
||||
task_definitions,
|
||||
key=lambda task_definition: (
|
||||
task_definition.registered_at is None,
|
||||
(
|
||||
-task_definition.registered_at.timestamp()
|
||||
if task_definition.registered_at
|
||||
else 0
|
||||
),
|
||||
task_definition.arn,
|
||||
),
|
||||
)
|
||||
|
||||
def _describe_task_definition(self, task_definition):
|
||||
logger.info("ECS - Describing Task Definition...")
|
||||
try:
|
||||
|
||||
+1
-1
@@ -67,7 +67,7 @@ class organizations_scp_check_deny_regions(Check):
|
||||
|
||||
# Allow if Condition = {"StringEquals": {"aws:RequestedRegion": [region1, region2]}}
|
||||
if (
|
||||
policy.content.get("Statement") == "Allow"
|
||||
statement.get("Effect") == "Allow"
|
||||
and "Condition" in statement
|
||||
and "StringEquals" in statement["Condition"]
|
||||
and "aws:RequestedRegion"
|
||||
|
||||
+3
-3
@@ -14,7 +14,7 @@ class app_function_application_insights_enabled(Check):
|
||||
subscription_id, subscription_id
|
||||
)
|
||||
for function in functions.values():
|
||||
if function.enviroment_variables is not None:
|
||||
if function.environment_variables is not None:
|
||||
report = Check_Report_Azure(
|
||||
metadata=self.metadata(), resource=function
|
||||
)
|
||||
@@ -22,9 +22,9 @@ class app_function_application_insights_enabled(Check):
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Function {function.name} from subscription {subscription_name} ({subscription_id}) is not using Application Insights."
|
||||
|
||||
if function.enviroment_variables.get(
|
||||
if function.environment_variables.get(
|
||||
"APPINSIGHTS_INSTRUMENTATIONKEY", None
|
||||
) or function.enviroment_variables.get(
|
||||
) or function.environment_variables.get(
|
||||
"APPLICATIONINSIGHTS_CONNECTION_STRING", None
|
||||
):
|
||||
report.status = "PASS"
|
||||
|
||||
+3
-3
@@ -14,7 +14,7 @@ class app_function_latest_runtime_version(Check):
|
||||
subscription_id, subscription_id
|
||||
)
|
||||
for function in functions.values():
|
||||
if function.enviroment_variables is not None:
|
||||
if function.environment_variables is not None:
|
||||
report = Check_Report_Azure(
|
||||
metadata=self.metadata(), resource=function
|
||||
)
|
||||
@@ -23,13 +23,13 @@ class app_function_latest_runtime_version(Check):
|
||||
report.status_extended = f"Function {function.name} from subscription {subscription_name} ({subscription_id}) is using the latest runtime."
|
||||
|
||||
if (
|
||||
function.enviroment_variables.get(
|
||||
function.environment_variables.get(
|
||||
"FUNCTIONS_EXTENSION_VERSION", ""
|
||||
)
|
||||
!= "~4"
|
||||
):
|
||||
report.status = "FAIL"
|
||||
report.status_extended = f"Function {function.name} from subscription {subscription_name} ({subscription_id}) is not using the latest runtime. The current runtime is '{function.enviroment_variables.get('FUNCTIONS_EXTENSION_VERSION', '')}' and should be '~4'."
|
||||
report.status_extended = f"Function {function.name} from subscription {subscription_name} ({subscription_id}) is not using the latest runtime. The current runtime is '{function.environment_variables.get('FUNCTIONS_EXTENSION_VERSION', '')}' and should be '~4'."
|
||||
|
||||
findings.append(report)
|
||||
|
||||
|
||||
@@ -158,7 +158,7 @@ class App(AzureService):
|
||||
location=function.location,
|
||||
kind=function.kind,
|
||||
function_keys=function_keys,
|
||||
enviroment_variables=getattr(
|
||||
environment_variables=getattr(
|
||||
application_settings, "properties", None
|
||||
),
|
||||
identity=getattr(function, "identity", None),
|
||||
@@ -225,7 +225,7 @@ class App(AzureService):
|
||||
name=name,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
logger.warning(
|
||||
f"Error getting host keys for {name} in {resource_group}: {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
return None
|
||||
@@ -249,7 +249,7 @@ class App(AzureService):
|
||||
name=name,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
logger.warning(
|
||||
f"Error getting application settings for {name} in {resource_group}: {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
|
||||
)
|
||||
return None
|
||||
@@ -296,7 +296,7 @@ class FunctionApp:
|
||||
location: str
|
||||
kind: str
|
||||
function_keys: Optional[Dict[str, str]]
|
||||
enviroment_variables: Optional[Dict[str, str]]
|
||||
environment_variables: Optional[Dict[str, str]]
|
||||
identity: ManagedServiceIdentity
|
||||
public_access: bool
|
||||
vnet_subnet_id: str
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
[tool.towncrier]
|
||||
directory = "changelog.d"
|
||||
filename = "CHANGELOG.md"
|
||||
start_string = "<!-- changelog: release notes start -->\n"
|
||||
title_format = "## [{version}] ({name})"
|
||||
issue_format = "[(#{issue})](https://github.com/prowler-cloud/prowler/pull/{issue})"
|
||||
template = "../.github/towncrier/template.md.jinja"
|
||||
underlines = ["", "", ""]
|
||||
ignore = [".gitkeep", "README.md"]
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "added"
|
||||
name = "🚀 Added"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "changed"
|
||||
name = "🔄 Changed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "deprecated"
|
||||
name = "⚠️ Deprecated"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "removed"
|
||||
name = "❌ Removed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "fixed"
|
||||
name = "🐞 Fixed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "security"
|
||||
name = "🔐 Security"
|
||||
showcontent = true
|
||||
+1
-1
@@ -125,7 +125,7 @@ maintainers = [{name = "Prowler Engineering", email = "engineering@prowler.com"}
|
||||
name = "prowler"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10,<3.14"
|
||||
version = "5.33.0"
|
||||
version = "5.33.2"
|
||||
|
||||
[project.scripts]
|
||||
prowler = "prowler.__main__:prowler"
|
||||
|
||||
@@ -195,7 +195,8 @@ When all matching principals can target the same independent resource set, colle
|
||||
```cypher
|
||||
WITH aws, collect(DISTINCT path_principal) AS principal_paths
|
||||
MATCH path_target = (aws)--(target)
|
||||
WITH principal_paths + collect(DISTINCT path_target) AS paths
|
||||
WITH principal_paths, collect(DISTINCT path_target) AS target_paths
|
||||
WITH principal_paths + target_paths AS paths
|
||||
```
|
||||
|
||||
Statements that constrain a target are still checked via `HAS_RESOURCE` traversals (`res`, `res2`). See IAM-015 or EC2-001 in `aws.py`.
|
||||
@@ -419,6 +420,7 @@ Queries must run on both Neo4j and Amazon Neptune. Avoid these constructs:
|
||||
| `FOREACH` | `WITH` + `UNWIND` + `SET` |
|
||||
| Regex `=~` | `toLower()` + exact match, or `STARTS WITH` / `CONTAINS` |
|
||||
| `CALL () { UNION }` | Multi-label `OR` in `WHERE` (see pattern above) |
|
||||
| Carried value plus aggregate expression | Project the aggregate first: `WITH principal_paths, collect(...) AS target_paths`, then combine lists in the next `WITH` |
|
||||
| `any(x IN list ...)` | `size([x IN list WHERE pred]) > 0` |
|
||||
| `all(x IN list ...)` | `size([x IN list WHERE pred]) = size(list)` |
|
||||
| `none(x IN list ...)` | `size([x IN list WHERE pred]) = 0` |
|
||||
|
||||
+101
-185
@@ -6,7 +6,7 @@ description: >
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: prowler-cloud
|
||||
version: "1.0"
|
||||
version: "2.0"
|
||||
scope: [root, ui, api, sdk, mcp_server]
|
||||
auto_invoke:
|
||||
- "Add changelog entry for a PR or feature"
|
||||
@@ -16,70 +16,80 @@ metadata:
|
||||
allowed-tools: Read, Edit, Write, Glob, Grep, Bash
|
||||
---
|
||||
|
||||
## Changelog Locations
|
||||
## How changelog entries work: fragments
|
||||
|
||||
| Component | File | Version Prefix | Current Version |
|
||||
|-----------|------|----------------|-----------------|
|
||||
| UI | `ui/CHANGELOG.md` | None | 1.x.x |
|
||||
| API | `api/CHANGELOG.md` | None | 1.x.x |
|
||||
| MCP Server | `mcp_server/CHANGELOG.md` | None | 0.x.x |
|
||||
| SDK | `prowler/CHANGELOG.md` | None | 5.x.x |
|
||||
A PR never edits unreleased `CHANGELOG.md` content directly; use fragments instead. Released-block typo/correction fixes are the only direct-edit exception and are described below. For regular entries, add one small **fragment file** per entry under the component's `changelog.d/` directory. Fragments are compiled into the component's `CHANGELOG.md` at release time (deleting the consumed fragments), so concurrent PRs never conflict on the changelog.
|
||||
|
||||
## Format Rules (keepachangelog.com)
|
||||
| Component | Fragments directory | Compiled file |
|
||||
|-----------|---------------------|---------------|
|
||||
| UI | `ui/changelog.d/` | `ui/CHANGELOG.md` |
|
||||
| API | `api/changelog.d/` | `api/CHANGELOG.md` |
|
||||
| MCP Server | `mcp_server/changelog.d/` | `mcp_server/CHANGELOG.md` |
|
||||
| SDK | `prowler/changelog.d/` | `prowler/CHANGELOG.md` |
|
||||
|
||||
### Section Order (ALWAYS this order)
|
||||
"What's unreleased" = "what's in `changelog.d/`". The compiled `CHANGELOG.md` files contain only released versions.
|
||||
|
||||
```markdown
|
||||
## [X.Y.Z] (Prowler vA.B.C) OR (Prowler UNRELEASED)
|
||||
## Fragment filename
|
||||
|
||||
### Added
|
||||
### Changed
|
||||
### Deprecated
|
||||
### Removed
|
||||
### Fixed
|
||||
### Security
|
||||
```text
|
||||
<slug>.<type>.md
|
||||
```
|
||||
|
||||
### Emoji Prefixes (REQUIRED for ALL components)
|
||||
- `<slug>` is free-form (`[A-Za-z0-9][A-Za-z0-9._-]*`), chosen by the author, ideally descriptive of the change (e.g. `securityhub-delegated-admin`). The PR number is also a valid slug (e.g. `11259`) when it is already known; it is never required.
|
||||
- `<type>` maps 1:1 to the keepachangelog sections:
|
||||
|
||||
| Section | Emoji | Usage |
|
||||
|---------|-------|-------|
|
||||
| Added | `### 🚀 Added` | New features, checks, endpoints |
|
||||
| Changed | `### 🔄 Changed` | Modifications to existing functionality |
|
||||
| Deprecated | `### ⚠️ Deprecated` | Features marked for removal |
|
||||
| Removed | `### ❌ Removed` | Deleted features |
|
||||
| Fixed | `### 🐞 Fixed` | Bug fixes |
|
||||
| Security | `### 🔐 Security` | Security patches, CVE fixes |
|
||||
| `<type>` | Section | Usage |
|
||||
|----------|---------|-------|
|
||||
| `added` | `### 🚀 Added` | New features, checks, endpoints |
|
||||
| `changed` | `### 🔄 Changed` | Modifications to existing functionality |
|
||||
| `deprecated` | `### ⚠️ Deprecated` | Features marked for removal |
|
||||
| `removed` | `### ❌ Removed` | Deleted features |
|
||||
| `fixed` | `### 🐞 Fixed` | Bug fixes |
|
||||
| `security` | `### 🔐 Security` | Security patches, CVE fixes |
|
||||
|
||||
### Entry Format
|
||||
- A PR adds as many fragment files as entries it needs, freely mixing types: one file per entry. E.g. a PR touching Added, Changed and Fixed ships `kms-rotation-check.added.md` + `kms-metadata-cache.changed.md` + `kms-disabled-keys.fixed.md`, and all compile with the same PR link into their own sections.
|
||||
- Several entries of the SAME type: a different slug per entry (`kms-rotation-check.added.md`, `kms-rotation-docs.added.md`).
|
||||
- At least one fragment per touched component, same as the old one-entry-per-changelog rule.
|
||||
|
||||
```markdown
|
||||
### Added
|
||||
## Fragment content
|
||||
|
||||
- Existing entry one [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
- Existing entry two [(#YYYY)](https://github.com/prowler-cloud/prowler/pull/YYYY)
|
||||
- NEW ENTRY GOES HERE at the BOTTOM [(#ZZZZ)](https://github.com/prowler-cloud/prowler/pull/ZZZZ)
|
||||
The file contains ONLY the entry text, exactly as it should appear in the changelog, on a single line ending with a trailing newline:
|
||||
|
||||
### Changed
|
||||
|
||||
- Existing change [(#AAAA)](https://github.com/prowler-cloud/prowler/pull/AAAA)
|
||||
- NEW CHANGE ENTRY at BOTTOM [(#BBBB)](https://github.com/prowler-cloud/prowler/pull/BBBB)
|
||||
```bash
|
||||
echo '`securityhub_delegated_admin_enabled_all_regions` check for AWS provider, verifying that Security Hub has a delegated administrator, is active in all opted-in regions, and has organization auto-enable on' > prowler/changelog.d/securityhub-delegated-admin.added.md
|
||||
```
|
||||
|
||||
**Rules:**
|
||||
- **ADD NEW ENTRIES AT THE BOTTOM of each section** (before next section header or `---`)
|
||||
- **Blank line after section header** before first entry
|
||||
- **Blank line between sections**
|
||||
**Rules (same prose conventions as always):**
|
||||
|
||||
- **NEVER write the PR link in the text.** It is attached automatically at compile time (the compile workflow resolves the PR that added the fragment from git history). Writing `[(#NNNN)](...)` in a fragment produces a duplicated link.
|
||||
- No period at the end
|
||||
- Do NOT start with redundant verbs (the section header already provides the action)
|
||||
- Be specific: what changed, not why (that's in the PR)
|
||||
- Keep entries readable: use spaces around inline code and product names, and wrap endpoints, commands, errors, task names, and file paths in backticks
|
||||
- Avoid long run-on sentences; split complex changes into one concise result plus one concise context clause
|
||||
- One entry per PR (can link multiple PRs for related changes)
|
||||
- No period at the end
|
||||
- Do NOT start with redundant verbs (section header already provides the action)
|
||||
- **CRITICAL: Preserve section order** — when adding a new section to the UNRELEASED block, insert it in the correct position relative to existing sections (Added → Changed → Deprecated → Removed → Fixed → Security). Never append a new section at the top or bottom without checking order
|
||||
- **CRITICAL: ALWAYS link to the PR, NEVER to the issue.** Every entry MUST use `https://github.com/prowler-cloud/prowler/pull/N`. Linking to `/issues/N` is FORBIDDEN, even when the PR fixes an issue. The issue↔PR relationship belongs in the PR body (`Fixes #N`), not in the changelog. If a fix has no PR yet, do not add the entry until the PR exists.
|
||||
|
||||
### Semantic Versioning Rules
|
||||
### Good fragments
|
||||
|
||||
```text
|
||||
# ui/changelog.d/provider-search-bar.added.md
|
||||
Search bar when adding a provider
|
||||
|
||||
# api/changelog.d/scan-dispatch-race.fixed.md
|
||||
`POST /api/v1/scans` no longer intermittently fails with `Scan matching query does not exist`; scan dispatch now publishes the `scan-perform` Celery task after the transaction commits
|
||||
|
||||
# ui/changelog.d/node-24-bump.security.md
|
||||
Node.js from 20.x to 24.13.0 LTS, patching 8 CVEs
|
||||
```
|
||||
|
||||
### Bad fragments
|
||||
|
||||
```text
|
||||
Fixed bug. # Too vague, has period, redundant verb
|
||||
Add search bar # Redundant verb (the section already says "Added")
|
||||
Search bar [(#9634)](https://github.com/prowler-cloud/prowler/pull/9634) # NEVER include the PR link; it is added at compile time
|
||||
```
|
||||
|
||||
## Semantic Versioning Rules
|
||||
|
||||
Prowler follows [semver.org](https://semver.org/):
|
||||
|
||||
@@ -89,69 +99,26 @@ Prowler follows [semver.org](https://semver.org/):
|
||||
| New features (backwards compatible) | MINOR (x.**Y**.0) | 1.16.2 → 1.17.0 |
|
||||
| Breaking changes, removals | MAJOR (**X**.0.0) | 1.17.0 → 2.0.0 |
|
||||
|
||||
**CRITICAL:** `### ❌ Removed` entries MUST only appear in MAJOR version releases. Removing features is a breaking change.
|
||||
|
||||
### Released Versions Are Immutable
|
||||
|
||||
**NEVER modify already released versions.** Once a version is released (has a Prowler version tag like `v5.16.0`), its changelog section is frozen.
|
||||
|
||||
**Common issue:** A PR is created during release cycle X, includes a changelog entry, but merges after release. The entry is now in the wrong section.
|
||||
|
||||
```markdown
|
||||
## [1.16.0] (Prowler v5.16.0) ← RELEASED, DO NOT MODIFY
|
||||
|
||||
### Added
|
||||
- Feature from merged PR [(#9999)] ← WRONG! PR merged after release
|
||||
|
||||
## [1.17.0] (Prowler UNRELEASED) ← Move entry HERE
|
||||
```
|
||||
|
||||
**Fix:** Move the entry from the released version to the UNRELEASED section.
|
||||
|
||||
### Version Header Format
|
||||
|
||||
```markdown
|
||||
## [1.17.0] (Prowler UNRELEASED) # For unreleased changes
|
||||
## [1.16.0] (Prowler v5.16.0) # For released versions
|
||||
|
||||
--- # Horizontal rule between versions
|
||||
```
|
||||
|
||||
## Mandatory Changelog Preflight
|
||||
|
||||
Before editing any `CHANGELOG.md`, always inspect the active release boundary:
|
||||
|
||||
1. Read the UNRELEASED block plus the latest three released version blocks:
|
||||
```bash
|
||||
awk '/^## \[/{n++} n<=4 {print}' ui/CHANGELOG.md
|
||||
```
|
||||
2. Identify the **only writable block**: the block whose header contains `(Prowler UNRELEASED)`.
|
||||
3. Treat every block whose header contains `(Prowler vX.Y.Z)` as immutable. Do not add, move, reword, reorder, or deduplicate entries there.
|
||||
4. If your PR's entry appears in any of the latest three released blocks, remove it from the released block and add it to the correct section in the UNRELEASED block.
|
||||
5. If there is no UNRELEASED block at the top, stop and ask before editing.
|
||||
|
||||
**Do not trust the current topmost matching section name.** A released block can contain the same section heading (`### 🚀 Added`, `### 🔄 Changed`, etc.). Always anchor edits to the `Prowler UNRELEASED` version block first.
|
||||
**CRITICAL:** `removed` fragments MUST only ship in MAJOR version releases. Removing features is a breaking change.
|
||||
|
||||
## Mandatory Human Confirmation Gate
|
||||
|
||||
Before creating or editing any changelog file (`CHANGELOG.md`), the agent MUST stop and get explicit user confirmation. This applies even when the changelog gate is failing, the required edit seems obvious, or the user asked to "fix the changelog".
|
||||
Before creating or editing any changelog fragment or `CHANGELOG.md` file, the agent MUST stop and get explicit user confirmation. This applies even when the changelog gate is failing, the required file seems obvious, or the user asked to "fix the changelog".
|
||||
|
||||
Present the proposed changelog action before writing:
|
||||
Present the proposed action before writing:
|
||||
|
||||
1. Target file path.
|
||||
2. Target version block and section.
|
||||
3. Exact entry to add, move, remove, or rewrite.
|
||||
4. Reason the changelog is needed.
|
||||
1. Target fragment path (component, slug, type) or CHANGELOG.md edit.
|
||||
2. Exact entry text.
|
||||
3. Reason the changelog entry is needed.
|
||||
|
||||
Only proceed after an explicit approval such as "confirm", "approved", "sí", or equivalent. If the user rejects or does not answer, do not edit or create the changelog. Offer alternatives such as adding `no-changelog` when appropriate.
|
||||
Only proceed after an explicit approval such as "confirm", "approved", "sí", or equivalent. If the user rejects or does not answer, do not create or edit anything. Offer alternatives such as adding `no-changelog` when appropriate.
|
||||
|
||||
## Adding a Changelog Entry
|
||||
|
||||
### Step 1: Determine Affected Component(s)
|
||||
|
||||
```bash
|
||||
# Check which files changed
|
||||
git diff main...HEAD --name-only
|
||||
git diff master...HEAD --name-only | grep -E '^(ui|api|mcp_server|prowler)/' | cut -d/ -f1 | sort -u
|
||||
```
|
||||
|
||||
| Path Pattern | Component |
|
||||
@@ -160,114 +127,63 @@ git diff main...HEAD --name-only
|
||||
| `api/**` | API |
|
||||
| `mcp_server/**` | MCP Server |
|
||||
| `prowler/**` | SDK |
|
||||
| Multiple | Update ALL affected changelogs |
|
||||
| Root `uv.lock` / `pyproject.toml` | SDK (the gate requires a `prowler/changelog.d/` fragment) |
|
||||
| Multiple | One fragment per affected component |
|
||||
|
||||
### Step 2: Determine Change Type
|
||||
### Step 2: Create the fragment(s)
|
||||
|
||||
| Change | Section |
|
||||
|--------|---------|
|
||||
| New feature, check, endpoint | 🚀 Added |
|
||||
| Behavior change, refactor | 🔄 Changed |
|
||||
| Bug fix | 🐞 Fixed |
|
||||
| CVE patch, security improvement | 🔐 Security |
|
||||
| Feature removal | ❌ Removed |
|
||||
| Deprecation notice | ⚠️ Deprecated |
|
||||
|
||||
### Step 3: Add Entry at BOTTOM of Appropriate Section
|
||||
|
||||
**CRITICAL:** Add new entries at the BOTTOM of each section, NOT at the top.
|
||||
|
||||
**CRITICAL:** The link MUST point to the PR (`/pull/N`). Linking to `/issues/N` is FORBIDDEN. If the PR closes an issue, that mapping goes in the PR body via `Fixes #N` — never in the changelog entry.
|
||||
|
||||
```markdown
|
||||
## [1.17.0] (Prowler UNRELEASED)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Existing fix one [(#9997)](https://github.com/prowler-cloud/prowler/pull/9997)
|
||||
- Existing fix two [(#9998)](https://github.com/prowler-cloud/prowler/pull/9998)
|
||||
- Button alignment in dashboard header [(#9999)](https://github.com/prowler-cloud/prowler/pull/9999) ← NEW ENTRY AT BOTTOM
|
||||
|
||||
### 🔐 Security
|
||||
```bash
|
||||
echo 'Entry text describing the change' > <component>/changelog.d/<slug>.<type>.md
|
||||
```
|
||||
|
||||
This maintains chronological order within each section (oldest at top, newest at bottom).
|
||||
### Step 3: Check pending fragments
|
||||
|
||||
## Examples
|
||||
|
||||
### Good Entries
|
||||
|
||||
```markdown
|
||||
### 🚀 Added
|
||||
- Search bar when adding a provider [(#9634)](https://github.com/prowler-cloud/prowler/pull/9634)
|
||||
|
||||
### 🐞 Fixed
|
||||
- OCI update credentials form failing silently due to missing provider UID [(#9746)](https://github.com/prowler-cloud/prowler/pull/9746)
|
||||
|
||||
### 🔐 Security
|
||||
- Node.js from 20.x to 24.13.0 LTS, patching 8 CVEs [(#9797)](https://github.com/prowler-cloud/prowler/pull/9797)
|
||||
```
|
||||
|
||||
### Readable Technical Entries
|
||||
|
||||
```markdown
|
||||
# GOOD - Technical but readable
|
||||
### 🐞 Fixed
|
||||
- `POST /api/v1/scans` no longer intermittently fails with `Scan matching query does not exist`; scan dispatch now publishes the `scan-perform` Celery task after the transaction commits [(#11122)](https://github.com/prowler-cloud/prowler/pull/11122)
|
||||
- `entra_users_mfa_capable` no longer flags disabled guest users; Microsoft Graph is now the source of truth for `account_enabled` because EXO `Get-User` omits guest users [(#11002)](https://github.com/prowler-cloud/prowler/pull/11002)
|
||||
```
|
||||
|
||||
### Bad Entries
|
||||
|
||||
```markdown
|
||||
# BAD - Wrong section order (Fixed before Added)
|
||||
### 🐞 Fixed
|
||||
- Some bug fix [(#123)](...)
|
||||
|
||||
### 🚀 Added
|
||||
- Some new feature [(#456)](...)
|
||||
|
||||
- Fixed bug. # Too vague, has period
|
||||
- Added new feature for users # Missing PR link, redundant verb
|
||||
- Add search bar [(#123)] # Redundant verb (section already says "Added")
|
||||
- This PR adds a cool new thing (#123) # Wrong link format, conversational
|
||||
- Some bug fix [(#123)](https://github.com/prowler-cloud/prowler/issues/123) # FORBIDDEN: must link to /pull/N, never /issues/N
|
||||
- POST /api/v1/scanswas intermittently failing withScan matching query does not existin thescan-performworker (#11122) # Missing spaces/backticks, unreadable
|
||||
- entra_users_mfa_capable no longer flags disabled guest users by requesting accountEnabled and userType from Microsoft Graph via $select and using Graph as the source of truth for account_enabled (EXO Get-User does not return guest users) (#11002) # Run-on sentence, identifiers not formatted
|
||||
```bash
|
||||
ls prowler/changelog.d/ api/changelog.d/ ui/changelog.d/ mcp_server/changelog.d/
|
||||
```
|
||||
|
||||
## PR Changelog Gate
|
||||
|
||||
The `pr-check-changelog.yml` workflow enforces changelog entries:
|
||||
The `pr-check-changelog.yml` workflow enforces fragments:
|
||||
|
||||
1. **REQUIRED**: PRs touching `ui/`, `api/`, `mcp_server/`, or `prowler/` MUST update the corresponding changelog
|
||||
2. **SKIP**: Add `no-changelog` label to bypass (use sparingly for docs-only, CI-only changes)
|
||||
1. **REQUIRED**: PRs touching `ui/`, `api/`, `mcp_server/`, or `prowler/` MUST add (or fix) a fragment under the corresponding `changelog.d/`
|
||||
2. **VALIDATED**: added fragment filenames must match `<slug>.<type>.md` with a valid type
|
||||
3. **LINTED**: fragment content must NOT contain a hand-written PR link (`[(#N)](...)`); the gate fails if one is found because the link is attached automatically at compile time
|
||||
4. **SKIP**: Add `no-changelog` label to bypass (use sparingly for docs-only, CI-only changes)
|
||||
|
||||
## Commands
|
||||
## Release flow (compile)
|
||||
|
||||
```bash
|
||||
# Check which changelogs need updates based on changed files
|
||||
git diff main...HEAD --name-only | grep -E '^(ui|api|mcp_server|prowler)/' | cut -d/ -f1 | sort -u
|
||||
- At release time, the `compile-changelogs` workflow (manual dispatch: `prowler_version` + `target_branch`; per-component versions are auto-derived by mirroring the Prowler version — SDK mirrors it directly, UI is `1.<minor>.<patch>`, API is `1.<minor + 1>.<patch>`, and only the MCP Server derives from its pending fragment types — with optional explicit overrides or `skip`) resolves each fragment's PR from git history, runs the compiler per component, and opens a `chore(changelog): vX.Y.Z` PR (labeled `no-changelog` and `skip-sync`) that inserts the stamped `## [X.Y.Z] (Prowler vX.Y.Z)` block into each `CHANGELOG.md` and deletes the consumed fragments. A human reviews and squash-merges it. `prepare-release.yml` then extracts the stamped sections exactly as before.
|
||||
- **Minor release (X.Y.0):** compile on `master` and merge the compile PR BEFORE cutting the `v5.X` branch.
|
||||
- **Patch release (X.Y.Z):** fixes are backported to `v5.X` with their fragment files (conflict-free); compile on `v5.X` and merge its PR there. The same workflow run automatically opens a second forward-sync PR against master (labeled `no-changelog` and `skip-sync`) that inserts the same stamped block under master's marker and deletes the consumed fragments, so the next minor cannot re-release them; merge it right after. Fragments that only existed on `v5.X` are skipped with a notice. No manual git is involved.
|
||||
- Entries within a section are ordered by PR number ascending (approximately chronological). Do not fight this ordering.
|
||||
|
||||
# View current UNRELEASED section
|
||||
head -50 ui/CHANGELOG.md
|
||||
head -50 api/CHANGELOG.md
|
||||
head -50 mcp_server/CHANGELOG.md
|
||||
head -50 prowler/CHANGELOG.md
|
||||
```
|
||||
## Fixing an already-released entry
|
||||
|
||||
## Migration Note
|
||||
Released version blocks in `CHANGELOG.md` are otherwise immutable, but typo/correction fixes to already-released entries are the one case where a PR edits `CHANGELOG.md` directly: make the edit and add the `no-changelog` label.
|
||||
|
||||
**API, MCP Server, and SDK changelogs currently lack emojis.** When editing these files, add emoji prefixes to section headers as you update them:
|
||||
If a PR's entry shipped in the wrong released block (e.g. the PR merged after its release was cut), move the entry back to a fragment: delete it from the released block and recreate it as `<component>/changelog.d/<PR>.<type>.md` (label the PR `no-changelog` since it edits `CHANGELOG.md`).
|
||||
|
||||
## Compiled CHANGELOG.md format (for reference)
|
||||
|
||||
The compiler renders, per release, into each `CHANGELOG.md` right under the `<!-- changelog: release notes start -->` marker (never remove that marker):
|
||||
|
||||
```markdown
|
||||
# Before (legacy)
|
||||
### Added
|
||||
## [X.Y.Z] (Prowler vA.B.C)
|
||||
|
||||
# After (standardized)
|
||||
### 🚀 Added
|
||||
|
||||
- Entry text [(#NNNN)](https://github.com/prowler-cloud/prowler/pull/NNNN)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- Fix entry [(#NNNN)](https://github.com/prowler-cloud/prowler/pull/NNNN)
|
||||
|
||||
---
|
||||
```
|
||||
|
||||
Section order is always: Added → Changed → Deprecated → Removed → Fixed → Security. `X.Y.Z` is the COMPONENT version; `A.B.C` is the Prowler release version. Every entry ends with its PR link; linking to `/issues/N` is forbidden (the issue↔PR mapping belongs in the PR body via `Fixes #N`).
|
||||
|
||||
## Resources
|
||||
|
||||
- **Templates**: See [assets/](assets/) for entry templates
|
||||
|
||||
@@ -1,101 +1,73 @@
|
||||
# Changelog Entry Templates
|
||||
# Changelog Fragment Templates
|
||||
|
||||
## Entry Placement Rule
|
||||
## Fragment basics
|
||||
|
||||
**CRITICAL:** Always add new entries at the **BOTTOM** of each section (before the next section header or `---`).
|
||||
One fragment file per entry, under the component's `changelog.d/`:
|
||||
|
||||
This maintains chronological order: oldest entries at top, newest at bottom.
|
||||
|
||||
## Section Headers
|
||||
|
||||
```markdown
|
||||
### 🚀 Added
|
||||
### 🔄 Changed
|
||||
### ⚠️ Deprecated
|
||||
### ❌ Removed
|
||||
### 🐞 Fixed
|
||||
### 🔐 Security
|
||||
```text
|
||||
<component>/changelog.d/<slug>.<type>.md
|
||||
```
|
||||
|
||||
## Entry Patterns
|
||||
- `<slug>`: free-form, descriptive (`[A-Za-z0-9][A-Za-z0-9._-]*`)
|
||||
- `<type>`: `added`, `changed`, `deprecated`, `removed`, `fixed`, `security`
|
||||
- Content: a single line with the entry text. **No PR link** (attached automatically at compile time) and no trailing period.
|
||||
|
||||
> **Note:** Section headers already provide the verb. Entries describe WHAT, not the action.
|
||||
>
|
||||
> **Link target rule:** Every entry MUST link to the PR (`https://github.com/prowler-cloud/prowler/pull/N`). Linking to `/issues/N` is FORBIDDEN — even when the PR fixes an issue. The issue↔PR mapping belongs in the PR body (`Fixes #N`), not here.
|
||||
> **Note:** The section header already provides the verb. Entries describe WHAT, not the action.
|
||||
|
||||
### Feature Addition (🚀 Added)
|
||||
```markdown
|
||||
- Search bar when adding a provider [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
- `{check_id}` check for {provider} provider [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
- `/api/v1/{endpoint}` endpoint to {description} [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
## Content Patterns by Type
|
||||
|
||||
### Feature Addition (`.added.md`)
|
||||
|
||||
```text
|
||||
Search bar when adding a provider
|
||||
`{check_id}` check for {provider} provider
|
||||
`/api/v1/{endpoint}` endpoint to {description}
|
||||
```
|
||||
|
||||
### Behavior Change (🔄 Changed)
|
||||
```markdown
|
||||
- Lighthouse AI MCP tool filtering from blacklist to whitelist approach [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
- {package} from {old} to {new} [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
### Behavior Change (`.changed.md`)
|
||||
|
||||
```text
|
||||
Lighthouse AI MCP tool filtering from blacklist to whitelist approach
|
||||
{package} from {old} to {new}
|
||||
```
|
||||
|
||||
### Bug Fix (🐞 Fixed)
|
||||
```markdown
|
||||
- OCI update credentials form failing silently due to missing provider UID [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
- {What was broken} in {component} [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
### Bug Fix (`.fixed.md`)
|
||||
|
||||
```text
|
||||
OCI update credentials form failing silently due to missing provider UID
|
||||
{What was broken} in {component}
|
||||
```
|
||||
|
||||
> When a PR fixes a reported issue, the link still goes to the PR (`/pull/N`), never the issue (`/issues/N`). Reference the issue from the PR body with `Fixes #N`.
|
||||
### Security Patch (`.security.md`)
|
||||
|
||||
### Security Patch (🔐 Security)
|
||||
```markdown
|
||||
- Node.js from 20.x to 24.13.0 LTS, patching 8 CVEs [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
- {package} to version {version} (CVE-XXXX-XXXXX) [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
```text
|
||||
Node.js from 20.x to 24.13.0 LTS, patching 8 CVEs
|
||||
{package} to version {version} (CVE-XXXX-XXXXX)
|
||||
```
|
||||
|
||||
### Removal (❌ Removed)
|
||||
```markdown
|
||||
- Deprecated {feature} from {location} [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
### Removal (`.removed.md`)
|
||||
|
||||
```text
|
||||
Deprecated {feature} from {location}
|
||||
```
|
||||
|
||||
## Version Header Templates
|
||||
## Full Examples
|
||||
|
||||
### Unreleased
|
||||
```markdown
|
||||
## [X.Y.Z] (Prowler UNRELEASED)
|
||||
```bash
|
||||
echo 'Search bar when adding a provider' > ui/changelog.d/provider-search-bar.added.md
|
||||
|
||||
echo '`kms_key_rotation_max_90_days` check for GCP provider, verifying KMS customer-managed keys are rotated every 90 days or less' > prowler/changelog.d/kms-rotation-90d.added.md
|
||||
|
||||
echo 'OCI update credentials form failing silently due to missing provider UID' > ui/changelog.d/oci-credentials-form.fixed.md
|
||||
```
|
||||
|
||||
### Released
|
||||
```markdown
|
||||
## [X.Y.Z] (Prowler vA.B.C)
|
||||
Several entries in one PR → one file per entry, freely mixing types (different slugs when the type repeats):
|
||||
|
||||
---
|
||||
```text
|
||||
prowler/changelog.d/kms-rotation-check.added.md
|
||||
prowler/changelog.d/kms-rotation-docs.added.md
|
||||
prowler/changelog.d/kms-metadata-cache.changed.md
|
||||
prowler/changelog.d/kms-disabled-keys.fixed.md
|
||||
```
|
||||
|
||||
## Full Entry Example
|
||||
|
||||
```markdown
|
||||
## [1.17.0] (Prowler UNRELEASED)
|
||||
|
||||
### 🚀 Added
|
||||
|
||||
- Search bar when adding a provider [(#9634)](https://github.com/prowler-cloud/prowler/pull/9634)
|
||||
- New findings table UI with new design system components [(#9699)](https://github.com/prowler-cloud/prowler/pull/9699)
|
||||
- YOUR NEW ENTRY GOES HERE AT BOTTOM [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
|
||||
### 🔄 Changed
|
||||
|
||||
- Lighthouse AI MCP tool filtering from blacklist to whitelist approach [(#9802)](https://github.com/prowler-cloud/prowler/pull/9802)
|
||||
- YOUR NEW CHANGE GOES HERE AT BOTTOM [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
|
||||
### 🐞 Fixed
|
||||
|
||||
- OCI update credentials form failing silently due to missing provider UID [(#9746)](https://github.com/prowler-cloud/prowler/pull/9746)
|
||||
- YOUR NEW FIX GOES HERE AT BOTTOM [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- Node.js from 20.x to 24.13.0 LTS, patching 8 CVEs [(#9797)](https://github.com/prowler-cloud/prowler/pull/9797)
|
||||
- YOUR NEW SECURITY FIX GOES HERE AT BOTTOM [(#XXXX)](https://github.com/prowler-cloud/prowler/pull/XXXX)
|
||||
|
||||
---
|
||||
```
|
||||
|
||||
> **Remember:** Each new entry is added at the BOTTOM of its section to maintain chronological order.
|
||||
> **Remember:** never include the PR link in the fragment text; the compile step resolves and appends it automatically.
|
||||
|
||||
@@ -31,7 +31,8 @@ Use this skill whenever you are:
|
||||
|
||||
- PR template: `.github/pull_request_template.md`
|
||||
- PR title validation: `.github/workflows/conventional-commit.yml`
|
||||
- Changelog gate: `.github/workflows/pr-check-changelog.yml`
|
||||
- Changelog gate: `.github/workflows/pr-check-changelog.yml` (requires a fragment under `<component>/changelog.d/`)
|
||||
- Changelog compile (release time): `.github/workflows/compile-changelogs.yml`
|
||||
- Conflict markers check: `.github/workflows/pr-conflict-checker.yml`
|
||||
- Secret scanning: `.github/workflows/find-secrets.yml`
|
||||
- Auto labels: `.github/workflows/labeler.yml` and `.github/labeler.yml`
|
||||
@@ -42,7 +43,7 @@ Use this skill whenever you are:
|
||||
1. Identify which workflow/job is failing (name + file under `.github/workflows/`).
|
||||
2. Check path filters: is the workflow supposed to run for your changed files?
|
||||
3. If it's a title check: verify PR title matches Conventional Commits.
|
||||
4. If it's changelog: verify the right `CHANGELOG.md` is updated OR apply `no-changelog` label.
|
||||
4. If it's changelog: verify a valid fragment exists under the right `<component>/changelog.d/` OR apply `no-changelog` label.
|
||||
5. If it's conflict checker: remove `<<<<<<<`, `=======`, `>>>>>>>` markers.
|
||||
6. If it's secrets (TruffleHog): see section below.
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ allowed-tools: Read, Edit, Write, Glob, Grep, Bash, WebFetch, WebSearch, Task
|
||||
| JSON Valid | `python3 -m json.tool file.json` | No syntax errors |
|
||||
| All Checks Exist | Run validation script | 0 missing checks |
|
||||
| No Duplicate IDs | Run validation script | 0 duplicate requirement IDs |
|
||||
| CHANGELOG Entry | Manual review | Present under correct version |
|
||||
| Changelog fragment | Manual review | Fragment present under `prowler/changelog.d/` |
|
||||
| Dashboard File | Compare with existing | Follows established pattern |
|
||||
| Framework Metadata | Manual review | All required fields populated |
|
||||
|
||||
@@ -63,8 +63,8 @@ JSON Valid?
|
||||
Duplicate Requirement IDs?
|
||||
├── Yes → FAIL: Fix duplicate IDs
|
||||
└── No ↓
|
||||
CHANGELOG Entry Present?
|
||||
├── No → REQUEST CHANGES: Add CHANGELOG entry
|
||||
Changelog Fragment Present?
|
||||
├── No → REQUEST CHANGES: Add changelog fragment
|
||||
└── Yes ↓
|
||||
Dashboard File Follows Pattern?
|
||||
├── No → REQUEST CHANGES: Fix dashboard pattern
|
||||
@@ -124,7 +124,7 @@ Compliance frameworks are JSON files in: `prowler/compliance/{provider}/{framewo
|
||||
| Empty Checks for Automated | AssessmentStatus is Automated but Checks is empty | Add checks or change to Manual |
|
||||
| Wrong file location | Framework not in `prowler/compliance/{provider}/` | Move to correct directory |
|
||||
| Missing dashboard file | No corresponding `dashboard/compliance/{framework}.py` | Create dashboard file following pattern |
|
||||
| CHANGELOG missing | Not under correct version section | Add entry to prowler/CHANGELOG.md |
|
||||
| Changelog fragment missing | No fragment file in the PR diff | Add a fragment under prowler/changelog.d/ |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
|-----------|----------|
|
||||
| Compliance JSON | `prowler/compliance/{provider}/{framework}.json` |
|
||||
| Dashboard | `dashboard/compliance/{framework}_{provider}.py` |
|
||||
| CHANGELOG | `prowler/CHANGELOG.md` |
|
||||
| Changelog fragment | `prowler/changelog.d/` |
|
||||
| Checks | `prowler/providers/{provider}/services/{service}/{check}/` |
|
||||
|
||||
## Validation Script
|
||||
@@ -40,7 +40,7 @@ When completing a compliance framework review, use this summary format:
|
||||
| JSON Valid | PASS/FAIL |
|
||||
| All Checks Exist | PASS/FAIL (N missing) |
|
||||
| No Duplicate IDs | PASS/FAIL |
|
||||
| CHANGELOG Entry | PASS/FAIL |
|
||||
| Changelog fragment | PASS/FAIL |
|
||||
| Dashboard File | PASS/FAIL |
|
||||
|
||||
### Statistics
|
||||
|
||||
+10
-10
@@ -56,7 +56,7 @@ allowed-tools: Read, Edit, Write, Glob, Grep, Bash, WebFetch, WebSearch, Task
|
||||
- [ ] Review if code is being documented following https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings
|
||||
- [ ] Review if backport is needed.
|
||||
- [ ] Review if is needed to change the Readme.md
|
||||
- [ ] Ensure new entries are added to CHANGELOG.md, if applicable.
|
||||
- [ ] Ensure a changelog fragment is added under <component>/changelog.d/, if applicable.
|
||||
|
||||
#### SDK/CLI
|
||||
- Are there new checks included in this PR? Yes / No
|
||||
@@ -67,7 +67,7 @@ allowed-tools: Read, Edit, Write, Glob, Grep, Bash, WebFetch, WebSearch, Task
|
||||
- [ ] Screenshots/Video - Mobile (X < 640px)
|
||||
- [ ] Screenshots/Video - Tablet (640px > X < 1024px)
|
||||
- [ ] Screenshots/Video - Desktop (X > 1024px)
|
||||
- [ ] Ensure new entries are added to ui/CHANGELOG.md
|
||||
- [ ] Ensure a changelog fragment is added under ui/changelog.d/
|
||||
|
||||
#### API (if applicable)
|
||||
- [ ] All issue/task requirements work as expected on the API
|
||||
@@ -77,7 +77,7 @@ allowed-tools: Read, Edit, Write, Glob, Grep, Bash, WebFetch, WebSearch, Task
|
||||
- [ ] Any other relevant evidence of the implementation (if applicable)
|
||||
- [ ] Verify if API specs need to be regenerated.
|
||||
- [ ] Check if version updates are required.
|
||||
- [ ] Ensure new entries are added to api/CHANGELOG.md
|
||||
- [ ] Ensure a changelog fragment is added under api/changelog.d/
|
||||
|
||||
### License
|
||||
|
||||
@@ -86,12 +86,12 @@ By submitting this pull request, I confirm that my contribution is made under th
|
||||
|
||||
## Component-Specific Rules
|
||||
|
||||
| Component | CHANGELOG | Extra Checks |
|
||||
|-----------|-----------|--------------|
|
||||
| SDK | `prowler/CHANGELOG.md` | New checks → permissions update? |
|
||||
| API | `api/CHANGELOG.md` | API specs, version bump, endpoint output, EXPLAIN ANALYZE, performance |
|
||||
| UI | `ui/CHANGELOG.md` | Screenshots for Mobile/Tablet/Desktop |
|
||||
| MCP | `mcp_server/CHANGELOG.md` | N/A |
|
||||
| Component | Changelog fragment | Extra Checks |
|
||||
|-----------|--------------------|--------------|
|
||||
| SDK | `prowler/changelog.d/` | New checks → permissions update? |
|
||||
| API | `api/changelog.d/` | API specs, version bump, endpoint output, EXPLAIN ANALYZE, performance |
|
||||
| UI | `ui/changelog.d/` | Screenshots for Mobile/Tablet/Desktop |
|
||||
| MCP | `mcp_server/changelog.d/` | N/A |
|
||||
|
||||
## Commands
|
||||
|
||||
@@ -128,7 +128,7 @@ Follow conventional commits:
|
||||
|
||||
1. ✅ All tests pass locally
|
||||
2. ✅ Linting passes (`make lint` or component-specific)
|
||||
3. ✅ CHANGELOG updated (if applicable)
|
||||
3. ✅ Changelog fragment added (if applicable)
|
||||
4. ✅ Branch is up to date with main
|
||||
5. ✅ Commits are clean and descriptive
|
||||
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
import importlib.util
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MODULE_PATH = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ ".github"
|
||||
/ "scripts"
|
||||
/ "changelog_attribution.py"
|
||||
)
|
||||
SPEC = importlib.util.spec_from_file_location("changelog_attribution", MODULE_PATH)
|
||||
changelog_attribution = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC.loader is not None
|
||||
SPEC.loader.exec_module(changelog_attribution)
|
||||
|
||||
|
||||
def make_component(tmp_path):
|
||||
component = tmp_path / "prowler"
|
||||
fragments_dir = component / "changelog.d"
|
||||
fragments_dir.mkdir(parents=True)
|
||||
return component, fragments_dir
|
||||
|
||||
|
||||
def fake_git_mv(*args):
|
||||
if args[0] != "mv":
|
||||
raise AssertionError(f"Unexpected git command: {args}")
|
||||
shutil.move(args[1], args[2])
|
||||
return ""
|
||||
|
||||
|
||||
def fail_git(*args):
|
||||
raise AssertionError(f"Unexpected git command: {args}")
|
||||
|
||||
|
||||
def fail_api(*args):
|
||||
raise AssertionError(f"Unexpected GitHub API call: {args}")
|
||||
|
||||
|
||||
def run_main(monkeypatch, component, *extra_args):
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["changelog_attribution.py", str(component), *extra_args],
|
||||
)
|
||||
return changelog_attribution.main()
|
||||
|
||||
|
||||
class TestChangelogAttribution:
|
||||
def test_renames_slug_fragment_to_resolved_pr_number(self, tmp_path, monkeypatch):
|
||||
component, fragments_dir = make_component(tmp_path)
|
||||
fragment = fragments_dir / "add-workflow.fixed.md"
|
||||
fragment.write_text("Fix changelog workflow.\n")
|
||||
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution, "find_adding_commit", lambda path: "abc123"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution, "pr_from_api", lambda repo, sha: 11572
|
||||
)
|
||||
monkeypatch.setattr(changelog_attribution, "git", fake_git_mv)
|
||||
|
||||
assert run_main(monkeypatch, component) == 0
|
||||
assert not fragment.exists()
|
||||
assert (fragments_dir / "11572.fixed.md").read_text() == (
|
||||
"Fix changelog workflow.\n"
|
||||
)
|
||||
|
||||
def test_appends_counter_when_pr_fragment_already_exists(
|
||||
self, tmp_path, monkeypatch
|
||||
):
|
||||
component, fragments_dir = make_component(tmp_path)
|
||||
(fragments_dir / "11572.fixed.md").write_text("Existing fix.\n")
|
||||
fragment = fragments_dir / "another-fix.fixed.md"
|
||||
fragment.write_text("Another fix.\n")
|
||||
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution, "find_adding_commit", lambda path: "abc123"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution, "pr_from_api", lambda repo, sha: 11572
|
||||
)
|
||||
monkeypatch.setattr(changelog_attribution, "git", fake_git_mv)
|
||||
|
||||
assert run_main(monkeypatch, component) == 0
|
||||
assert (fragments_dir / "11572.fixed.md").read_text() == "Existing fix.\n"
|
||||
assert (fragments_dir / "11572.fixed.1.md").read_text() == "Another fix.\n"
|
||||
|
||||
def test_uses_subject_fallback_when_api_is_disabled(self, tmp_path, monkeypatch):
|
||||
component, fragments_dir = make_component(tmp_path)
|
||||
fragment = fragments_dir / "fallback.added.md"
|
||||
fragment.write_text("Add fallback behavior.\n")
|
||||
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution, "find_adding_commit", lambda path: "abc123"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution,
|
||||
"pr_from_api",
|
||||
fail_api,
|
||||
)
|
||||
monkeypatch.setattr(changelog_attribution, "pr_from_subject", lambda sha: 42)
|
||||
monkeypatch.setattr(changelog_attribution, "git", fake_git_mv)
|
||||
|
||||
assert run_main(monkeypatch, component, "--no-api") == 0
|
||||
assert not fragment.exists()
|
||||
assert (fragments_dir / "42.added.md").read_text() == "Add fallback behavior.\n"
|
||||
|
||||
def test_renames_unresolved_fragment_to_orphan(self, tmp_path, monkeypatch, capsys):
|
||||
component, fragments_dir = make_component(tmp_path)
|
||||
fragment = fragments_dir / "manual.changed.md"
|
||||
fragment.write_text("Change manual entry.\n")
|
||||
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution, "find_adding_commit", lambda path: None
|
||||
)
|
||||
monkeypatch.setattr(changelog_attribution, "git", fake_git_mv)
|
||||
|
||||
assert run_main(monkeypatch, component) == 0
|
||||
assert not fragment.exists()
|
||||
assert (fragments_dir / "+manual.changed.md").read_text() == (
|
||||
"Change manual entry.\n"
|
||||
)
|
||||
assert "Could not resolve a PR" in capsys.readouterr().out
|
||||
|
||||
def test_rejects_malformed_fragment_names(self, tmp_path, monkeypatch, capsys):
|
||||
component, fragments_dir = make_component(tmp_path)
|
||||
fragment = fragments_dir / "bad.bugfix.md"
|
||||
fragment.write_text("Invalid type.\n")
|
||||
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution,
|
||||
"git",
|
||||
fail_git,
|
||||
)
|
||||
|
||||
assert run_main(monkeypatch, component) == 1
|
||||
assert fragment.exists()
|
||||
assert "Malformed fragment filename" in capsys.readouterr().out
|
||||
|
||||
def test_rejects_malformed_names_before_renaming_any_fragment(
|
||||
self, tmp_path, monkeypatch, capsys
|
||||
):
|
||||
component, fragments_dir = make_component(tmp_path)
|
||||
valid_fragment = fragments_dir / "valid.fixed.md"
|
||||
malformed_fragment = fragments_dir / "bad.bugfix.md"
|
||||
valid_fragment.write_text("Valid fix.\n")
|
||||
malformed_fragment.write_text("Invalid type.\n")
|
||||
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution, "find_adding_commit", lambda path: "abc123"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution, "pr_from_api", lambda repo, sha: 11572
|
||||
)
|
||||
monkeypatch.setattr(changelog_attribution, "git", fail_git)
|
||||
|
||||
assert run_main(monkeypatch, component) == 1
|
||||
assert valid_fragment.read_text() == "Valid fix.\n"
|
||||
assert malformed_fragment.read_text() == "Invalid type.\n"
|
||||
assert not (fragments_dir / "11572.fixed.md").exists()
|
||||
assert "Malformed fragment filename" in capsys.readouterr().out
|
||||
|
||||
def test_skips_fragments_that_already_start_with_pr_number(
|
||||
self, tmp_path, monkeypatch
|
||||
):
|
||||
component, fragments_dir = make_component(tmp_path)
|
||||
fragment = fragments_dir / "11572.added.md"
|
||||
fragment.write_text("Already attributed.\n")
|
||||
|
||||
monkeypatch.setattr(
|
||||
changelog_attribution,
|
||||
"git",
|
||||
fail_git,
|
||||
)
|
||||
|
||||
assert run_main(monkeypatch, component) == 0
|
||||
assert fragment.read_text() == "Already attributed.\n"
|
||||
@@ -0,0 +1,379 @@
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
EXPECTED_CHANGELOG_TYPE_ORDER = [
|
||||
"added",
|
||||
"changed",
|
||||
"deprecated",
|
||||
"removed",
|
||||
"fixed",
|
||||
"security",
|
||||
]
|
||||
EXPECTED_CHANGELOG_SECTION_NAMES = [
|
||||
"🚀 Added",
|
||||
"🔄 Changed",
|
||||
"⚠️ Deprecated",
|
||||
"❌ Removed",
|
||||
"🐞 Fixed",
|
||||
"🔐 Security",
|
||||
]
|
||||
COMPONENTS = ["prowler", "api", "ui", "mcp_server"]
|
||||
|
||||
|
||||
def read_workflow(name):
|
||||
return (REPO_ROOT / ".github" / "workflows" / name).read_text()
|
||||
|
||||
|
||||
def render_towncrier(tmp_path, type_definitions, fragments):
|
||||
pytest.importorskip("towncrier")
|
||||
|
||||
fragments_dir = tmp_path / "changelog.d"
|
||||
fragments_dir.mkdir()
|
||||
(tmp_path / "CHANGELOG.md").write_text(
|
||||
"# Changelog\n\n<!-- changelog: release notes start -->\n"
|
||||
)
|
||||
for filename, content in fragments.items():
|
||||
(fragments_dir / filename).write_text(f"{content}\n")
|
||||
|
||||
type_config = "\n".join(
|
||||
"\n".join(
|
||||
[
|
||||
"[[tool.towncrier.type]]",
|
||||
f'directory = "{directory}"',
|
||||
f'name = "{name}"',
|
||||
"showcontent = true",
|
||||
]
|
||||
)
|
||||
for directory, name in type_definitions
|
||||
)
|
||||
config = "\n".join(
|
||||
[
|
||||
"[tool.towncrier]",
|
||||
'directory = "changelog.d"',
|
||||
'filename = "CHANGELOG.md"',
|
||||
'start_string = "<!-- changelog: release notes start -->\\n"',
|
||||
f'template = "{(REPO_ROOT / ".github/towncrier/template.md.jinja").as_posix()}"',
|
||||
'title_format = "## [{version}] ({name})"',
|
||||
'issue_format = "[(#{issue})](https://example.com/pull/{issue})"',
|
||||
'underlines = ["", "", ""]',
|
||||
type_config,
|
||||
]
|
||||
)
|
||||
config_path = tmp_path / "towncrier.toml"
|
||||
config_path.write_text(config)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"towncrier",
|
||||
"build",
|
||||
"--config",
|
||||
str(config_path),
|
||||
"--version",
|
||||
"0.1.0",
|
||||
"--name",
|
||||
"Test",
|
||||
"--draft",
|
||||
],
|
||||
cwd=tmp_path,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.split("## [0.1.0] (Test)", 1)[1]
|
||||
|
||||
|
||||
def test_towncrier_template_uses_configured_changelog_section_order(tmp_path):
|
||||
template = (REPO_ROOT / ".github/towncrier/template.md.jinja").read_text()
|
||||
|
||||
assert "{% set category_order = definitions.keys() %}" in template
|
||||
assert (
|
||||
"{% for category in category_order if category in sections[section] %}"
|
||||
in template
|
||||
)
|
||||
assert "definitions.items()" not in template
|
||||
assert (
|
||||
'["added", "changed", "deprecated", "removed", "fixed", "security"]'
|
||||
not in template
|
||||
)
|
||||
|
||||
output = render_towncrier(
|
||||
tmp_path,
|
||||
[("fixed", "Fixed"), ("added", "Added"), ("custom", "Custom")],
|
||||
{
|
||||
"1.added.md": "Entry one",
|
||||
"2.added.md": "Entry two",
|
||||
"3.fixed.md": "Fix entry",
|
||||
"4.custom.md": "Custom entry",
|
||||
},
|
||||
)
|
||||
|
||||
headings = re.findall(r"^### (.+)$", output, re.MULTILINE)
|
||||
assert headings == ["Fixed", "Added", "Custom"]
|
||||
assert (
|
||||
"- Entry one [(#1)](https://example.com/pull/1)\n"
|
||||
"- Entry two [(#2)](https://example.com/pull/2)"
|
||||
) in output
|
||||
assert (
|
||||
"- Entry one [(#1)](https://example.com/pull/1)\n\n"
|
||||
"- Entry two [(#2)](https://example.com/pull/2)"
|
||||
) not in output
|
||||
|
||||
|
||||
def test_component_towncrier_configs_keep_changelog_section_order():
|
||||
for component in COMPONENTS:
|
||||
config = tomllib.loads((REPO_ROOT / component / "towncrier.toml").read_text())
|
||||
type_definitions = config["tool"]["towncrier"]["type"]
|
||||
|
||||
assert [item["directory"] for item in type_definitions] == (
|
||||
EXPECTED_CHANGELOG_TYPE_ORDER
|
||||
)
|
||||
assert [item["name"] for item in type_definitions] == (
|
||||
EXPECTED_CHANGELOG_SECTION_NAMES
|
||||
)
|
||||
|
||||
|
||||
def test_pull_request_template_links_to_all_fragment_locations():
|
||||
template = (REPO_ROOT / ".github/pull_request_template.md").read_text()
|
||||
|
||||
assert "[Prowler Community Slack](https://goto.prowler.com/slack)" in template
|
||||
assert "[Prowler Community Slack](goto.prowler.com/slack)" not in template
|
||||
for component in COMPONENTS:
|
||||
assert f"{component}/changelog.d/" in template
|
||||
|
||||
|
||||
def test_changelog_gate_rejects_direct_changelog_edits():
|
||||
workflow = read_workflow("pr-check-changelog.yml")
|
||||
has_update = re.search(
|
||||
r"(?ms)^\s*has_changelog_update\(\) \{\n(?P<body>.*?)^\s*\}\n",
|
||||
workflow,
|
||||
)
|
||||
|
||||
assert has_update is not None
|
||||
assert "CHANGELOG.md" not in has_update.group("body")
|
||||
assert "handwritten_changelogs" in workflow
|
||||
assert "Direct CHANGELOG.md edits are not allowed" in workflow
|
||||
assert "direct CHANGELOG.md edit" not in workflow
|
||||
|
||||
|
||||
def test_changelog_gate_tests_compile_workflow_changes():
|
||||
workflow = read_workflow("pr-check-changelog.yml")
|
||||
|
||||
assert ".github/workflows/compile-changelogs.yml" in workflow
|
||||
|
||||
|
||||
def test_changelog_attribution_tests_use_pinned_python():
|
||||
workflow = read_workflow("pr-check-changelog.yml")
|
||||
match = re.search(
|
||||
r"(?ms)^ test-changelog-attribution:\n(?P<body>.*?)^ check-changelog:",
|
||||
workflow,
|
||||
)
|
||||
|
||||
assert match is not None
|
||||
job = match.group("body")
|
||||
assert "uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c" in job
|
||||
assert "python-version: '3.12'" in job
|
||||
assert "python3 -m pip install" in job
|
||||
assert "python3 -m pytest tests/github" in job
|
||||
assert "objects.githubusercontent.com:443" in job
|
||||
|
||||
|
||||
def test_changelog_gate_rejects_common_manual_pr_link_forms():
|
||||
workflow = read_workflow("pr-check-changelog.yml")
|
||||
|
||||
assert "manual_pr_link_re=" in workflow
|
||||
assert r"\[\(#[0-9]+\)\]" in workflow
|
||||
assert r"\[#[0-9]+\]\(" in workflow
|
||||
assert r"\(#[0-9]+\)" in workflow
|
||||
assert r"github\.com/[^[:space:]/]+/[^[:space:]/]+/(pull|issues)/[0-9]+" in workflow
|
||||
|
||||
|
||||
def test_changelog_gate_derives_fragment_paths_from_monitored_folders():
|
||||
workflow = read_workflow("pr-check-changelog.yml")
|
||||
|
||||
assert "folder_alt=$(echo \"$MONITORED_FOLDERS\" | tr ' ' '|')" in workflow
|
||||
assert "^(api|ui|prowler|mcp_server)/changelog\\.d/" not in workflow
|
||||
|
||||
|
||||
def test_changelog_gate_lints_renamed_fragments():
|
||||
workflow = read_workflow("pr-check-changelog.yml")
|
||||
|
||||
assert "STEPS_CHANGED_FILES_OUTPUTS_RENAMED_FILES" in workflow
|
||||
assert "${{ steps.changed-files.outputs.renamed_files }}" in workflow
|
||||
assert "added_or_renamed=" in workflow
|
||||
assert "added_modified_or_renamed=" in workflow
|
||||
assert (
|
||||
'echo "$added_or_renamed" | grep -E "^(${folder_alt})/changelog\\.d/"'
|
||||
in workflow
|
||||
)
|
||||
assert (
|
||||
'echo "$added_modified_or_renamed" | grep -E "^(${folder_alt})/changelog\\.d/"'
|
||||
in workflow
|
||||
)
|
||||
|
||||
|
||||
def test_changelog_gate_uses_random_github_output_delimiters():
|
||||
workflow = read_workflow("pr-check-changelog.yml")
|
||||
|
||||
assert "write_multiline_output()" in workflow
|
||||
assert "openssl rand -hex 16" in workflow
|
||||
assert '>> "$GITHUB_OUTPUT"' in workflow
|
||||
assert "<<EOF" not in workflow
|
||||
|
||||
|
||||
def test_compile_workflow_blocks_egress_for_privileged_job():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
|
||||
assert "Harden the runner (Block outbound calls)" in workflow
|
||||
assert "egress-policy: block" in workflow
|
||||
for endpoint in [
|
||||
"api.github.com:443",
|
||||
"github.com:443",
|
||||
"objects.githubusercontent.com:443",
|
||||
"pypi.org:443",
|
||||
"files.pythonhosted.org:443",
|
||||
]:
|
||||
assert endpoint in workflow
|
||||
|
||||
|
||||
def test_forward_sync_inserts_release_blocks_by_prowler_version_order():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
|
||||
assert "insert_changelog_block_ordered()" in workflow
|
||||
assert 'incoming_release=$(release_from_heading "$incoming_heading")' in workflow
|
||||
assert 'incoming_key=$(version_key "$incoming_release")' in workflow
|
||||
assert '[[ "$incoming_key" > "$existing_key" ]]' in workflow
|
||||
assert "already contains a block for Prowler v${incoming_release}" in workflow
|
||||
assert 'head -n "$marker_line" "$component/CHANGELOG.md"' not in workflow
|
||||
|
||||
|
||||
def test_compile_workflow_rejects_explicit_versions_that_do_not_bump():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
|
||||
assert 'current=$(latest_released_version "$component")' in workflow
|
||||
assert 'current_key=$(version_key "$current")' in workflow
|
||||
assert 'effective_key=$(version_key "$effective")' in workflow
|
||||
assert (
|
||||
'[[ "$effective_key" < "$current_key" || "$effective_key" == "$current_key" ]]'
|
||||
in workflow
|
||||
)
|
||||
assert (
|
||||
"explicit version '${effective}' must be greater than the latest released version"
|
||||
in workflow
|
||||
)
|
||||
|
||||
|
||||
def test_compile_workflow_requires_target_branch_to_match_prowler_version():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
|
||||
assert (
|
||||
'IFS=. read -r prowler_major prowler_minor prowler_patch <<< "$PROWLER_VERSION"'
|
||||
in workflow
|
||||
)
|
||||
assert "prowler_patch=$((10#$prowler_patch))" in workflow
|
||||
assert 'if [ "$prowler_patch" -eq 0 ]; then' in workflow
|
||||
assert "target_branch must be 'master' for Prowler ${PROWLER_VERSION}" in workflow
|
||||
assert 'expected_target_branch="v${prowler_major}.${prowler_minor}"' in workflow
|
||||
assert 'if [ "$TARGET_BRANCH" != "$expected_target_branch" ]; then' in workflow
|
||||
assert (
|
||||
"target_branch must be '${expected_target_branch}' for Prowler ${PROWLER_VERSION}"
|
||||
in workflow
|
||||
)
|
||||
|
||||
|
||||
def test_compile_workflow_normalizes_version_segments_before_arithmetic():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
|
||||
assert (
|
||||
'printf \'%06d.%06d.%06d\' "$((10#$major))" "$((10#$minor))" "$((10#$patch))"'
|
||||
in workflow
|
||||
)
|
||||
assert "major=$((10#$major))" in workflow
|
||||
assert "minor=$((10#$minor))" in workflow
|
||||
assert "patch=$((10#$patch))" in workflow
|
||||
assert "current_major=$((10#$current_major))" in workflow
|
||||
assert "effective_major=$((10#$effective_major))" in workflow
|
||||
assert "effective_minor=$((10#$effective_minor))" in workflow
|
||||
assert "effective_patch=$((10#$effective_patch))" in workflow
|
||||
|
||||
|
||||
def test_compile_workflow_requires_removed_fragments_in_major_releases():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
|
||||
assert "has_removed_fragments" in workflow
|
||||
assert "removed fragments require a major component release" in workflow
|
||||
assert "effective_major" in workflow
|
||||
assert "effective_minor" in workflow
|
||||
assert "effective_patch" in workflow
|
||||
|
||||
|
||||
def test_compile_workflow_prs_skip_cloud_sync():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
labels_blocks = re.findall(r"labels: \|\n((?:\s+[a-z-]+\n)+)", workflow)
|
||||
|
||||
assert len(labels_blocks) == 2
|
||||
for block in labels_blocks:
|
||||
assert "no-changelog" in block.split()
|
||||
assert "skip-sync" in block.split()
|
||||
|
||||
|
||||
def test_compile_workflow_auto_derives_versions_by_mirroring_prowler_version():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
|
||||
assert 'prowler) effective="$PROWLER_VERSION" ;;' in workflow
|
||||
assert 'ui) effective="1.${prowler_minor}.${prowler_patch}" ;;' in workflow
|
||||
assert 'api) effective="1.$((prowler_minor + 1)).${prowler_patch}" ;;' in workflow
|
||||
assert (
|
||||
"auto-derived version '${effective}' is not greater than the latest released version"
|
||||
in workflow
|
||||
)
|
||||
|
||||
|
||||
def test_compile_workflow_auto_derives_mcp_patch_bumps_on_patch_releases():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
|
||||
assert "'added'/'deprecated' fragments are shipping in a Prowler patch" in workflow
|
||||
assert (
|
||||
"elif echo \"$fragments\" | grep -qE '\\.(added|changed|deprecated)(\\.[0-9]+)?\\.md$'; then"
|
||||
in workflow
|
||||
)
|
||||
|
||||
|
||||
def test_forward_sync_pads_release_blocks_with_blank_lines():
|
||||
workflow = read_workflow("compile-changelogs.yml")
|
||||
|
||||
assert "block-normalized.md" in workflow
|
||||
assert 'total_lines=$(wc -l < "$changelog")' in workflow
|
||||
assert '[ -n "$(sed -n "$((insertion_line - 1))p" "$changelog")" ]' in workflow
|
||||
assert 'if [ "$insertion_line" -le "$total_lines" ]; then' in workflow
|
||||
|
||||
|
||||
def test_component_changelogs_separate_release_blocks_with_blank_lines():
|
||||
for component in COMPONENTS:
|
||||
lines = (REPO_ROOT / component / "CHANGELOG.md").read_text().splitlines()
|
||||
marker_line = lines.index("<!-- changelog: release notes start -->")
|
||||
|
||||
assert (
|
||||
lines[marker_line + 1] == ""
|
||||
), f"{component}/CHANGELOG.md: expected a blank line after the marker"
|
||||
assert (
|
||||
lines[marker_line + 2] != ""
|
||||
), f"{component}/CHANGELOG.md: expected a single blank line after the marker"
|
||||
for index, line in enumerate(lines):
|
||||
if line == "---" and index + 1 < len(lines):
|
||||
assert lines[index + 1] == "", (
|
||||
f"{component}/CHANGELOG.md line {index + 2}: "
|
||||
"expected a blank line after '---'"
|
||||
)
|
||||
if line.startswith("## ["):
|
||||
assert lines[index - 1] == "", (
|
||||
f"{component}/CHANGELOG.md line {index}: "
|
||||
"expected a blank line before a release heading"
|
||||
)
|
||||
@@ -16,8 +16,11 @@ from prowler.lib.outputs.jira.exceptions.exceptions import (
|
||||
JiraGetProjectsError,
|
||||
JiraGetProjectsResponseError,
|
||||
JiraNoProjectsError,
|
||||
JiraNoTokenError,
|
||||
JiraRefreshTokenError,
|
||||
JiraRefreshTokenResponseError,
|
||||
JiraRequiredCustomFieldsError,
|
||||
JiraSendFindingsResponseError,
|
||||
JiraTestConnectionError,
|
||||
)
|
||||
from prowler.lib.outputs.jira.jira import Jira
|
||||
@@ -1692,7 +1695,7 @@ class TestJiraIntegration:
|
||||
mock_cloud_id,
|
||||
mock_get_access_token,
|
||||
):
|
||||
"""Test that send_finding returns False when the request fails."""
|
||||
"""Test that send_finding raises with Jira JSON error details."""
|
||||
# To disable vulture
|
||||
mock_cloud_id = mock_cloud_id
|
||||
mock_get_access_token = mock_get_access_token
|
||||
@@ -1702,19 +1705,66 @@ class TestJiraIntegration:
|
||||
# Mock failed response
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 400
|
||||
mock_response.json.return_value = {"errors": {"summary": "Required field"}}
|
||||
mock_response.json.return_value = {
|
||||
"errors": {"Team": "Team is required."},
|
||||
"errorMessages": ["Field 'Team' cannot be set."],
|
||||
}
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
result = self.jira_integration.send_finding(
|
||||
check_id="test-check",
|
||||
check_title="Test Finding",
|
||||
severity="High",
|
||||
status="FAIL",
|
||||
project_key="TEST",
|
||||
issue_type="Bug",
|
||||
)
|
||||
with pytest.raises(JiraSendFindingsResponseError) as error:
|
||||
self.jira_integration.send_finding(
|
||||
check_id="test-check",
|
||||
check_title="Test Finding",
|
||||
severity="High",
|
||||
status="FAIL",
|
||||
project_key="TEST",
|
||||
issue_type="Bug",
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert "Failed to create Jira issue" in str(error.value)
|
||||
assert "'Team': 'Team is required.'" in str(error.value)
|
||||
assert "Field 'Team' cannot be set." in str(error.value)
|
||||
mock_post.assert_called_once()
|
||||
|
||||
@patch.object(Jira, "get_access_token", return_value="valid_access_token")
|
||||
@patch.object(
|
||||
Jira, "cloud_id", new_callable=PropertyMock, return_value="test_cloud_id"
|
||||
)
|
||||
@patch.object(Jira, "get_projects", return_value={"TEST": {"name": "Test Project"}})
|
||||
@patch.object(Jira, "get_available_issue_types", return_value=["Bug"])
|
||||
@patch("prowler.lib.outputs.jira.jira.requests.post")
|
||||
def test_send_finding_response_error_without_json_body(
|
||||
self,
|
||||
mock_post,
|
||||
mock_get_issue_types,
|
||||
mock_get_projects,
|
||||
mock_cloud_id,
|
||||
mock_get_access_token,
|
||||
):
|
||||
"""Test send_finding raises with status-code context for non-JSON errors."""
|
||||
# To disable vulture
|
||||
mock_cloud_id = mock_cloud_id
|
||||
mock_get_access_token = mock_get_access_token
|
||||
mock_get_projects = mock_get_projects
|
||||
mock_get_issue_types = mock_get_issue_types
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 502
|
||||
mock_response.json.side_effect = ValueError("No JSON body")
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
with pytest.raises(JiraSendFindingsResponseError) as error:
|
||||
self.jira_integration.send_finding(
|
||||
check_id="test-check",
|
||||
check_title="Test Finding",
|
||||
severity="High",
|
||||
status="FAIL",
|
||||
project_key="TEST",
|
||||
issue_type="Bug",
|
||||
)
|
||||
|
||||
assert "Failed to create Jira issue" in str(error.value)
|
||||
assert "Jira returned status code 502" in str(error.value)
|
||||
mock_post.assert_called_once()
|
||||
|
||||
@patch.object(Jira, "get_access_token", return_value="valid_access_token")
|
||||
@@ -1732,7 +1782,7 @@ class TestJiraIntegration:
|
||||
mock_cloud_id,
|
||||
mock_get_access_token,
|
||||
):
|
||||
"""Test that send_finding returns False when custom fields cause an error."""
|
||||
"""Test that send_finding raises when custom fields cause an error."""
|
||||
# To disable vulture
|
||||
mock_cloud_id = mock_cloud_id
|
||||
mock_get_access_token = mock_get_access_token
|
||||
@@ -1750,18 +1800,89 @@ class TestJiraIntegration:
|
||||
}
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
result = self.jira_integration.send_finding(
|
||||
check_id="test-check",
|
||||
check_title="Test Finding",
|
||||
severity="High",
|
||||
status="FAIL",
|
||||
project_key="TEST",
|
||||
issue_type="Bug",
|
||||
)
|
||||
with pytest.raises(JiraRequiredCustomFieldsError) as error:
|
||||
self.jira_integration.send_finding(
|
||||
check_id="test-check",
|
||||
check_title="Test Finding",
|
||||
severity="High",
|
||||
status="FAIL",
|
||||
project_key="TEST",
|
||||
issue_type="Bug",
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert "Jira project requires custom fields" in str(error.value)
|
||||
assert "customfield_10001" in str(error.value)
|
||||
mock_post.assert_called_once()
|
||||
|
||||
@patch.object(
|
||||
Jira,
|
||||
"get_access_token",
|
||||
side_effect=JiraRefreshTokenError(message="Failed to refresh the access token"),
|
||||
)
|
||||
def test_send_finding_reraises_refresh_token_error(self, mock_get_access_token):
|
||||
"""Test send_finding re-raises refresh token errors for API propagation."""
|
||||
# To disable vulture
|
||||
mock_get_access_token = mock_get_access_token
|
||||
|
||||
with pytest.raises(JiraRefreshTokenError) as error:
|
||||
self.jira_integration.send_finding(
|
||||
check_id="test-check",
|
||||
check_title="Test Finding",
|
||||
severity="High",
|
||||
status="FAIL",
|
||||
project_key="TEST",
|
||||
issue_type="Bug",
|
||||
)
|
||||
|
||||
assert error.value.message == "Failed to refresh the access token"
|
||||
|
||||
@patch.object(Jira, "get_access_token", return_value=None)
|
||||
def test_send_finding_reraises_no_token_error(self, mock_get_access_token):
|
||||
"""Test send_finding re-raises missing token errors for API propagation."""
|
||||
# To disable vulture
|
||||
mock_get_access_token = mock_get_access_token
|
||||
|
||||
with pytest.raises(JiraNoTokenError) as error:
|
||||
self.jira_integration.send_finding(
|
||||
check_id="test-check",
|
||||
check_title="Test Finding",
|
||||
severity="High",
|
||||
status="FAIL",
|
||||
project_key="TEST",
|
||||
issue_type="Bug",
|
||||
)
|
||||
|
||||
assert error.value.message == "No token was found"
|
||||
|
||||
@patch.object(
|
||||
Jira,
|
||||
"get_access_token",
|
||||
side_effect=JiraRefreshTokenResponseError(
|
||||
message="Failed to refresh the access token, response code did not match 200"
|
||||
),
|
||||
)
|
||||
def test_send_finding_reraises_refresh_token_response_error(
|
||||
self, mock_get_access_token
|
||||
):
|
||||
"""Test send_finding re-raises refresh token response errors for API propagation."""
|
||||
# To disable vulture
|
||||
mock_get_access_token = mock_get_access_token
|
||||
|
||||
with pytest.raises(JiraRefreshTokenResponseError) as error:
|
||||
self.jira_integration.send_finding(
|
||||
check_id="test-check",
|
||||
check_title="Test Finding",
|
||||
severity="High",
|
||||
status="FAIL",
|
||||
project_key="TEST",
|
||||
issue_type="Bug",
|
||||
)
|
||||
|
||||
assert (
|
||||
error.value.message
|
||||
== "Failed to refresh the access token, response code did not match 200"
|
||||
)
|
||||
|
||||
def test_get_headers_oauth_with_access_token(self):
|
||||
"""Test get_headers returns correct OAuth headers with access token."""
|
||||
self.jira_integration._using_basic_auth = False
|
||||
|
||||
+80
-39
@@ -1,3 +1,5 @@
|
||||
import builtins
|
||||
import sys
|
||||
from unittest import mock
|
||||
|
||||
from boto3 import client, resource
|
||||
@@ -12,38 +14,42 @@ from tests.providers.aws.utils import (
|
||||
)
|
||||
|
||||
LIFECYCLE_POLICY_ID = "policy-XXXXXXXXXXXX"
|
||||
CHECK_MODULE = (
|
||||
"prowler.providers.aws.services.dlm."
|
||||
"dlm_ebs_snapshot_lifecycle_policy_exists."
|
||||
"dlm_ebs_snapshot_lifecycle_policy_exists"
|
||||
)
|
||||
DLM_CLIENT_MODULE = "prowler.providers.aws.services.dlm.dlm_client"
|
||||
EC2_CLIENT_MODULE = "prowler.providers.aws.services.ec2.ec2_client"
|
||||
|
||||
|
||||
def unload_dlm_check_modules():
|
||||
sys.modules.pop(CHECK_MODULE, None)
|
||||
sys.modules.pop(DLM_CLIENT_MODULE, None)
|
||||
|
||||
|
||||
class Test_dlm_ebs_snapshot_lifecycle_policy_exists:
|
||||
@mock_aws
|
||||
def test_no_ebs_snapshot_no_lifecycle_policies(self):
|
||||
# DLM Mock Client
|
||||
dlm_client = mock.MagicMock
|
||||
dlm_client = mock.MagicMock()
|
||||
dlm_client.audited_account = AWS_ACCOUNT_NUMBER
|
||||
dlm_client.audited_account_arn = AWS_ACCOUNT_ARN
|
||||
dlm_client.lifecycle_policies = {}
|
||||
dlm_client.regions_with_snapshots = {}
|
||||
|
||||
aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1])
|
||||
|
||||
from prowler.providers.aws.services.ec2.ec2_service import EC2
|
||||
unload_dlm_check_modules()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dlm.dlm_service.DLM",
|
||||
new=dlm_client,
|
||||
new=mock.MagicMock(return_value=dlm_client),
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.ec2.ec2_service.EC2",
|
||||
return_value=EC2(aws_provider),
|
||||
) as ec2_client,
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.ec2.ec2_client.ec2_client",
|
||||
new=ec2_client,
|
||||
),
|
||||
):
|
||||
from prowler.providers.aws.services.dlm.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_ebs_snapshot_lifecycle_policy_exists import (
|
||||
dlm_ebs_snapshot_lifecycle_policy_exists,
|
||||
@@ -92,22 +98,22 @@ class Test_dlm_ebs_snapshot_lifecycle_policy_exists:
|
||||
)
|
||||
}
|
||||
}
|
||||
dlm_client.regions_with_snapshots = {AWS_REGION_US_EAST_1: True}
|
||||
dlm_client.lifecycle_policy_arn_template = f"arn:{dlm_client.audited_partition}:dlm:{dlm_client.region}:{dlm_client.audited_account}:policy"
|
||||
dlm_client._get_lifecycle_policy_arn_template = mock.MagicMock(
|
||||
return_value=dlm_client.lifecycle_policy_arn_template
|
||||
)
|
||||
aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1])
|
||||
|
||||
from prowler.providers.aws.services.ec2.ec2_service import EC2
|
||||
unload_dlm_check_modules()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
"prowler.providers.aws.services.dlm.dlm_service.DLM",
|
||||
new=mock.MagicMock(return_value=dlm_client),
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dlm.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_ebs_snapshot_lifecycle_policy_exists.ec2_client",
|
||||
new=EC2(aws_provider),
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dlm.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_client",
|
||||
@@ -154,25 +160,23 @@ class Test_dlm_ebs_snapshot_lifecycle_policy_exists:
|
||||
)["SnapshotId"]
|
||||
|
||||
# DLM Mock Client
|
||||
dlm_client = mock.MagicMock
|
||||
dlm_client = mock.MagicMock()
|
||||
dlm_client.audited_account = AWS_ACCOUNT_NUMBER
|
||||
dlm_client.audited_account_arn = AWS_ACCOUNT_ARN
|
||||
dlm_client.lifecycle_policies = {}
|
||||
|
||||
# from prowler.providers.aws.services.ec2.ec2_service import EC2
|
||||
dlm_client.regions_with_snapshots = {AWS_REGION_US_EAST_1: True}
|
||||
|
||||
aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1])
|
||||
|
||||
from prowler.providers.aws.services.ec2.ec2_service import EC2
|
||||
unload_dlm_check_modules()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
"prowler.providers.aws.services.dlm.dlm_service.DLM",
|
||||
new=mock.MagicMock(return_value=dlm_client),
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dlm.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_ebs_snapshot_lifecycle_policy_exists.ec2_client",
|
||||
new=EC2(aws_provider),
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dlm.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_client",
|
||||
@@ -190,7 +194,7 @@ class Test_dlm_ebs_snapshot_lifecycle_policy_exists:
|
||||
@mock_aws
|
||||
def test_no_ebs_snapshot_and_dlm_lifecycle_policy(self):
|
||||
# DLM Mock Client
|
||||
dlm_client = mock.MagicMock
|
||||
dlm_client = mock.MagicMock()
|
||||
dlm_client.audited_account = AWS_ACCOUNT_NUMBER
|
||||
dlm_client.audited_account_arn = AWS_ACCOUNT_ARN
|
||||
dlm_client.lifecycle_policies = {
|
||||
@@ -203,30 +207,25 @@ class Test_dlm_ebs_snapshot_lifecycle_policy_exists:
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
# from prowler.providers.aws.services.ec2.ec2_service import EC2
|
||||
dlm_client.regions_with_snapshots = {}
|
||||
|
||||
aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1])
|
||||
|
||||
from prowler.providers.aws.services.ec2.ec2_service import EC2
|
||||
unload_dlm_check_modules()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dlm.dlm_service.DLM",
|
||||
new=mock.MagicMock(return_value=dlm_client),
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dlm.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_ebs_snapshot_lifecycle_policy_exists.ec2_client",
|
||||
new=EC2(aws_provider),
|
||||
) as ec2_client,
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dlm.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_client",
|
||||
new=dlm_client,
|
||||
),
|
||||
):
|
||||
# Remove all snapshots
|
||||
ec2_client.regions_with_snapshots = {}
|
||||
|
||||
from prowler.providers.aws.services.dlm.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_ebs_snapshot_lifecycle_policy_exists import (
|
||||
dlm_ebs_snapshot_lifecycle_policy_exists,
|
||||
)
|
||||
@@ -234,3 +233,45 @@ class Test_dlm_ebs_snapshot_lifecycle_policy_exists:
|
||||
check = dlm_ebs_snapshot_lifecycle_policy_exists()
|
||||
result = check.execute()
|
||||
assert len(result) == 0
|
||||
|
||||
@mock_aws
|
||||
def test_check_does_not_import_ec2_service_client(self):
|
||||
dlm_client = mock.MagicMock()
|
||||
dlm_client.audited_account = AWS_ACCOUNT_NUMBER
|
||||
dlm_client.audited_account_arn = AWS_ACCOUNT_ARN
|
||||
dlm_client.audited_partition = "aws"
|
||||
dlm_client.lifecycle_policies = {AWS_REGION_US_EAST_1: {}}
|
||||
dlm_client.regions_with_snapshots = {AWS_REGION_US_EAST_1: True}
|
||||
dlm_client._get_lifecycle_policy_arn_template = mock.MagicMock(
|
||||
return_value=f"arn:aws:dlm:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:policy"
|
||||
)
|
||||
|
||||
aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1])
|
||||
unload_dlm_check_modules()
|
||||
sys.modules.pop(EC2_CLIENT_MODULE, None)
|
||||
real_import = builtins.__import__
|
||||
|
||||
def guarded_import(name, *args, **kwargs):
|
||||
if name == EC2_CLIENT_MODULE:
|
||||
raise AssertionError("DLM check must not import the EC2 service client")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dlm.dlm_service.DLM",
|
||||
new=mock.MagicMock(return_value=dlm_client),
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
),
|
||||
mock.patch("builtins.__import__", side_effect=guarded_import),
|
||||
):
|
||||
from prowler.providers.aws.services.dlm.dlm_ebs_snapshot_lifecycle_policy_exists.dlm_ebs_snapshot_lifecycle_policy_exists import (
|
||||
dlm_ebs_snapshot_lifecycle_policy_exists,
|
||||
)
|
||||
|
||||
result = dlm_ebs_snapshot_lifecycle_policy_exists().execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].status == "FAIL"
|
||||
|
||||
@@ -34,6 +34,8 @@ def mock_make_api_call(self, operation_name, kwargs):
|
||||
}
|
||||
]
|
||||
}
|
||||
if operation_name == "DescribeSnapshots":
|
||||
return {"Snapshots": [{"SnapshotId": "snap-1234567890abcdef0"}]}
|
||||
|
||||
return make_api_call(self, operation_name, kwargs)
|
||||
|
||||
@@ -46,6 +48,13 @@ def mock_generate_regional_clients(provider, service):
|
||||
return {AWS_REGION_US_EAST_1: regional_client}
|
||||
|
||||
|
||||
def mock_generate_regional_clients_without_ec2(provider, service):
|
||||
if service == "ec2":
|
||||
return None
|
||||
|
||||
return mock_generate_regional_clients(provider, service)
|
||||
|
||||
|
||||
@patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.generate_regional_clients",
|
||||
new=mock_generate_regional_clients,
|
||||
@@ -92,3 +101,62 @@ class Test_DLM_Service:
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
def test_get_regions_with_snapshots(self):
|
||||
aws_provider = set_mocked_aws_provider()
|
||||
dlm = DLM(aws_provider)
|
||||
assert dlm.regions_with_snapshots == {AWS_REGION_US_EAST_1: True}
|
||||
|
||||
|
||||
@patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.generate_regional_clients",
|
||||
new=mock_generate_regional_clients_without_ec2,
|
||||
)
|
||||
@patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call)
|
||||
class Test_DLM_Service_Without_EC2_Regional_Clients:
|
||||
def test_service_handles_missing_ec2_regional_clients(self):
|
||||
aws_provider = set_mocked_aws_provider()
|
||||
dlm = DLM(aws_provider)
|
||||
|
||||
assert dlm.regions_with_snapshots == {}
|
||||
|
||||
|
||||
class FakeEC2RegionalClient:
|
||||
def __init__(self, responses):
|
||||
self.region = AWS_REGION_US_EAST_1
|
||||
self.requests = []
|
||||
self.responses = list(responses)
|
||||
|
||||
def describe_snapshots(self, **kwargs):
|
||||
self.requests.append(kwargs)
|
||||
return self.responses.pop(0)
|
||||
|
||||
|
||||
class Test_DLM_Regions_With_Snapshots:
|
||||
def test_get_regions_without_snapshots(self):
|
||||
dlm = DLM.__new__(DLM)
|
||||
dlm.regions_with_snapshots = {}
|
||||
regional_client = FakeEC2RegionalClient([{"Snapshots": []}])
|
||||
|
||||
dlm._get_regions_with_snapshots(regional_client)
|
||||
|
||||
assert dlm.regions_with_snapshots == {AWS_REGION_US_EAST_1: False}
|
||||
assert regional_client.requests == [{"OwnerIds": ["self"], "MaxResults": 5}]
|
||||
|
||||
def test_get_regions_with_snapshots_after_pagination(self):
|
||||
dlm = DLM.__new__(DLM)
|
||||
dlm.regions_with_snapshots = {}
|
||||
regional_client = FakeEC2RegionalClient(
|
||||
[
|
||||
{"Snapshots": [], "NextToken": "next-page"},
|
||||
{"Snapshots": [{"SnapshotId": "snap-1234567890abcdef0"}]},
|
||||
]
|
||||
)
|
||||
|
||||
dlm._get_regions_with_snapshots(regional_client)
|
||||
|
||||
assert dlm.regions_with_snapshots == {AWS_REGION_US_EAST_1: True}
|
||||
assert regional_client.requests == [
|
||||
{"OwnerIds": ["self"], "MaxResults": 5},
|
||||
{"OwnerIds": ["self"], "MaxResults": 5, "NextToken": "next-page"},
|
||||
]
|
||||
|
||||
+57
-4
@@ -1,3 +1,5 @@
|
||||
import ast
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import botocore
|
||||
@@ -61,6 +63,42 @@ def mock_make_api_call_private(self, operation_name, kwargs):
|
||||
|
||||
|
||||
class Test_dms_instance_no_public_access:
|
||||
def test_ec2_client_is_not_imported_at_module_level(self):
|
||||
repo_root = Path(__file__).parents[6]
|
||||
check_source = repo_root / (
|
||||
"prowler/providers/aws/services/dms/dms_instance_no_public_access/dms_instance_no_public_access.py"
|
||||
)
|
||||
check_tree = ast.parse(check_source.read_text())
|
||||
|
||||
top_level_imports = [
|
||||
node
|
||||
for node in check_tree.body
|
||||
if isinstance(node, (ast.Import, ast.ImportFrom))
|
||||
]
|
||||
|
||||
top_level_ec2_client_imports = [
|
||||
node
|
||||
for node in top_level_imports
|
||||
if (
|
||||
isinstance(node, ast.ImportFrom)
|
||||
and node.module == "prowler.providers.aws.services.ec2.ec2_client"
|
||||
)
|
||||
or (
|
||||
isinstance(node, ast.ImportFrom)
|
||||
and node.module == "prowler.providers.aws.services.ec2"
|
||||
and any(alias.name == "ec2_client" for alias in node.names)
|
||||
)
|
||||
or (
|
||||
isinstance(node, ast.Import)
|
||||
and any(
|
||||
alias.name == "prowler.providers.aws.services.ec2.ec2_client"
|
||||
for alias in node.names
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
assert top_level_ec2_client_imports == []
|
||||
|
||||
@mock_aws
|
||||
def test_dms_no_instances(self):
|
||||
dms_client = client("dms", region_name=AWS_REGION_US_EAST_1)
|
||||
@@ -79,6 +117,10 @@ class Test_dms_instance_no_public_access:
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access.dms_client",
|
||||
new=DMS(aws_provider),
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access._get_ec2_client",
|
||||
side_effect=AssertionError("EC2 client should not be loaded"),
|
||||
) as get_ec2_client_mock,
|
||||
):
|
||||
from prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access import (
|
||||
dms_instance_no_public_access,
|
||||
@@ -87,6 +129,7 @@ class Test_dms_instance_no_public_access:
|
||||
check = dms_instance_no_public_access()
|
||||
result = check.execute()
|
||||
assert len(result) == 0
|
||||
get_ec2_client_mock.assert_not_called()
|
||||
|
||||
@mock_aws
|
||||
def test_dms_private(self):
|
||||
@@ -108,6 +151,10 @@ class Test_dms_instance_no_public_access:
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access.dms_client",
|
||||
new=DMS(aws_provider),
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access._get_ec2_client",
|
||||
side_effect=AssertionError("EC2 client should not be loaded"),
|
||||
) as get_ec2_client_mock,
|
||||
):
|
||||
from prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access import (
|
||||
dms_instance_no_public_access,
|
||||
@@ -125,6 +172,7 @@ class Test_dms_instance_no_public_access:
|
||||
assert result[0].resource_id == DMS_INSTANCE_NAME
|
||||
assert result[0].resource_arn == DMS_INSTANCE_ARN
|
||||
assert result[0].resource_tags == []
|
||||
get_ec2_client_mock.assert_not_called()
|
||||
|
||||
@mock_aws
|
||||
def test_dms_public(self):
|
||||
@@ -146,6 +194,10 @@ class Test_dms_instance_no_public_access:
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access.dms_client",
|
||||
new=DMS(aws_provider),
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access._get_ec2_client",
|
||||
side_effect=AssertionError("EC2 client should not be loaded"),
|
||||
) as get_ec2_client_mock,
|
||||
):
|
||||
from prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access import (
|
||||
dms_instance_no_public_access,
|
||||
@@ -163,6 +215,7 @@ class Test_dms_instance_no_public_access:
|
||||
assert result[0].resource_id == DMS_INSTANCE_NAME
|
||||
assert result[0].resource_arn == DMS_INSTANCE_ARN
|
||||
assert result[0].resource_tags == []
|
||||
get_ec2_client_mock.assert_not_called()
|
||||
|
||||
@mock_aws
|
||||
def test_dms_public_with_public_sg(self):
|
||||
@@ -218,8 +271,8 @@ class Test_dms_instance_no_public_access:
|
||||
new=dms_client,
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access.ec2_client",
|
||||
new=EC2(aws_provider),
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access._get_ec2_client",
|
||||
return_value=EC2(aws_provider),
|
||||
),
|
||||
):
|
||||
# Test Check
|
||||
@@ -298,8 +351,8 @@ class Test_dms_instance_no_public_access:
|
||||
new=dms_client,
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access.ec2_client",
|
||||
new=EC2(aws_provider),
|
||||
"prowler.providers.aws.services.dms.dms_instance_no_public_access.dms_instance_no_public_access._get_ec2_client",
|
||||
return_value=EC2(aws_provider),
|
||||
),
|
||||
):
|
||||
# Test Check
|
||||
|
||||
@@ -141,3 +141,86 @@ class Test_ec2_ami_public:
|
||||
)
|
||||
assert result[0].region == AWS_REGION_US_EAST_1
|
||||
assert result[0].resource_tags == []
|
||||
|
||||
@mock_aws
|
||||
def test_multiple_self_owned_amis_mixed_public_and_private(self):
|
||||
ec2 = client("ec2", region_name=AWS_REGION_US_EAST_1)
|
||||
|
||||
reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)
|
||||
instance = reservation["Instances"][0]
|
||||
instance_id = instance["InstanceId"]
|
||||
|
||||
private_image_id = ec2.create_image(
|
||||
InstanceId=instance_id,
|
||||
Name="test-private-ami",
|
||||
Description="this is a private test ami",
|
||||
)["ImageId"]
|
||||
public_image_id = ec2.create_image(
|
||||
InstanceId=instance_id,
|
||||
Name="test-public-ami",
|
||||
Description="this is a public test ami",
|
||||
)["ImageId"]
|
||||
|
||||
image = resource("ec2", region_name=AWS_REGION_US_EAST_1).Image(public_image_id)
|
||||
image.modify_attribute(
|
||||
ImageId=public_image_id,
|
||||
Attribute="launchPermission",
|
||||
OperationType="add",
|
||||
UserGroups=["all"],
|
||||
)
|
||||
|
||||
from prowler.providers.aws.services.ec2.ec2_service import EC2
|
||||
|
||||
aws_provider = set_mocked_aws_provider(
|
||||
[AWS_REGION_EU_WEST_1, AWS_REGION_US_EAST_1]
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.ec2.ec2_ami_public.ec2_ami_public.ec2_client",
|
||||
new=EC2(aws_provider),
|
||||
),
|
||||
):
|
||||
from prowler.providers.aws.services.ec2.ec2_ami_public.ec2_ami_public import (
|
||||
ec2_ami_public,
|
||||
)
|
||||
|
||||
check = ec2_ami_public()
|
||||
result = check.execute()
|
||||
|
||||
findings_by_resource_id = {
|
||||
finding.resource_id: finding for finding in result
|
||||
}
|
||||
|
||||
assert len(result) == 2
|
||||
assert set(findings_by_resource_id) == {private_image_id, public_image_id}
|
||||
|
||||
private_finding = findings_by_resource_id[private_image_id]
|
||||
assert private_finding.status == "PASS"
|
||||
assert (
|
||||
private_finding.status_extended
|
||||
== "EC2 AMI test-private-ami is not public."
|
||||
)
|
||||
assert (
|
||||
private_finding.resource_arn
|
||||
== f"arn:{aws_provider.identity.partition}:ec2:{AWS_REGION_US_EAST_1}:{aws_provider.identity.account}:image/{private_image_id}"
|
||||
)
|
||||
assert private_finding.region == AWS_REGION_US_EAST_1
|
||||
assert private_finding.resource_tags == []
|
||||
|
||||
public_finding = findings_by_resource_id[public_image_id]
|
||||
assert public_finding.status == "FAIL"
|
||||
assert (
|
||||
public_finding.status_extended
|
||||
== "EC2 AMI test-public-ami is currently public."
|
||||
)
|
||||
assert (
|
||||
public_finding.resource_arn
|
||||
== f"arn:{aws_provider.identity.partition}:ec2:{AWS_REGION_US_EAST_1}:{aws_provider.identity.account}:image/{public_image_id}"
|
||||
)
|
||||
assert public_finding.region == AWS_REGION_US_EAST_1
|
||||
assert public_finding.resource_tags == []
|
||||
|
||||
+51
-7
@@ -1,15 +1,64 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
from moto import mock_aws
|
||||
|
||||
from tests.providers.aws.utils import (
|
||||
AWS_ACCOUNT_NUMBER,
|
||||
AWS_COMMERCIAL_PARTITION,
|
||||
AWS_REGION_EU_WEST_1,
|
||||
AWS_REGION_US_EAST_1,
|
||||
set_mocked_aws_provider,
|
||||
)
|
||||
|
||||
|
||||
class Test_ec2_instance_account_imdsv2_enabled:
|
||||
@mock_aws
|
||||
def test_ec2_imdsv2_uses_region_in_resource_arn(self):
|
||||
from prowler.providers.aws.services.ec2.ec2_service import (
|
||||
InstanceMetadataDefaults,
|
||||
)
|
||||
|
||||
ec2_client = SimpleNamespace(
|
||||
instance_metadata_defaults=[
|
||||
InstanceMetadataDefaults(
|
||||
http_tokens=None,
|
||||
instances=True,
|
||||
region=AWS_REGION_US_EAST_1,
|
||||
),
|
||||
InstanceMetadataDefaults(
|
||||
http_tokens=None,
|
||||
instances=True,
|
||||
region=AWS_REGION_EU_WEST_1,
|
||||
),
|
||||
],
|
||||
audited_account=AWS_ACCOUNT_NUMBER,
|
||||
audited_partition=AWS_COMMERCIAL_PARTITION,
|
||||
provider=SimpleNamespace(scan_unused_services=False),
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=set_mocked_aws_provider(),
|
||||
),
|
||||
mock.patch(
|
||||
"prowler.providers.aws.services.ec2.ec2_instance_account_imdsv2_enabled.ec2_instance_account_imdsv2_enabled.ec2_client",
|
||||
new=ec2_client,
|
||||
),
|
||||
):
|
||||
from prowler.providers.aws.services.ec2.ec2_instance_account_imdsv2_enabled.ec2_instance_account_imdsv2_enabled import (
|
||||
ec2_instance_account_imdsv2_enabled,
|
||||
)
|
||||
|
||||
result = ec2_instance_account_imdsv2_enabled().execute()
|
||||
|
||||
assert len(result) == 2
|
||||
assert {report.resource_arn for report in result} == {
|
||||
f"arn:aws:ec2:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:account",
|
||||
f"arn:aws:ec2:{AWS_REGION_EU_WEST_1}:{AWS_ACCOUNT_NUMBER}:account",
|
||||
}
|
||||
|
||||
@mock_aws
|
||||
def test_ec2_imdsv2_required(self):
|
||||
from prowler.providers.aws.services.ec2.ec2_service import (
|
||||
@@ -23,10 +72,8 @@ class Test_ec2_instance_account_imdsv2_enabled:
|
||||
)
|
||||
]
|
||||
ec2_client.audited_account = AWS_ACCOUNT_NUMBER
|
||||
ec2_client.audited_partition = AWS_COMMERCIAL_PARTITION
|
||||
ec2_client.region = AWS_REGION_US_EAST_1
|
||||
ec2_client.account_arn_template = (
|
||||
f"arn:aws:ec2:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:account"
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
@@ -71,12 +118,9 @@ class Test_ec2_instance_account_imdsv2_enabled:
|
||||
)
|
||||
]
|
||||
ec2_client.audited_account = AWS_ACCOUNT_NUMBER
|
||||
ec2_client.audited_partition = AWS_COMMERCIAL_PARTITION
|
||||
ec2_client.region = AWS_REGION_US_EAST_1
|
||||
|
||||
ec2_client.account_arn_template = (
|
||||
f"arn:aws:ec2:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:account"
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
|
||||
+84
-4
@@ -1,11 +1,17 @@
|
||||
from unittest import mock
|
||||
|
||||
import botocore
|
||||
from moto import mock_aws
|
||||
import pytest
|
||||
|
||||
from tests.providers.aws.utils import AWS_REGION_US_EAST_1, set_mocked_aws_provider
|
||||
|
||||
make_api_call = botocore.client.BaseClient._make_api_call
|
||||
describe_images_calls = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_describe_images_calls():
|
||||
describe_images_calls.clear()
|
||||
|
||||
|
||||
def mock_make_api_call(self, operation_name, kwarg):
|
||||
@@ -27,16 +33,25 @@ def mock_make_api_call(self, operation_name, kwarg):
|
||||
]
|
||||
}
|
||||
elif operation_name == "DescribeImages":
|
||||
if "Owners" in kwarg and kwarg["Owners"] == ["amazon"]:
|
||||
describe_images_calls.append(kwarg)
|
||||
if kwarg.get("Owners") == ["self"]:
|
||||
return {"Images": []}
|
||||
if kwarg.get("Owners") == ["amazon"]:
|
||||
raise AssertionError(
|
||||
"Amazon AMIs must not be fetched with a broad owner lookup"
|
||||
)
|
||||
if kwarg.get("ImageIds") == ["ami-12345678"]:
|
||||
return {
|
||||
"Images": [
|
||||
{
|
||||
"ImageId": "ami-12345678",
|
||||
"DeprecationTime": "2050-01-01T00:00:00.000Z",
|
||||
"Public": True,
|
||||
"ImageOwnerAlias": "amazon",
|
||||
}
|
||||
]
|
||||
}
|
||||
return {"Images": []}
|
||||
return make_api_call(self, operation_name, kwarg)
|
||||
|
||||
|
||||
@@ -59,6 +74,13 @@ def mock_make_api_call_private(self, operation_name, kwarg):
|
||||
]
|
||||
}
|
||||
elif operation_name == "DescribeImages":
|
||||
describe_images_calls.append(kwarg)
|
||||
if kwarg.get("Owners") == ["amazon"]:
|
||||
raise AssertionError(
|
||||
"Amazon AMIs must not be fetched with a broad owner lookup"
|
||||
)
|
||||
if kwarg.get("Owners") == ["self"]:
|
||||
return {"Images": []}
|
||||
return {
|
||||
"Images": [
|
||||
{
|
||||
@@ -90,16 +112,25 @@ def mock_make_api_call_outdated_ami(self, operation_name, kwarg):
|
||||
]
|
||||
}
|
||||
elif operation_name == "DescribeImages":
|
||||
if "Owners" in kwarg and kwarg["Owners"] == ["amazon"]:
|
||||
describe_images_calls.append(kwarg)
|
||||
if kwarg.get("Owners") == ["self"]:
|
||||
return {"Images": []}
|
||||
if kwarg.get("Owners") == ["amazon"]:
|
||||
raise AssertionError(
|
||||
"Amazon AMIs must not be fetched with a broad owner lookup"
|
||||
)
|
||||
if kwarg.get("ImageIds") == ["ami-87654321"]:
|
||||
return {
|
||||
"Images": [
|
||||
{
|
||||
"ImageId": "ami-87654321",
|
||||
"DeprecationTime": "2022-01-01T00:00:00.000Z",
|
||||
"Public": True,
|
||||
"ImageOwnerAlias": "amazon",
|
||||
}
|
||||
]
|
||||
}
|
||||
return {"Images": []}
|
||||
return make_api_call(self, operation_name, kwarg)
|
||||
|
||||
|
||||
@@ -122,12 +153,32 @@ def mock_make_api_call_missing_ami(self, operation_name, kwarg):
|
||||
]
|
||||
}
|
||||
elif operation_name == "DescribeImages":
|
||||
describe_images_calls.append(kwarg)
|
||||
if kwarg.get("Owners") == ["amazon"]:
|
||||
raise AssertionError(
|
||||
"Amazon AMIs must not be fetched with a broad owner lookup"
|
||||
)
|
||||
return {"Images": []}
|
||||
return make_api_call(self, operation_name, kwarg)
|
||||
|
||||
|
||||
def mock_make_api_call_no_instances(self, operation_name, kwarg):
|
||||
if operation_name == "DescribeInstances":
|
||||
return {"Reservations": []}
|
||||
elif operation_name == "DescribeImages":
|
||||
describe_images_calls.append(kwarg)
|
||||
if kwarg.get("Owners") == ["amazon"]:
|
||||
raise AssertionError(
|
||||
"Amazon AMIs must not be fetched with a broad owner lookup"
|
||||
)
|
||||
return {"Images": []}
|
||||
return make_api_call(self, operation_name, kwarg)
|
||||
|
||||
|
||||
class Test_ec2_instance_with_outdated_ami:
|
||||
@mock_aws
|
||||
@mock.patch(
|
||||
"botocore.client.BaseClient._make_api_call", new=mock_make_api_call_no_instances
|
||||
)
|
||||
def test_ec2_no_instances(self):
|
||||
from prowler.providers.aws.services.ec2.ec2_service import EC2
|
||||
|
||||
@@ -151,6 +202,7 @@ class Test_ec2_instance_with_outdated_ami:
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 0
|
||||
assert not any("ImageIds" in call for call in describe_images_calls)
|
||||
|
||||
@mock.patch(
|
||||
"botocore.client.BaseClient._make_api_call", new=mock_make_api_call_private
|
||||
@@ -178,6 +230,13 @@ class Test_ec2_instance_with_outdated_ami:
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 0
|
||||
assert not any(
|
||||
call.get("Owners") == ["amazon"] for call in describe_images_calls
|
||||
)
|
||||
assert any(
|
||||
call.get("ImageIds") == ["ami-12345678"]
|
||||
for call in describe_images_calls
|
||||
)
|
||||
|
||||
@mock.patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call)
|
||||
def test_instance_ami_not_outdated(self):
|
||||
@@ -209,6 +268,13 @@ class Test_ec2_instance_with_outdated_ami:
|
||||
result[0].status_extended
|
||||
== "EC2 Instance i-0123456789abcdef0 is not using an outdated AMI."
|
||||
)
|
||||
assert not any(
|
||||
call.get("Owners") == ["amazon"] for call in describe_images_calls
|
||||
)
|
||||
assert any(
|
||||
call.get("ImageIds") == ["ami-12345678"]
|
||||
for call in describe_images_calls
|
||||
)
|
||||
|
||||
@mock.patch(
|
||||
"botocore.client.BaseClient._make_api_call", new=mock_make_api_call_outdated_ami
|
||||
@@ -242,6 +308,13 @@ class Test_ec2_instance_with_outdated_ami:
|
||||
result[0].status_extended
|
||||
== "EC2 Instance i-0123456789abcdef0 is using outdated AMI ami-87654321."
|
||||
)
|
||||
assert not any(
|
||||
call.get("Owners") == ["amazon"] for call in describe_images_calls
|
||||
)
|
||||
assert any(
|
||||
call.get("ImageIds") == ["ami-87654321"]
|
||||
for call in describe_images_calls
|
||||
)
|
||||
|
||||
@mock.patch(
|
||||
"botocore.client.BaseClient._make_api_call", new=mock_make_api_call_missing_ami
|
||||
@@ -269,3 +342,10 @@ class Test_ec2_instance_with_outdated_ami:
|
||||
result = check.execute()
|
||||
|
||||
assert result == []
|
||||
assert not any(
|
||||
call.get("Owners") == ["amazon"] for call in describe_images_calls
|
||||
)
|
||||
assert any(
|
||||
call.get("ImageIds") == ["ami-missing"]
|
||||
for call in describe_images_calls
|
||||
)
|
||||
|
||||
+9
-3
@@ -39,6 +39,12 @@ def mock_make_api_call(self, operation_name, kwarg):
|
||||
return make_api_call(self, operation_name, kwarg)
|
||||
|
||||
|
||||
def get_mocked_ec2_client():
|
||||
ec2_client = mock.MagicMock()
|
||||
ec2_client.audit_config = {}
|
||||
return ec2_client
|
||||
|
||||
|
||||
class Test_ec2_launch_template_no_public_ip:
|
||||
@mock_aws
|
||||
def test_no_launch_templates(self):
|
||||
@@ -124,7 +130,7 @@ class Test_ec2_launch_template_no_public_ip:
|
||||
assert result[0].resource_tags == []
|
||||
|
||||
def test_launch_template_public_ip_auto_assign(self):
|
||||
ec2_client = mock.MagicMock()
|
||||
ec2_client = get_mocked_ec2_client()
|
||||
launch_template_name = "tester"
|
||||
launch_template_id = "lt-1234567890"
|
||||
launch_template_arn = (
|
||||
@@ -190,7 +196,7 @@ class Test_ec2_launch_template_no_public_ip:
|
||||
def test_network_interface_with_public_ipv4_network_interface_autoassign_true_and_false(
|
||||
self,
|
||||
):
|
||||
ec2_client = mock.MagicMock()
|
||||
ec2_client = get_mocked_ec2_client()
|
||||
launch_template_name = "tester"
|
||||
launch_template_id = "lt-1234567890"
|
||||
launch_template_arn = (
|
||||
@@ -290,7 +296,7 @@ class Test_ec2_launch_template_no_public_ip:
|
||||
def test_network_interface_with_public_ipv6_network_interface_autoassign_true_and_false(
|
||||
self,
|
||||
):
|
||||
ec2_client = mock.MagicMock()
|
||||
ec2_client = get_mocked_ec2_client()
|
||||
launch_template_name = "tester"
|
||||
launch_template_id = "lt-1234567890"
|
||||
launch_template_arn = (
|
||||
|
||||
@@ -6,12 +6,17 @@ from datetime import datetime
|
||||
import botocore
|
||||
import mock
|
||||
from boto3 import client, resource
|
||||
from botocore.exceptions import ClientError
|
||||
from dateutil.tz import tzutc
|
||||
from freezegun import freeze_time
|
||||
from moto import mock_aws
|
||||
|
||||
from prowler.config.config import encoding_format_utf_8
|
||||
from prowler.providers.aws.services.ec2.ec2_service import EC2, Snapshot
|
||||
from prowler.providers.aws.services.ec2.ec2_service import (
|
||||
DESCRIBE_IMAGES_IMAGE_IDS_BATCH_SIZE,
|
||||
EC2,
|
||||
Snapshot,
|
||||
)
|
||||
from tests.providers.aws.utils import (
|
||||
AWS_ACCOUNT_NUMBER,
|
||||
AWS_REGION_EU_WEST_1,
|
||||
@@ -196,6 +201,123 @@ class Test_EC2_Service:
|
||||
assert ec2.volumes_with_snapshots == {"vol-old": True, "vol-new": True}
|
||||
assert [snapshot.id for snapshot in ec2.snapshots] == ["snap-new"]
|
||||
|
||||
def test_describe_images_by_id_preserves_valid_amis_when_batch_has_missing_ids(
|
||||
self,
|
||||
):
|
||||
missing_image_id = "ami-0000-missing"
|
||||
valid_image_ids = [
|
||||
f"ami-{index:04d}"
|
||||
for index in range(1, DESCRIBE_IMAGES_IMAGE_IDS_BATCH_SIZE + 1)
|
||||
]
|
||||
instance_image_ids = valid_image_ids + [missing_image_id]
|
||||
|
||||
class FakeInstance:
|
||||
def __init__(self, image_id):
|
||||
self.region = AWS_REGION_US_EAST_1
|
||||
self.image_id = image_id
|
||||
|
||||
class FakeEC2Client:
|
||||
region = AWS_REGION_US_EAST_1
|
||||
|
||||
def __init__(self):
|
||||
self.image_id_calls = []
|
||||
|
||||
def describe_images(self, **kwargs):
|
||||
if kwargs.get("Owners") == ["self"]:
|
||||
return {"Images": []}
|
||||
|
||||
image_ids = kwargs["ImageIds"]
|
||||
self.image_id_calls.append(image_ids)
|
||||
if missing_image_id in image_ids:
|
||||
raise ClientError(
|
||||
{
|
||||
"Error": {
|
||||
"Code": "InvalidAMIID.NotFound",
|
||||
"Message": "The image id does not exist",
|
||||
}
|
||||
},
|
||||
"DescribeImages",
|
||||
)
|
||||
|
||||
return {
|
||||
"Images": [
|
||||
{
|
||||
"ImageId": image_id,
|
||||
"Public": True,
|
||||
"ImageOwnerAlias": "amazon",
|
||||
}
|
||||
for image_id in image_ids
|
||||
]
|
||||
}
|
||||
|
||||
regional_client = FakeEC2Client()
|
||||
ec2 = EC2.__new__(EC2)
|
||||
ec2.instances = [FakeInstance(image_id) for image_id in instance_image_ids]
|
||||
ec2.images = []
|
||||
ec2.images_by_id = {}
|
||||
ec2.audit_resources = []
|
||||
ec2.audited_partition = "aws"
|
||||
ec2.audited_account = AWS_ACCOUNT_NUMBER
|
||||
|
||||
ec2._describe_images(regional_client)
|
||||
|
||||
assert (
|
||||
len(regional_client.image_id_calls[0])
|
||||
== DESCRIBE_IMAGES_IMAGE_IDS_BATCH_SIZE
|
||||
)
|
||||
assert regional_client.image_id_calls[-1] == [valid_image_ids[-1]]
|
||||
assert missing_image_id not in ec2.images_by_id
|
||||
assert set(ec2.images_by_id) == set(valid_image_ids)
|
||||
assert {image.owner for image in ec2.images} == {"amazon"}
|
||||
|
||||
def test_describe_images_ignores_non_amazon_public_images_returned_by_image_id(
|
||||
self,
|
||||
):
|
||||
marketplace_image_id = "ami-marketplace-public"
|
||||
vendor_image_id = "ami-vendor-public"
|
||||
|
||||
class FakeInstance:
|
||||
def __init__(self, image_id):
|
||||
self.region = AWS_REGION_US_EAST_1
|
||||
self.image_id = image_id
|
||||
|
||||
class FakeEC2Client:
|
||||
region = AWS_REGION_US_EAST_1
|
||||
|
||||
def describe_images(self, **kwargs):
|
||||
if kwargs.get("Owners") == ["self"]:
|
||||
return {"Images": []}
|
||||
|
||||
return {
|
||||
"Images": [
|
||||
{
|
||||
"ImageId": marketplace_image_id,
|
||||
"Public": True,
|
||||
"ImageOwnerAlias": "aws-marketplace",
|
||||
},
|
||||
{
|
||||
"ImageId": vendor_image_id,
|
||||
"Public": True,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
ec2 = EC2.__new__(EC2)
|
||||
ec2.instances = [
|
||||
FakeInstance(marketplace_image_id),
|
||||
FakeInstance(vendor_image_id),
|
||||
]
|
||||
ec2.images = []
|
||||
ec2.images_by_id = {}
|
||||
ec2.audit_resources = []
|
||||
ec2.audited_partition = "aws"
|
||||
ec2.audited_account = AWS_ACCOUNT_NUMBER
|
||||
|
||||
ec2._describe_images(FakeEC2Client())
|
||||
|
||||
assert ec2.images == []
|
||||
assert ec2.images_by_id == {}
|
||||
|
||||
# Test EC2 Describe Instances
|
||||
@mock_aws
|
||||
@freeze_time(MOCK_DATETIME)
|
||||
@@ -743,9 +865,10 @@ class Test_EC2_Service:
|
||||
{"Key": "OS_Version", "Value": "AWS Linux 2"},
|
||||
]
|
||||
|
||||
# Verify that Amazon images are also present
|
||||
amazon_images = [img for img in ec2.images if img.owner == "amazon"]
|
||||
assert len(amazon_images) > 0 # Should have Amazon AMIs
|
||||
# Amazon public AMIs are fetched by targeted instance ImageIds only when
|
||||
# AWS identifies them as Amazon-owned. Moto does not expose that owner
|
||||
# alias for its fixture AMIs, so this service test only verifies the
|
||||
# self-owned AMI behavior used by ec2_ami_public.
|
||||
|
||||
# Test EC2 Describe Volumes
|
||||
@mock_aws
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import patch
|
||||
|
||||
import botocore
|
||||
@@ -136,6 +137,110 @@ def mock_generate_multi_region_clients(provider, service):
|
||||
}
|
||||
|
||||
|
||||
def mock_make_api_call_task_definitions_by_registration_date(
|
||||
self, operation_name, kwarg
|
||||
):
|
||||
task_definition_dates = {
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/eu-old:1": datetime(
|
||||
2024, 1, 1, tzinfo=timezone.utc
|
||||
),
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/eu-second-newest:1": datetime(
|
||||
2024, 5, 1, tzinfo=timezone.utc
|
||||
),
|
||||
f"arn:aws:ecs:{AWS_REGION_US_EAST_1}:123456789012:task-definition/us-newest:1": datetime(
|
||||
2024, 6, 1, tzinfo=timezone.utc
|
||||
),
|
||||
f"arn:aws:ecs:{AWS_REGION_US_EAST_1}:123456789012:task-definition/us-old:1": datetime(
|
||||
2024, 2, 1, tzinfo=timezone.utc
|
||||
),
|
||||
}
|
||||
task_definitions_by_region = {
|
||||
AWS_REGION_EU_WEST_1: [
|
||||
task_definition
|
||||
for task_definition in task_definition_dates
|
||||
if f":{AWS_REGION_EU_WEST_1}:" in task_definition
|
||||
],
|
||||
AWS_REGION_US_EAST_1: [
|
||||
task_definition
|
||||
for task_definition in task_definition_dates
|
||||
if f":{AWS_REGION_US_EAST_1}:" in task_definition
|
||||
],
|
||||
}
|
||||
|
||||
if operation_name == "ListTaskDefinitions":
|
||||
return {"taskDefinitionArns": task_definitions_by_region[self.region]}
|
||||
if operation_name == "DescribeTaskDefinition":
|
||||
return {
|
||||
"taskDefinition": {
|
||||
"containerDefinitions": [],
|
||||
"registeredAt": task_definition_dates[kwarg["taskDefinition"]],
|
||||
},
|
||||
"tags": [],
|
||||
}
|
||||
if operation_name == "ListClusters":
|
||||
return {"clusterArns": []}
|
||||
return make_api_call(self, operation_name, kwarg)
|
||||
|
||||
|
||||
def mock_make_api_call_task_definitions_with_equal_registration_dates(
|
||||
self, operation_name, kwarg
|
||||
):
|
||||
registered_at = datetime(2024, 1, 1, tzinfo=timezone.utc)
|
||||
task_definition_dates = {
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/zzz-task:1": registered_at,
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/aaa-task:1": registered_at,
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/mmm-task:1": registered_at,
|
||||
}
|
||||
|
||||
if operation_name == "ListTaskDefinitions":
|
||||
return {"taskDefinitionArns": list(task_definition_dates)}
|
||||
if operation_name == "DescribeTaskDefinition":
|
||||
return {
|
||||
"taskDefinition": {
|
||||
"containerDefinitions": [],
|
||||
"registeredAt": task_definition_dates[kwarg["taskDefinition"]],
|
||||
},
|
||||
"tags": [],
|
||||
}
|
||||
if operation_name == "ListClusters":
|
||||
return {"clusterArns": []}
|
||||
return make_api_call(self, operation_name, kwarg)
|
||||
|
||||
|
||||
DESCRIBED_TASK_DEFINITIONS = []
|
||||
|
||||
|
||||
def mock_make_api_call_task_definitions_with_audit_resources(
|
||||
self, operation_name, kwarg
|
||||
):
|
||||
task_definition_dates = {
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/audited-older:1": datetime(
|
||||
2024, 1, 1, tzinfo=timezone.utc
|
||||
),
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/audited-newer:1": datetime(
|
||||
2024, 2, 1, tzinfo=timezone.utc
|
||||
),
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/unaudited-newest:1": datetime(
|
||||
2024, 3, 1, tzinfo=timezone.utc
|
||||
),
|
||||
}
|
||||
|
||||
if operation_name == "ListTaskDefinitions":
|
||||
return {"taskDefinitionArns": list(task_definition_dates)}
|
||||
if operation_name == "DescribeTaskDefinition":
|
||||
DESCRIBED_TASK_DEFINITIONS.append(kwarg["taskDefinition"])
|
||||
return {
|
||||
"taskDefinition": {
|
||||
"containerDefinitions": [],
|
||||
"registeredAt": task_definition_dates[kwarg["taskDefinition"]],
|
||||
},
|
||||
"tags": [],
|
||||
}
|
||||
if operation_name == "ListClusters":
|
||||
return {"clusterArns": []}
|
||||
return make_api_call(self, operation_name, kwarg)
|
||||
|
||||
|
||||
@patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.generate_regional_clients",
|
||||
new=mock_generate_regional_clients,
|
||||
@@ -292,9 +397,9 @@ class Test_ECS_Service:
|
||||
ecs = ECS(aws_provider)
|
||||
|
||||
assert [td.revision for td in ecs.task_definitions.values()] == ["3", "2"]
|
||||
assert len(describe_calls) == 2
|
||||
assert len(describe_calls) == 3
|
||||
|
||||
def test_task_definition_limit_bounds_describe_calls(self):
|
||||
def test_task_definition_limit_describes_candidates_before_exposing_limit(self):
|
||||
describe_calls = []
|
||||
|
||||
def counting_make_api_call(self, operation_name, kwarg):
|
||||
@@ -326,9 +431,82 @@ class Test_ECS_Service:
|
||||
ecs = ECS(aws_provider)
|
||||
|
||||
assert [td.revision for td in ecs.task_definitions.values()] == ["3"]
|
||||
assert describe_calls == [
|
||||
"arn:aws:ecs:eu-west-1:123456789012:task-definition/fam:3"
|
||||
]
|
||||
assert sorted(describe_calls) == sorted(
|
||||
[
|
||||
"arn:aws:ecs:eu-west-1:123456789012:task-definition/fam:3",
|
||||
"arn:aws:ecs:eu-west-1:123456789012:task-definition/fam:2",
|
||||
"arn:aws:ecs:eu-west-1:123456789012:task-definition/fam:1",
|
||||
]
|
||||
)
|
||||
|
||||
@patch(
|
||||
"botocore.client.BaseClient._make_api_call",
|
||||
new=mock_make_api_call_task_definitions_by_registration_date,
|
||||
)
|
||||
def test_task_definition_limit_uses_global_latest_registration_dates(self):
|
||||
aws_provider = set_mocked_aws_provider(
|
||||
[AWS_REGION_EU_WEST_1, AWS_REGION_US_EAST_1],
|
||||
audit_config={"max_ecs_task_definitions": 2},
|
||||
)
|
||||
with patch(
|
||||
"prowler.providers.aws.aws_provider.AwsProvider.generate_regional_clients",
|
||||
new=mock_generate_multi_region_clients,
|
||||
):
|
||||
ecs = ECS(aws_provider)
|
||||
|
||||
assert list(ecs.task_definitions) == [
|
||||
f"arn:aws:ecs:{AWS_REGION_US_EAST_1}:123456789012:task-definition/us-newest:1",
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/eu-second-newest:1",
|
||||
]
|
||||
|
||||
@patch(
|
||||
"botocore.client.BaseClient._make_api_call",
|
||||
new=mock_make_api_call_task_definitions_with_equal_registration_dates,
|
||||
)
|
||||
def test_task_definition_limit_uses_arn_order_for_equal_registration_dates(self):
|
||||
aws_provider = set_mocked_aws_provider(
|
||||
[AWS_REGION_EU_WEST_1], audit_config={"max_ecs_task_definitions": 2}
|
||||
)
|
||||
ecs = ECS(aws_provider)
|
||||
|
||||
assert list(ecs.task_definitions) == [
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/aaa-task:1",
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:task-definition/mmm-task:1",
|
||||
]
|
||||
|
||||
@patch(
|
||||
"botocore.client.BaseClient._make_api_call",
|
||||
new=mock_make_api_call_task_definitions_with_audit_resources,
|
||||
)
|
||||
def test_task_definition_limit_applies_after_audit_resources_filter(self):
|
||||
DESCRIBED_TASK_DEFINITIONS.clear()
|
||||
audited_older_task_definition = (
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:"
|
||||
"task-definition/audited-older:1"
|
||||
)
|
||||
audited_newer_task_definition = (
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:"
|
||||
"task-definition/audited-newer:1"
|
||||
)
|
||||
unaudited_newest_task_definition = (
|
||||
f"arn:aws:ecs:{AWS_REGION_EU_WEST_1}:123456789012:"
|
||||
"task-definition/unaudited-newest:1"
|
||||
)
|
||||
aws_provider = set_mocked_aws_provider(
|
||||
[AWS_REGION_EU_WEST_1], audit_config={"max_ecs_task_definitions": 1}
|
||||
)
|
||||
aws_provider._audit_resources = [
|
||||
audited_older_task_definition,
|
||||
audited_newer_task_definition,
|
||||
]
|
||||
|
||||
ecs = ECS(aws_provider)
|
||||
|
||||
assert sorted(DESCRIBED_TASK_DEFINITIONS) == sorted(
|
||||
[audited_older_task_definition, audited_newer_task_definition]
|
||||
)
|
||||
assert list(ecs.task_definitions) == [audited_newer_task_definition]
|
||||
assert unaudited_newest_task_definition not in ecs.task_definitions
|
||||
|
||||
def test_task_definition_limit_does_not_starve_later_regions(self):
|
||||
describe_calls = []
|
||||
@@ -381,6 +559,8 @@ class Test_ECS_Service:
|
||||
]
|
||||
assert set(describe_calls) == {
|
||||
"arn:aws:ecs:eu-west-1:123456789012:task-definition/fam:3",
|
||||
"arn:aws:ecs:eu-west-1:123456789012:task-definition/fam:2",
|
||||
"arn:aws:ecs:eu-west-1:123456789012:task-definition/fam:1",
|
||||
"arn:aws:ecs:us-east-1:123456789012:task-definition/fam:9",
|
||||
}
|
||||
|
||||
|
||||
+138
@@ -17,6 +17,10 @@ def scp_restrict_regions_with_deny():
|
||||
return '{"Version":"2012-10-17","Statement":{"Effect":"Deny","NotAction":"s3:*","Resource":"*","Condition":{"StringNotEquals":{"aws:RequestedRegion":["eu-central-1","eu-west-1"]}}}}'
|
||||
|
||||
|
||||
def scp_restrict_regions_with_allow():
|
||||
return '{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":"*","Resource":"*","Condition":{"StringEquals":{"aws:RequestedRegion":["eu-central-1","eu-west-1"]}}}}'
|
||||
|
||||
|
||||
def scp_restrict_regions_without_statement():
|
||||
return '{"Version":"2012-10-17"}'
|
||||
|
||||
@@ -352,3 +356,137 @@ class Test_organizations_scp_check_deny_regions:
|
||||
== f"AWS Organization {org_id} has SCP policies but don't restrict AWS Regions."
|
||||
)
|
||||
assert result[0].region == AWS_REGION_EU_WEST_1
|
||||
|
||||
@mock_aws
|
||||
def test_organization_with_scp_allow_regions_valid(self):
|
||||
aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1])
|
||||
|
||||
conn = client("organizations", region_name=AWS_REGION_EU_WEST_1)
|
||||
response = conn.describe_organization()
|
||||
response_policy = conn.create_policy(
|
||||
Content=scp_restrict_regions_with_allow(),
|
||||
Description="Test",
|
||||
Name="Test",
|
||||
Type="SERVICE_CONTROL_POLICY",
|
||||
)
|
||||
org_id = response["Organization"]["Id"]
|
||||
policy_id = response_policy["Policy"]["PolicySummary"]["Id"]
|
||||
|
||||
aws_provider._audit_config = {"organizations_enabled_regions": ["eu-central-1"]}
|
||||
|
||||
with mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
):
|
||||
with mock.patch(
|
||||
"prowler.providers.aws.services.organizations.organizations_scp_check_deny_regions.organizations_scp_check_deny_regions.organizations_client",
|
||||
new=Organizations(aws_provider),
|
||||
):
|
||||
from prowler.providers.aws.services.organizations.organizations_scp_check_deny_regions.organizations_scp_check_deny_regions import (
|
||||
organizations_scp_check_deny_regions,
|
||||
)
|
||||
|
||||
check = organizations_scp_check_deny_regions()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].status == "PASS"
|
||||
assert result[0].resource_id == response["Organization"]["Id"]
|
||||
assert result[0].resource_arn == response["Organization"]["Arn"]
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== f"AWS Organization {org_id} has SCP policy {policy_id} restricting all configured regions found."
|
||||
)
|
||||
assert result[0].region == AWS_REGION_EU_WEST_1
|
||||
|
||||
@mock_aws
|
||||
def test_organization_with_scp_allow_regions_not_valid(self):
|
||||
aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1])
|
||||
|
||||
conn = client("organizations", region_name=AWS_REGION_EU_WEST_1)
|
||||
response = conn.describe_organization()
|
||||
response_policy = conn.create_policy(
|
||||
Content=scp_restrict_regions_with_allow(),
|
||||
Description="Test",
|
||||
Name="Test",
|
||||
Type="SERVICE_CONTROL_POLICY",
|
||||
)
|
||||
org_id = response["Organization"]["Id"]
|
||||
policy_id = response_policy["Policy"]["PolicySummary"]["Id"]
|
||||
|
||||
aws_provider._audit_config = {"organizations_enabled_regions": ["us-east-1"]}
|
||||
|
||||
with mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
):
|
||||
with mock.patch(
|
||||
"prowler.providers.aws.services.organizations.organizations_scp_check_deny_regions.organizations_scp_check_deny_regions.organizations_client",
|
||||
new=Organizations(aws_provider),
|
||||
):
|
||||
from prowler.providers.aws.services.organizations.organizations_scp_check_deny_regions.organizations_scp_check_deny_regions import (
|
||||
organizations_scp_check_deny_regions,
|
||||
)
|
||||
|
||||
check = organizations_scp_check_deny_regions()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].status == "FAIL"
|
||||
assert result[0].resource_id == response["Organization"]["Id"]
|
||||
assert (
|
||||
"arn:aws:organizations::123456789012:organization/o-"
|
||||
in result[0].resource_arn
|
||||
)
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== f"AWS Organization {org_id} has SCP policies {policy_id} restricting some AWS Regions, but not all the configured ones, please check config."
|
||||
)
|
||||
assert result[0].region == AWS_REGION_EU_WEST_1
|
||||
|
||||
@mock_aws
|
||||
def test_organization_with_scp_allow_all_regions_valid(self):
|
||||
aws_provider = set_mocked_aws_provider([AWS_REGION_EU_WEST_1])
|
||||
|
||||
conn = client("organizations", region_name=AWS_REGION_EU_WEST_1)
|
||||
response = conn.describe_organization()
|
||||
response_policy = conn.create_policy(
|
||||
Content=scp_restrict_regions_with_allow(),
|
||||
Description="Test",
|
||||
Name="Test",
|
||||
Type="SERVICE_CONTROL_POLICY",
|
||||
)
|
||||
org_id = response["Organization"]["Id"]
|
||||
policy_id = response_policy["Policy"]["PolicySummary"]["Id"]
|
||||
|
||||
aws_provider._audit_config = {
|
||||
"organizations_enabled_regions": [
|
||||
AWS_REGION_EU_WEST_1,
|
||||
AWS_REGION_EU_CENTRAL_1,
|
||||
]
|
||||
}
|
||||
|
||||
with mock.patch(
|
||||
"prowler.providers.common.provider.Provider.get_global_provider",
|
||||
return_value=aws_provider,
|
||||
):
|
||||
with mock.patch(
|
||||
"prowler.providers.aws.services.organizations.organizations_scp_check_deny_regions.organizations_scp_check_deny_regions.organizations_client",
|
||||
new=Organizations(aws_provider),
|
||||
):
|
||||
from prowler.providers.aws.services.organizations.organizations_scp_check_deny_regions.organizations_scp_check_deny_regions import (
|
||||
organizations_scp_check_deny_regions,
|
||||
)
|
||||
|
||||
check = organizations_scp_check_deny_regions()
|
||||
result = check.execute()
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].status == "PASS"
|
||||
assert result[0].resource_id == response["Organization"]["Id"]
|
||||
assert result[0].resource_arn == response["Organization"]["Arn"]
|
||||
assert (
|
||||
result[0].status_extended
|
||||
== f"AWS Organization {org_id} has SCP policy {policy_id} restricting all configured regions found."
|
||||
)
|
||||
assert result[0].region == AWS_REGION_EU_WEST_1
|
||||
|
||||
+2
-2
@@ -87,7 +87,7 @@ class Test_app_function_access_keys_configured:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={},
|
||||
environment_variables={},
|
||||
identity=None,
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
@@ -143,7 +143,7 @@ class Test_app_function_access_keys_configured:
|
||||
"default": "key1",
|
||||
"key2": "key2",
|
||||
},
|
||||
enviroment_variables={},
|
||||
environment_variables={},
|
||||
identity=None,
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
|
||||
+8
-4
@@ -87,7 +87,7 @@ class Test_app_function_application_insights_enabled:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={},
|
||||
environment_variables={},
|
||||
identity=None,
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
@@ -138,7 +138,9 @@ class Test_app_function_application_insights_enabled:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={"APPINSIGHTS_INSTRUMENTATIONKEY": "1234"},
|
||||
environment_variables={
|
||||
"APPINSIGHTS_INSTRUMENTATIONKEY": "1234"
|
||||
},
|
||||
identity=None,
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
@@ -189,7 +191,9 @@ class Test_app_function_application_insights_enabled:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={"APPINSIGHTS_INSTRUMENTATIONKEY": "1234"},
|
||||
environment_variables={
|
||||
"APPINSIGHTS_INSTRUMENTATIONKEY": "1234"
|
||||
},
|
||||
identity=None,
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
@@ -240,7 +244,7 @@ class Test_app_function_application_insights_enabled:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={},
|
||||
environment_variables={},
|
||||
identity=None,
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
|
||||
+3
-3
@@ -87,7 +87,7 @@ class Test_app_function_ftps_deployment_disabled:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={},
|
||||
environment_variables={},
|
||||
identity=mock.MagicMock(type="SystemAssigned"),
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
@@ -138,7 +138,7 @@ class Test_app_function_ftps_deployment_disabled:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={},
|
||||
environment_variables={},
|
||||
identity=mock.MagicMock(type="SystemAssigned"),
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
@@ -189,7 +189,7 @@ class Test_app_function_ftps_deployment_disabled:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={},
|
||||
environment_variables={},
|
||||
identity=mock.MagicMock(type="SystemAssigned"),
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
|
||||
+2
-2
@@ -87,7 +87,7 @@ class Test_app_function_identity_is_configured:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={},
|
||||
environment_variables={},
|
||||
identity=None,
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
@@ -138,7 +138,7 @@ class Test_app_function_identity_is_configured:
|
||||
location="West Europe",
|
||||
kind="functionapp,linux",
|
||||
function_keys={},
|
||||
enviroment_variables={},
|
||||
environment_variables={},
|
||||
identity=mock.MagicMock(type="SystemAssigned"),
|
||||
public_access=False,
|
||||
vnet_subnet_id=None,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user